x/opencafe
Code Issues Proposed changes

Initial Commit

Browse Source
This commit is contained in:
samu4924
2013-03-30 14:47:00 -05:00
commit c177eb7cce
52 changed files with 4532 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
cafe/engine/__init__.py Normal file
View File

@@ -0,0 +1,15 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

76
cafe/engine/behaviors.py Normal file
View File

@@ -0,0 +1,76 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import decorator
from cafe.common.reporting import cclogging
class RequiredClientNotDefinedError(Exception):
"""Raised when a behavior method call can't find a required client """
pass
def behavior(*required_clients):
'''Decorator that tags method as a behavior, and optionally adds
required client objects to an internal attribute. Causes calls to this
method to throw RequiredClientNotDefinedError exception if the containing
class does not have the proper client instances defined.
'''
#@decorator.decorator
def _decorator(func):
#Unused for now
setattr(func, '__is_behavior__', True)
setattr(func, '__required_clients__', [])
for client in required_clients:
func.__required_clients__.append(client)
def _wrap(self, *args, **kwargs):
available_attributes = vars(self)
missing_clients = []
all_requirements_satisfied = True
if required_clients:
for required_client in required_clients:
required_client_found = False
for attr in available_attributes:
attribute = getattr(self, attr, None)
if isinstance(attribute, required_client):
required_client_found = True
break
all_requirements_satisfied = (
all_requirements_satisfied and
required_client_found)
missing_clients.append(required_client)
if not all_requirements_satisfied:
msg_plurality = ("an instance" if len(missing_clients) <= 1
else "instances")
msg = ("Behavior {0} expected {1} of {2} but couldn't"
" find one".format(
func, msg_plurality, missing_clients))
raise RequiredClientNotDefinedError(msg)
return func(self, *args, **kwargs)
return _wrap
return _decorator
class BaseBehavior(object):
def __init__(self):
self._log = cclogging.getLogger(
cclogging.get_object_namespace(self.__class__))

16
cafe/engine/clients/__init__.py Normal file
View File

@@ -0,0 +1,16 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

21
cafe/engine/clients/base.py Normal file
View File

@@ -0,0 +1,21 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from cafe.common.reporting import cclogging
class BaseClient(object):
_log = cclogging.getLogger(__name__)

158
cafe/engine/clients/commandline.py Normal file
View File

@@ -0,0 +1,158 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
'''Provides low level connectivity to the commandline via popen()
@note: Primarily intended to serve as base classes for a specific
command line client Class
'''
import os
import sys
import subprocess
from cafe.engine.models.commandline_response import CommandLineResponse
from cafe.engine.clients.base import BaseClient
class BaseCommandLineClient(BaseClient):
'''Wrapper for driving/parsing a command line program
@ivar base_command: This processes base command string. (I.E. 'ls', 'pwd')
@type base_command: C{str}
@note: This class is dependent on a local installation of the wrapped
client process. The thing you run has to be there!
'''
def __init__(self, base_command=None, env_var_dict=None):
'''
@param base_command: This processes base command string.
(I.E. 'ls', 'pwd')
@type base_command: C{str}
'''
super(BaseCommandLineClient, self).__init__()
self.base_command = base_command
self.env_var_dict = env_var_dict or {}
self.set_environment_variables(self.env_var_dict)
def set_environment_variables(self, env_var_dict=None):
'''Sets all os environment variables provided in env_var_dict'''
for key, value in env_var_dict.items():
self._log.debug('setting {0}={1}'.format(key, value))
os.putenv(str(key), str(value))
def unset_environment_variables(self, env_var_list=None):
'''Unsets all os environment variables provided in env_var_dict
by default.
If env_var_list is passed, attempts to unset all environment vars in
list'''
env_var_list = env_var_list or self.env_var_dict.keys() or []
for key, _ in env_var_list:
self._log.debug('unsetting {0}'.format(key))
os.unsetenv(str(key))
def run_command(self, cmd, *args):
'''Sends a command directly to this instance's command line
@param cmd: Command to sent to command line
@type cmd: C{str}
@param args: Optional list of args to be passed with the command
@type args: C{list}
@raise exception: If unable to close process after running the command
@return: The full response details from the command line
@rtype: L{CommandLineResponse}
@note: PRIVATE. Can be over-ridden in a child class
'''
os_process = None
os_response = CommandLineResponse()
#Process command we received
os_response.command = "{0} {1}".format(self.base_command, cmd)
if args and args[0]:
for arg in args[0]:
os_response.command += "{0} {1}".format(
os_response.command, arg)
"""@TODO: Turn this into a decorator like the rest client"""
try:
logline = ''.join([
'\n{0}\nCOMMAND LINE REQUEST\n{0}\n'.format('-' * 4),
"args..........: {0}".format(args),
"command.......: {0}".format(os_response.command)])
except Exception as exception:
self._log.exception(exception)
try:
self._log.debug(logline.decode('utf-8', 'replace'))
except Exception as exception:
#Ignore all exceptions that happen in logging, then log them
self._log.debug('\n{0}\nCOMMAND LINE REQUEST INFO\n{0}\n'.format(
'-' * 12))
self._log.exception(exception)
#Run the command
try:
os_process = subprocess.Popen(os_response.command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
shell=True)
except subprocess.CalledProcessError() as cpe:
self._log.exception(
"Exception running commandline command {0}\n{1}".format(
str(os_response.command), str(cpe)))
#Wait for the process to complete and then read the lines.
#for some reason if you read each line as the process is running
#and use os_process.Poll() you don't always get all output
std_out, std_err = os_process.communicate()
os_response.return_code = os_process.returncode
#Pass the full output of the process_command back. It is important to
#not parse, strip or otherwise massage this output in the private send
#since a child class could override and contain actual command
#processing logic.
os_response.standard_out = str(std_out).splitlines()
if std_err is not None:
os_response.standard_error = str(std_err).splitlines()
"""@TODO: Turn this into a decorator like in the rest client"""
try:
logline = ''.join([
'\n{0}\nCOMMAND LINE RESPONSE\n{0}\n'.format('-' * 4),
"standard out...: {0}".format(os_response.standard_out),
"standard error.: {0}".format(os_response.standard_error),
"return code....: {0}".format(os_response.return_code)])
except Exception as exception:
self._log.exception(exception)
try:
self._log.debug(logline.decode('utf-8', 'replace'))
except Exception as exception:
#Ignore all exceptions that happen in logging, then log them
self._log.debug('\n{0}\nCOMMAND LINE RESPONSE INFO\n{0}\n'.format(
'-' * 12))
self._log.exception(exception)
#Clean up the process to avoid any leakage/wonkiness with stdout/stderr
try:
os_process.kill()
except OSError:
#An OS Error is valid if the process has exited. We only
#need to be concerned about other exceptions
sys.exc_clear()
except Exception, kill_exception:
raise Exception(
"Exception forcing %s Process to close: {0}".format(
self.base_command, kill_exception))
finally:
del os_process
return os_response

64
cafe/engine/clients/ping.py Normal file
View File

@@ -0,0 +1,64 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import subprocess
import re
from cloudcafe.common.constants import InstanceClientConstants
class PingClient(object):
"""
@summary: Client to ping windows or linux servers
"""
@classmethod
def ping(cls, ip, ip_address_version_for_ssh):
"""
@summary: Ping a server with a IP
@param ip: IP address to ping
@type ip: string
@return: True if the server was reachable, False otherwise
@rtype: bool
"""
'''
Porting only Linux OS
'''
ping_command = InstanceClientConstants.PING_IPV6_COMMAND_LINUX if ip_address_version_for_ssh == 6 else InstanceClientConstants.PING_IPV4_COMMAND_LINUX
command = ping_command + ip
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
process.wait()
try:
packet_loss_percent = re.search(InstanceClientConstants.PING_PACKET_LOSS_REGEX,
process.stdout.read()).group(1)
except:
# If there is no match, fail
return False
return packet_loss_percent != '100'
@classmethod
def ping_using_remote_machine(cls, remote_client, ping_ip_address):
"""
@summary: Ping a server using a remote machine
@param remote_client: Client to remote machine
@param ip: IP address to ping
@type ip: string
@return: True if the server was reachable, False otherwise
@rtype: bool
"""
command = InstanceClientConstants.PING_IPV4_COMMAND_LINUX
ping_response = remote_client.exec_command(command + ping_ip_address)
packet_loss_percent = re.search(InstanceClientConstants.PING_PACKET_LOSS_REGEX, ping_response).group(1)
return packet_loss_percent != '100'

16
cafe/engine/clients/remote_instance/__init__.py Normal file
View File

@@ -0,0 +1,16 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

156
cafe/engine/clients/remote_instance/instance_client.py Normal file
View File

@@ -0,0 +1,156 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from cafe.common.reporting import cclogging
from cafe.engine.clients.remote_instance.linux.linux_instance_client import LinuxClient
from cafe.engine.clients.remote_instance.windows.windows_instance_client import WindowsClient
class InstanceClientFactory(object):
"""
@summary: Factory class which will create appropriate utility object
based on the operating system of the server.
"""
clientList = {'windows': 'WindowsClient', 'linux': 'LinuxClient', 'gentoo': 'LinuxClient',
'arch': 'LinuxClient', 'freebsd': 'FreeBSDClient'}
@classmethod
def get_instance_client(cls, ip_address, username, password, os_distro, server_id):
"""
@summary: Returns utility class based on the OS type of server
@param ip_address: IP Address of the server
@type ip_address: string
@param password: The administrator user password
@type password: string
@param username: The administrator user name
@type username: string
@return: Utility class based on the OS type of server
@rtype: LinuxClient or WindowsClient
"""
instanceClient = cls.clientList.get(os_distro.lower())
if instanceClient is None:
instanceClient = cls.clientList.get(cls.os_type.lower())
target_str = "globals().get('" + instanceClient + "')"
instanceClient = eval(target_str)
return instanceClient(ip_address=ip_address, username=username,
password=password, os_distro=os_distro,
server_id=server_id)
class InstanceClient(object):
"""
@summary: Wrapper class around different operating system utilities.
"""
def __init__(self, ip_address, password, os_distro, username=None, server_id=None):
self._client = InstanceClientFactory.get_instance_client(ip_address, password, os_distro, username, server_id)
self.client_log = cclogging.getLogger(cclogging.get_object_namespace(self.__class__))
def can_authenticate(self):
"""
@summary: Checks if you can authenticate to the server
@return: True if you can connect, False otherwise
@rtype: bool
"""
return self._client.test_connection_auth()
def get_hostname(self):
"""
@summary: Gets the host name of the server
@return: The host name of the server
@rtype: string
"""
return self._client.get_hostname()
def get_files(self, path):
"""
@summary: Gets the list of filenames from the path
@param path: Path from where to get the filenames
@type path: string
@return: List of filenames
@rtype: List of strings
"""
return self._client.get_files(path)
def get_ram_size_in_mb(self):
"""
@summary: Returns the RAM size in MB
@return: The RAM size in MB
@rtype: string
"""
return self._client.get_ram_size_in_mb()
def get_disk_size_in_gb(self):
"""
@summary: Returns the disk size in GB
@return: The disk size in GB
@rtype: int
"""
return self._client.get_disk_size_in_gb()
def get_number_of_vcpus(self):
"""
@summary: Get the number of vcpus assigned to the server
@return: The number of vcpus assigned to the server
@rtype: int
"""
return self._client.get_number_of_vcpus()
def get_partitions(self):
"""
@summary: Returns the contents of /proc/partitions
@return: The partitions attached to the instance
@rtype: string
"""
return self._client.get_partitions()
def get_uptime(self):
"""
@summary: Get the boot time of the server
@return: The boot time of the server
@rtype: time.struct_time
"""
return self._client.get_uptime()
def create_file(self, filedetails):
'''
@summary: Create a new file
@param filedetails: File details such as content, name
@type filedetails; FileDetails
'''
return self._client.create_file()
def get_file_details(self, filepath):
"""
@summary: Get the file details
@param filepath: Path to the file
@type filepath: string
@return: File details including permissions and content
@rtype: FileDetails
"""
return self._client.get_file_details()
def is_file_present(self, filepath):
"""
@summary: Check if the given file is present
@param filepath: Path to the file
@type filepath: string
@return: True if File exists, False otherwise
"""
return self._client.is_file_present()

20
cafe/engine/clients/remote_instance/linux/__init__.py Normal file
View File

@@ -0,0 +1,20 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
LAST_REBOOT_TIME_FORMAT = '%Y-%m-%d %H:%M:%S'
PING_IPV4_COMMAND = 'ping -c 3 '
PING_IPV6_COMMAND = 'ping6 -c 3 '
PING_PACKET_LOSS_REGEX = '(\d{1,3})\.?\d*\% packet loss'

103
cafe/engine/clients/remote_instance/linux/base_client.py Normal file
View File

@@ -0,0 +1,103 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import re
import os
from cafe.engine.clients.ssh import SSHBaseClient
class BasePersistentLinuxClient(object):
def __init__(self, ip_address, username, password, ssh_timeout=600, prompt=None):
self.ssh_client = SSHBaseClient(ip_address, username, password, ssh_timeout)
self.prompt = prompt
if not self.ssh_client.test_connection_auth():
raise
def format_disk_device(self, device, fstype):
'''Formats entire device, does not create partitions'''
return self.ssh_client.exec_command("mkfs.%s %s\n" % (str(fstype).lower(), str(device)))
def mount_disk_device(self, device, mountpoint, fstype, create_mountpoint=True):
'''
Mounts a disk at a specified mountpoint. performs 'touch mountpoint' before executing
'''
self.ssh_client.exec_command("mkdir %s" % str(mountpoint))
return self.ssh_client.exec_command("mount -t %s %s %s\n" % (str(fstype).lower(), str(device), str(mountpoint)))
def unmount_disk_device(self, mountpoint):
'''
Forces unmounts (umount -f) a disk at a specified mountpoint.
'''
return self.ssh_client.exec_command("umount -f %s\n" % (str(mountpoint)))
def write_random_data_to_disk(self, dir_path, filename, blocksize=1024,
count=1024):
'''Uses dd command to write blocksize*count bytes to dir_path/filename
via ssh on remote machine.
By default writes one mebibyte (2^20 bytes) if blocksize and count
are not defined.
NOTE: 1 MEBIbyte (2^20) != 1 MEGAbyte (10^6) for all contexts
Note: dd if=/dev/urandom
'''
dd_of = os.path.join(dir_path, str(filename))
return self.ssh_client.exec_command(
"dd if=/dev/urandom of=%s bs=%s count=%s\n" %
(str(dd_of), str(blocksize), str(count)))
def write_zeroes_data_to_disk(self, disk_mountpoint, filename, blocksize=1024, count=1024):
'''By default writes one mebibyte (2^20 bytes)'''
of = '%s/%s' % (disk_mountpoint, str(filename))
return self.ssh_client.exec_command(
"dd if=/dev/zero of=%s bs=%s count=%s\n" %
(str(of), str(blocksize), str(count)))
def execute_resource_bomb(self):
'''By default executes :(){ :|:& };:'''
return self.ssh_client.exec_command(":(){ :|:& };:")
def stat_file(self, filepath):
sshresp = self.ssh_client.exec_command("stat %s\n" % str(filepath))
return sshresp
def get_file_size_bytes(self, filepath):
'''
Performs wc -c on path provided, returning the numerical count in
the result
'''
sshresp = self.ssh_client.exec_command("wc -c %s\n" % str(filepath))
result = re.search('^(.*)\s', sshresp)
try:
return result.groups()[0]
except:
return None
def get_file_md5hash(self, filepath):
'''
Performs binary mode md5sum of file and returns hash.
(md5sum -b <file>)
'''
sshresp = self.ssh_client.exec_command("md5sum -b %s\n" % str(filepath))
result = re.search('^(.*)\s', sshresp)
try:
return result.groups()[0]
except:
return None

51
cafe/engine/clients/remote_instance/linux/freebsd_client.py Normal file
View File

@@ -0,0 +1,51 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import time
import re
from cafe.engine.clients.remote_instance.linux.linux_instance_client import LinuxClient
from cloudcafe.common.constants import InstanceClientConstants
class FreeBSDClient(LinuxClient):
def get_boot_time(self):
"""
@summary: Get the boot time of the server
@return: The boot time of the server
@rtype: time.struct_time
"""
uptime_string = self.ssh_client.exec_command('uptime')
uptime = uptime_string.replace('\n', '').split(',')[0].split()[2]
uptime_unit = uptime_string.replace('\n', '').split(',')[0].split()[3]
if (uptime_unit == 'mins'):
uptime_unit_format = 'M'
else:
uptime_unit_format = 'S'
reboot_time = self.ssh_client.exec_command('date -v -' + uptime + uptime_unit_format + ' "+%Y-%m-%d %H:%M"').replace('\n', '')
return time.strptime(reboot_time, InstanceClientConstants.LAST_REBOOT_TIME_FORMAT)
def get_disk_size_in_gb(self):
"""
@summary: Returns the disk size in GB
@return: The disk size in GB
@rtype: int
"""
output = self.ssh_client.exec_command('gpart show -p | grep "GPT"').replace('\n', '')
disk_size = re.search(r'([0-9]+)G', output).group(1)
return int(disk_size)

34
cafe/engine/clients/remote_instance/linux/gentoo_arch_client.py Normal file
View File

@@ -0,0 +1,34 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import time
from cafe.engine.clients.remote_instance.linux.linux_instance_client import LinuxClient
from cloudcafe.common.constants import InstanceClientConstants
class GentooArchClient(LinuxClient):
def get_boot_time(self):
"""
@summary: Get the boot time of the server
@return: The boot time of the server
@rtype: time.struct_time
"""
boot_time_string = self.ssh_client.exec_command('who -b | grep -o "[A-Za-z]* [0-9].*"').replace('\n', ' ')
year = self.ssh_client.exec_command('date | grep -o "[0-9]\{4\}$"').replace('\n', '')
boot_time = boot_time_string + year
return time.strptime(boot_time, InstanceClientConstants.LAST_REBOOT_TIME_FORMAT_GENTOO)

366
cafe/engine/clients/remote_instance/linux/linux_instance_client.py Normal file
View File

@@ -0,0 +1,366 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import time
import re
from cafe.engine.clients.ssh import SSHBaseClient
from cafe.common.reporting import cclogging
from cafe.engine.clients.ping import PingClient
from cloudcafe.compute.common.models.file_details import FileDetails
from cloudcafe.compute.common.models.partition import Partition, DiskSize
from cafe.engine.clients.remote_instance.linux.base_client import BasePersistentLinuxClient
from cloudcafe.compute.common.exceptions import FileNotFoundException, ServerUnreachable, SshConnectionException
class LinuxClient(BasePersistentLinuxClient):
def __init__(self, ip_address, server_id, os_distro, username, password):
self.client_log = cclogging.getLogger \
(cclogging.get_object_namespace(self.__class__))
ssh_timeout = 600
if ip_address is None:
raise ServerUnreachable("None")
self.ip_address = ip_address
self.username = username
if self.username is None:
self.username = 'root'
self.password = password
self.server_id = server_id
self.ssh_client = SSHBaseClient(self.ip_address,
self.username,
self.password,
timeout=ssh_timeout)
if not self.ssh_client.test_connection_auth():
self.client_log.error("Ssh connection failed for: IP:{0} \
Username:{1} Password: {2}".format(self.ip_address,
self.username, self.password))
raise SshConnectionException("ssh connection failed")
def can_connect_to_public_ip(self):
"""
@summary: Checks if you can connect to server using public ip
@return: True if you can connect, False otherwise
@rtype: bool
"""
# This returns true since the connection has already been tested in the
# init method
return self.ssh_client is not None
def can_ping_public_ip(self, public_addresses, ip_address_version_for_ssh):
"""
@summary: Checks if you can ping a public ip
@param addresses: List of public addresses
@type addresses: Address List
@return: True if you can ping, False otherwise
@rtype: bool
"""
for public_address in public_addresses:
if public_address.version == 4 and not PingClient.ping(public_address.addr, ip_address_version_for_ssh):
return False
return True
def can_authenticate(self):
"""
@summary: Checks if you can authenticate to the server
@return: True if you can connect, False otherwise
@rtype: bool
"""
return self.ssh_client.test_connection_auth()
def reboot(self, timeout=100):
'''
@timeout: max timeout for the machine to reboot
'''
ssh_connector = SSHConnector(self.ip_address, self.username,
self.password)
response, prompt = ssh_connector.exec_shell_command("sudo reboot")
response, prompt = ssh_connector.exec_shell_command(self.password)
self.client_log.info("Reboot response for %s: %s" % (self.ip_address,
response))
max_time = time.time() + timeout
while time.time() < max_time:
time.sleep(5)
if self.ssh_client.test_connection_auth():
self.client_log.info("Reboot successful for %s"
% (self.ip_address))
return True
def get_hostname(self):
"""
@summary: Gets the host name of the server
@return: The host name of the server
@rtype: string
"""
return self.ssh_client.exec_command("hostname").rstrip()
def can_remote_ping_private_ip(self, private_addresses):
"""
@summary: Checks if you can ping a private ip from this server.
@param private_addresses: List of private addresses
@type private_addresses: Address List
@return: True if you can ping, False otherwise
@rtype: bool
"""
for private_address in private_addresses:
if private_address.version == 4 and not PingClient.ping_using_remote_machine(self.ssh_client, private_address.addr):
return False
return True
def get_files(self, path):
"""
@summary: Gets the list of filenames from the path
@param path: Path from where to get the filenames
@type path: string
@return: List of filenames
@rtype: List of strings
"""
command = "ls -m " + path
return self.ssh_client.exec_command(command).rstrip('\n').split(', ')
def get_ram_size_in_mb(self):
"""
@summary: Returns the RAM size in MB
@return: The RAM size in MB
@rtype: string
"""
output = self.ssh_client.exec_command('free -m | grep Mem')
# TODO (dwalleck): We should handle the failure case here
if output:
return output.split()[1]
def get_swap_size_in_mb(self):
"""
@summary: Returns the Swap size in MB
@return: The Swap size in MB
@rtype: int
"""
output = self.ssh_client.exec_command(
'fdisk -l /dev/xvdc1 2>/dev/null | grep "Disk.*bytes"').rstrip('\n')
if output:
return int(output.split()[2])
def get_disk_size_in_gb(self, disk_path):
"""
@summary: Returns the disk size in GB
@return: The disk size in GB
@rtype: int
"""
command = "df -h | grep '{0}'".format(disk_path)
output = self.ssh_client.exec_command(command)
size = output.split()[1]
def is_decimal(char):
return str.isdigit(char) or char == "."
size = filter(is_decimal, size)
return float(size)
def get_number_of_vcpus(self):
"""
@summary: Get the number of vcpus assigned to the server
@return: The number of vcpus assigned to the server
@rtype: int
"""
command = 'cat /proc/cpuinfo | grep processor | wc -l'
output = self.ssh_client.exec_command(command)
return int(output)
def get_partitions(self):
"""
@summary: Returns the contents of /proc/partitions
@return: The partitions attached to the instance
@rtype: string
"""
command = 'cat /proc/partitions'
output = self.ssh_client.exec_command(command)
return output
def get_uptime(self):
"""
@summary: Get the uptime time of the server
@return: The uptime of the server
"""
result = self.ssh_client.exec_command('cat /proc/uptime')
uptime = float(result.split(' ')[0])
return uptime
def create_file(self, file_name, file_content, file_path=None):
'''
@summary: Create a new file
@param file_name: File Name
@type file_name: String
@param file_content: File Content
@type file_content: String
@return filedetails: File details such as content, name and path
@rtype filedetails; FileDetails
'''
if file_path is None:
file_path = "/root/" + file_name
self.ssh_client.exec_command(
'echo -n ' + file_content + '>>' + file_path)
return FileDetails("644", file_content, file_path)
def get_file_details(self, filepath):
"""
@summary: Get the file details
@param filepath: Path to the file
@type filepath: string
@return: File details including permissions and content
@rtype: FileDetails
"""
output = self.ssh_client.exec_command(
'[ -f ' + filepath + ' ] && echo "File exists" || echo "File does not exist"')
if not output.rstrip('\n') == 'File exists':
raise FileNotFoundException(
"File:" + filepath + " not found on instance.")
file_permissions = self.ssh_client.exec_command(
'stat -c %a ' + filepath).rstrip("\n")
file_contents = self.ssh_client.exec_command('cat ' + filepath)
return FileDetails(file_permissions, file_contents, filepath)
def is_file_present(self, filepath):
"""
@summary: Check if the given file is present
@param filepath: Path to the file
@type filepath: string
@return: True if File exists, False otherwise
"""
output = self.ssh_client.exec_command(
'[ -f ' + filepath + ' ] && echo "File exists" || echo "File does not exist"')
return output.rstrip('\n') == 'File exists'
def get_partition_types(self):
"""
@summary: Return the partition types for all partitions
@return: The partition types for all partitions
@rtype: Dictionary
"""
partitions_list = self.ssh_client.exec_command(
'blkid').rstrip('\n').split('\n')
partition_types = {}
for row in partitions_list:
partition_name = row.split()[0].rstrip(':')
partition_types[partition_name] = re.findall(
r'TYPE="([^"]+)"', row)[0]
return partition_types
def get_partition_details(self):
"""
@summary: Return the partition details
@return: The partition details
@rtype: Partition List
"""
# Return a list of partition objects that each contains the name and
# size of the partition in bytes and the type of the partition
partition_types = self.get_partition_types()
partition_names = ' '.join(partition_types.keys())
partition_size_output = self.ssh_client.exec_command(
'fdisk -l %s 2>/dev/null | grep "Disk.*bytes"' % (partition_names)).rstrip('\n').split('\n')
partitions = []
for row in partition_size_output:
row_details = row.split()
partition_name = row_details[1].rstrip(':')
partition_type = partition_types[partition_name]
if partition_type == 'swap':
partition_size = DiskSize(
float(row_details[2]), row_details[3].rstrip(','))
else:
partition_size = DiskSize(
int(row_details[4]) / 1073741824, 'GB')
partitions.append(
Partition(partition_name, partition_size, partition_type))
return partitions
def verify_partitions(self, expected_disk_size, expected_swap_size, server_status, actual_partitions):
"""
@summary: Verify the partition details of the server
@param expected_disk_size: The expected value of the Disk size in GB
@type expected_disk_size: string
@param expected_swap_size: The expected value of the Swap size in GB
@type expected_swap_size: string
@param server_status: The status of the server
@type server_status: string
@param actual_partitions: The actual partition details of the server
@type actual_partitions: Partition List
@return: The result of verification and the message to be displayed
@rtype: Tuple (bool,string)
"""
expected_partitions = self._get_expected_partitions(
expected_disk_size, expected_swap_size, server_status)
if actual_partitions is None:
actual_partitions = self.get_partition_details()
for partition in expected_partitions:
if partition not in actual_partitions:
return False, self._construct_partition_mismatch_message(expected_partitions, actual_partitions)
return True, "Partitions Matched"
def _get_expected_partitions(self, expected_disk_size, expected_swap_size, server_status):
"""
@summary: Returns the expected partitions for a server based on server status
@param expected_disk_size: The Expected disk size of the server in GB
@type expected_disk_size: string
@param expected_swap_size: The Expected swap size of the server in MB
@type expected_swap_size: string
@param server_status: Status of the server (ACTIVE or RESCUE)
@type server_status: string
@return: The expected partitions
@rtype: Partition List
"""
# ignoring swap untill the rescue functionality is clarified
expected_partitions = [Partition(
'/dev/xvda1', DiskSize(expected_disk_size, 'GB'), 'ext3'),
Partition('/dev/xvdc1', DiskSize(expected_swap_size, 'MB'), 'swap')]
if str.upper(server_status) == 'RESCUE':
expected_partitions = [Partition(
'/dev/xvdb1', DiskSize(expected_disk_size, 'GB'), 'ext3')]
# expected_partitions.append(Partition('/dev/xvdd1',
# DiskSize(expected_swap_size, 'MB'), 'swap'))
return expected_partitions
def _construct_partition_mismatch_message(self, expected_partitions, actual_partitions):
"""
@summary: Constructs the partition mismatch message based on expected_partitions and actual_partitions
@param expected_partitions: Expected partitions of the server
@type expected_partitions: Partition List
@param actual_partitions: Actual Partitions of the server
@type actual_partitions: Partition List
@return: The partition mismatch message
@rtype: string
"""
message = 'Partitions Mismatch \n Expected Partitions:\n'
for partition in expected_partitions:
message += str(partition) + '\n'
message += ' Actual Partitions:\n'
for partition in actual_partitions:
message += str(partition) + '\n'
return message
def mount_file_to_destination_directory(self, source_path, destination_path):
'''
@summary: Mounts the file to destination directory
@param source_path: Path to file source
@type source_path: String
@param destination_path: Path to mount destination
@type destination_path: String
'''
self.ssh_client.exec_command(
'mount ' + source_path + ' ' + destination_path)

16
cafe/engine/clients/remote_instance/windows/__init__.py Normal file
View File

@@ -0,0 +1,16 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

18
cafe/engine/clients/remote_instance/windows/windows_instance_client.py Normal file
View File

@@ -0,0 +1,18 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
class WindowsClient:
pass

304
cafe/engine/clients/rest.py Normal file
View File

@@ -0,0 +1,304 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import requests
from time import time
from cafe.common.reporting import cclogging
from cafe.engine.clients.base import BaseClient
def _log_transaction(log, level=cclogging.logging.DEBUG):
""" Paramaterized decorator
Takes a python Logger object and an optional logging level.
"""
def _decorator(func):
"""Accepts a function and returns wrapped version of that function."""
def _wrapper(*args, **kwargs):
"""Logging wrapper for any method that returns a requests response.
Logs requestslib response objects, and the args and kwargs
sent to the request() method, to the provided log at the provided
log level.
"""
logline = '{0} {1}'.format(args, kwargs)
try:
log.debug(logline.decode('utf-8', 'replace'))
except Exception as exception:
#Ignore all exceptions that happen in logging, then log them
log.info(
'Exception occured while logging signature of calling'
'method in rest connector')
log.exception(exception)
#Make the request and time it's execution
response = None
elapsed = None
try:
start = time()
response = func(*args, **kwargs)
elapsed = time() - start
except Exception as exception:
log.critical('Call to Requests failed due to exception')
log.exception(exception)
raise exception
#requests lib 1.0.0 renamed body to data in the request object
request_body = ''
if 'body' in dir(response.request):
request_body = response.request.body
elif 'data' in dir(response.request):
request_body = response.request.data
else:
log.info(
"Unable to log request body, neither a 'data' nor a "
"'body' object could be found")
#requests lib 1.0.4 removed params from response.request
request_params = ''
request_url = response.request.url
if 'params' in dir(response.request):
request_params = response.request.params
elif '?' in request_url:
request_url, request_params = request_url.split('?')
logline = ''.join([
'\n{0}\nREQUEST SENT\n{0}\n'.format('-' * 12),
'request method..: {0}\n'.format(response.request.method),
'request url.....: {0}\n'.format(request_url),
'request params..: {0}\n'.format(request_params),
'request headers.: {0}\n'.format(response.request.headers),
'request body....: {0}\n'.format(request_body)])
try:
log.log(level, logline.decode('utf-8', 'replace'))
except Exception as exception:
#Ignore all exceptions that happen in logging, then log them
log.log(level, '\n{0}\nREQUEST INFO\n{0}\n'.format('-' * 12))
log.exception(exception)
logline = ''.join([
'\n{0}\nRESPONSE RECIEVED\n{0}\n'.format('-' * 17),
'response status..: {0}\n'.format(response),
'response time....: {0}\n'.format(elapsed),
'response headers.: {0}\n'.format(response.headers),
'response body....: {0}\n'.format(response.content),
'-' * 79])
try:
log.log(level, logline.decode('utf-8', 'replace'))
except Exception as exception:
#Ignore all exceptions that happen in logging, then log them
log.log(level, '\n{0}\nRESPONSE INFO\n{0}\n'.format('-' * 13))
log.exception(exception)
return response
return _wrapper
return _decorator
def _inject_exception(exception_handlers):
"""Paramaterized decorator takes a list of exception_handler objects"""
def _decorator(func):
"""Accepts a function and returns wrapped version of that function."""
def _wrapper(*args, **kwargs):
"""Wrapper for any function that returns a Requests response.
Allows exception handlers to raise custom exceptions based on
response object attributes such as status_code.
"""
response = func(*args, **kwargs)
if exception_handlers:
for handler in exception_handlers:
handler.check_for_errors(response)
return response
return _wrapper
return _decorator
class BaseRestClient(BaseClient):
"""Re-implementation of Requests' api.py that removes many assumptions.
Adds verbose logging.
Adds support for response-code based exception injection.
(Raising exceptions based on response code)
@see: http://docs.python-requests.org/en/latest/api/#configurations
"""
_exception_handlers = []
_log = cclogging.getLogger(__name__)
def __init__(self):
super(BaseRestClient, self).__init__()
@_inject_exception(_exception_handlers)
@_log_transaction(log=_log)
def request(self, method, url, **kwargs):
""" Performs <method> HTTP request to <url> using the requests lib"""
return requests.request(method, url, **kwargs)
def put(self, url, **kwargs):
""" HTTP PUT request """
return self.request('PUT', url, **kwargs)
def copy(self, url, **kwargs):
""" HTTP COPY request """
return self.request('COPY', url, **kwargs)
def post(self, url, data=None, **kwargs):
""" HTTP POST request """
return self.request('POST', url, data=data, **kwargs)
def get(self, url, **kwargs):
""" HTTP GET request """
return self.request('GET', url, **kwargs)
def head(self, url, **kwargs):
""" HTTP HEAD request """
return self.request('HEAD', url, **kwargs)
def delete(self, url, **kwargs):
""" HTTP DELETE request """
return self.request('DELETE', url, **kwargs)
def options(self, url, **kwargs):
""" HTTP OPTIONS request """
return self.request('OPTIONS', url, **kwargs)
def patch(self, url, **kwargs):
""" HTTP PATCH request """
return self.request('PATCH', url, **kwargs)
@classmethod
def add_exception_handler(cls, handler):
"""Adds a specific L{ExceptionHandler} to the rest connector
@warning: SHOULD ONLY BE CALLED FROM A PROVIDER THROUGH A TEST
FIXTURE
"""
cls._exception_handlers.append(handler)
@classmethod
def delete_exception_handler(cls, handler):
"""Removes a L{ExceptionHandler} from the rest connector
@warning: SHOULD ONLY BE CALLED FROM A PROVIDER THROUGH A TEST
FIXTURE
"""
if handler in cls._exception_handlers:
cls._exception_handlers.remove(handler)
class RestClient(BaseRestClient):
"""
@summary: Allows clients to inherit all requests-defined RESTfull
verbs. Redefines request() so that keyword args are passed
through a named dictionary instead of kwargs.
Client methods can then take paramaters that may overload
request paramaters, which allows client method calls to
override parts of the request with paramters sent directly
to requests, overiding the client method logic either in
part or whole on the fly.
@see: http://docs.python-requests.org/en/latest/api/#configurations
"""
def __init__(self):
super(RestClient, self).__init__()
self.default_headers = {}
def request(
self, method, url, headers=None, params=None, data=None,
requestslib_kwargs=None):
#set requestslib_kwargs to an empty dict if None
requestslib_kwargs = requestslib_kwargs if (
requestslib_kwargs is not None) else {}
#Set defaults
params = params if params is not None else {}
verify = False
#If headers are provided by both, headers "wins" over default_headers
headers = dict(self.default_headers, **(headers or {}))
#Override url if present in requestslib_kwargs
if 'url' in requestslib_kwargs.keys():
url = requestslib_kwargs.get('url', None) or url
del requestslib_kwargs['url']
#Override method if present in requestslib_kwargs
if 'method' in requestslib_kwargs.keys():
method = requestslib_kwargs.get('method', None) or method
del requestslib_kwargs['method']
#The requests lib already removes None key/value pairs, but we force it
#here in case that behavior ever changes
for key in requestslib_kwargs.keys():
if requestslib_kwargs[key] is None:
del requestslib_kwargs[key]
#Create the final paramaters for the call to the base request()
#Wherever a paramater is provided both by the calling method AND
#the requests_lib kwargs dictionary, requestslib_kwargs "wins"
requestslib_kwargs = dict({'headers': headers,
'params': params,
'verify': verify,
'data': data},
**requestslib_kwargs)
#Make the request
return super(RestClient, self).request(method, url,
**requestslib_kwargs)
class AutoMarshallingRestClient(RestClient):
"""@TODO: Turn serialization and deserialization into decorators so
that we can support serialization and deserialization on a per-method
basis"""
def __init__(self, serialize_format=None, deserialize_format=None):
super(AutoMarshallingRestClient, self).__init__()
self.serialize_format = serialize_format
self.deserialize_format = deserialize_format or self.serialize_format
def request(self, method, url, headers=None, params=None, data=None,
response_entity_type=None, request_entity=None,
requestslib_kwargs=None):
#defaults requestslib_kwargs to a dictionary if it is None
requestslib_kwargs = requestslib_kwargs if (requestslib_kwargs is not
None) else {}
#set the 'data' paramater of the request to either what's already in
#requestslib_kwargs, or the deserialized output of the request_entity
if request_entity is not None:
requestslib_kwargs = dict(
{'data': request_entity.serialize(self.serialize_format)},
**requestslib_kwargs)
#Make the request
response = super(AutoMarshallingRestClient, self).request(
method, url, headers=headers, params=params, data=data,
requestslib_kwargs=requestslib_kwargs)
#Append the deserialized data object to the response
response.request.__dict__['entity'] = None
response.__dict__['entity'] = None
#If present, append the serialized request data object to
#response.request
if response.request is not None:
response.request.__dict__['entity'] = request_entity
if response_entity_type is not None:
response.__dict__['entity'] = response_entity_type.deserialize(
response.content,
self.deserialize_format)
return response

16
cafe/engine/clients/soap.py Normal file
View File

@@ -0,0 +1,16 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

285
cafe/engine/clients/ssh.py Normal file
View File

@@ -0,0 +1,285 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import time
import socket
import exceptions
import warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore")
from paramiko.resource import ResourceManager
from paramiko.client import SSHClient
import paramiko
from cafe.common.reporting import cclogging
from cafe.engine.clients.base import BaseClient
class SSHBaseClient(BaseClient):
_log = cclogging.getLogger(__name__)
def __init__(self, host, username, password, timeout=20, port=22):
super(SSHBaseClient, self).__init__()
self.host = host
self.port = port
self.username = username
self.password = password
self.timeout = int(timeout)
self._chan = None
def _get_ssh_connection(self):
"""Returns an ssh connection to the specified host"""
_timeout = True
ssh = SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
_start_time = time.time()
saved_exception = exceptions.StandardError()
#doing this because the log file fills up with these messages
#this way it only logs it once
log_attempted = False
socket_error_logged = False
auth_error_logged = False
ssh_error_logged = False
while not self._is_timed_out(self.timeout, _start_time):
try:
if not log_attempted:
self._log.debug('Attempting to SSH connect to: ')
self._log.debug('host: %s, username: %s' %
(self.host, self.username))
log_attempted = True
ssh.connect(hostname=self.host,
username=self.username,
password=self.password,
timeout=self.timeout,
key_filename=[],
look_for_keys=False,
allow_agent=False)
_timeout = False
break
except socket.error as e:
if not socket_error_logged:
self._log.error('Socket Error: %s' % str(e))
socket_error_logged = True
saved_exception = e
continue
except paramiko.AuthenticationException as e:
if not auth_error_logged:
self._log.error('Auth Exception: %s' % str(e))
auth_error_logged = True
saved_exception = e
time.sleep(2)
continue
except paramiko.SSHException as e:
if not ssh_error_logged:
self._log.error('SSH Exception: %s' % str(e))
ssh_error_logged = True
saved_exception = e
time.sleep(2)
continue
#Wait 2 seconds otherwise
time.sleep(2)
if _timeout:
self._log.error('SSHConnector timed out while trying to establish a connection')
raise saved_exception
#This MUST be done because the transport gets garbage collected if it
#is not done here, which causes the connection to close on invoke_shell
#which is needed for exec_shell_command
ResourceManager.register(self, ssh.get_transport())
return ssh
def _is_timed_out(self, timeout, start_time):
return (time.time() - timeout) > start_time
def connect_until_closed(self):
"""Connect to the server and wait until connection is lost"""
try:
ssh = self._get_ssh_connection()
_transport = ssh.get_transport()
_start_time = time.time()
_timed_out = self._is_timed_out(self.timeout, _start_time)
while _transport.is_active() and not _timed_out:
time.sleep(5)
_timed_out = self._is_timed_out(self.timeout, _start_time)
ssh.close()
except (EOFError, paramiko.AuthenticationException, socket.error):
return
def exec_command(self, cmd):
"""Execute the specified command on the server.
:returns: data read from standard output of the command
"""
self._log.debug('EXECing: %s' % str(cmd))
ssh = self._get_ssh_connection()
stdin, stdout, stderr = ssh.exec_command(cmd)
output = stdout.read()
ssh.close()
self._log.debug('EXEC-OUTPUT: %s' % str(output))
return output
def test_connection_auth(self):
""" Returns true if ssh can connect to server"""
try:
connection = self._get_ssh_connection()
connection.close()
except paramiko.AuthenticationException:
return False
return True
def start_shell(self):
"""Starts a shell instance of SSH to use with multiple commands."""
#large width and height because of need to parse output of commands
#in exec_shell_command
self._chan = self._get_ssh_connection().invoke_shell(width=9999999,
height=9999999)
#wait until buffer has data
while not self._chan.recv_ready():
time.sleep(1)
#clearing initial buffer, usually login information
while self._chan.recv_ready():
self._chan.recv(1024)
def exec_shell_command(self, cmd, stop_after_send=False):
"""
Executes a command in shell mode and receives all of the response.
Parses the response and returns the output of the command and the
prompt.
"""
if not cmd.endswith('\n'):
cmd = '%s\n' % cmd
self._log.debug('EXEC-SHELLing: %s' % cmd)
if self._chan is None or self._chan.closed:
self.start_shell()
while not self._chan.send_ready():
time.sleep(1)
self._chan.send(cmd)
if stop_after_send:
self._chan.get_transport().set_keepalive(1000)
return None
while not self._chan.recv_ready():
time.sleep(1)
output = ''
while self._chan.recv_ready():
output += self._chan.recv(1024)
self._log.debug('SHELL-COMMAND-RETURN: \n%s' % output)
prompt = output[output.rfind('\r\n') + 2:]
output = output[output.find('\r\n') + 2:output.rfind('\r\n')]
self._chan.get_transport().set_keepalive(1000)
return output, prompt
def exec_shell_command_wait_for_prompt(self, cmd, prompt='#', timeout=300):
"""
Executes a command in shell mode and receives all of the response.
Parses the response and returns the output of the command and the
prompt.
"""
if not cmd.endswith('\n'):
cmd = '%s\n' % cmd
self._log.debug('EXEC-SHELLing: %s' % cmd)
if self._chan is None or self._chan.closed:
self.start_shell()
while not self._chan.send_ready():
time.sleep(1)
self._chan.send(cmd)
while not self._chan.recv_ready():
time.sleep(1)
output = ''
max_time = time.time() + timeout
while time.time() < max_time:
current = self._chan.recv(1024)
output += current
if current.find(prompt) != -1:
self._log.debug('SHELL-PROMPT-FOUND: %s' % prompt)
break
self._log.debug('Current response: %s' % current)
self._log.debug('Looking for prompt: %s. Time remaining until timeout: %s'
% (prompt, max_time - time.time()))
while not self._chan.recv_ready() and time.time() < max_time:
time.sleep(5)
self._chan.get_transport().set_keepalive(1000)
self._log.debug('SHELL-COMMAND-RETURN: %s' % output)
prompt = output[output.rfind('\r\n') + 2:]
output = output[output.find('\r\n') + 2:output.rfind('\r\n')]
return output, prompt
def make_directory(self, directory_name):
self._log.info('Making a Directory')
transport = paramiko.Transport((self.host, self.port))
transport.connect(username=self.username, password=self.password)
sftp = paramiko.SFTPClient.from_transport(transport)
try:
sftp.mkdir(directory_name)
except IOError, exception:
self._log.warning("Exception in making a directory: %s" % exception)
return False
else:
sftp.close()
transport.close()
return True
def browse_folder(self):
self._log.info('Browsing a folder')
transport = paramiko.Transport((self.host, self.port))
transport.connect(username=self.username, password=self.password)
sftp = paramiko.SFTPClient.from_transport(transport)
try:
sftp.listdir()
except IOError, exception:
self._log.warning("Exception in browsing folder file: %s" % exception)
return False
else:
sftp.close()
transport.close()
return True
def upload_a_file(self, server_file_path, client_file_path):
self._log.info("uploading file from %s to %s"
% (client_file_path, server_file_path))
transport = paramiko.Transport((self.host, self.port))
transport.connect(username=self.username, password=self.password)
sftp = paramiko.SFTPClient.from_transport(transport)
try:
sftp.put(client_file_path, server_file_path)
except IOError, exception:
self._log.warning("Exception in uploading file: %s" % exception)
return False
else:
sftp.close()
transport.close()
return True
def download_a_file(self, server_filepath, client_filepath):
transport = paramiko.Transport(self.host)
transport.connect(username=self.username, password=self.password)
sftp = paramiko.SFTPClient.from_transport(transport)
try:
sftp.get(server_filepath, client_filepath)
except IOError:
return False
else:
sftp.close()
transport.close()
return True
def end_shell(self):
if not self._chan.closed:
self._chan.close()
self._chan = None

128
cafe/engine/config.py Normal file
View File

@@ -0,0 +1,128 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import os
import ConfigParser
_ENGINE_CONFIG_FILE_ENV_VAR = 'CCTNG_CONFIG_FILE'
class NonExistentConfigPathError(Exception):
pass
class ConfigEnvironmentVariableError(Exception):
pass
class EngineConfig(object):
'''
Config interface for the global engine configuration
'''
SECTION_NAME = 'CCTNG_ENGINE'
def __init__(self, config_file_path=None, section_name=None):
#Support for setting the section name as a class or instance
#constant, as both 'SECTION_NAME' and 'CONFIG_SECTION_NAME'
self._section_name = (section_name or
getattr(self, 'SECTION_NAME', None) or
getattr(self, 'CONFIG_SECTION_NAME', None))
self._datasource = None
config_file_path = config_file_path or self.default_config_file
#Check the path
if not os.path.exists(config_file_path):
msg = 'Could not verify the existence of config file at {0}'\
.format(config_file_path)
raise NonExistentConfigPathError(msg)
#Read the file in and turn it into a SafeConfigParser instance
try:
self._datasource = ConfigParser.SafeConfigParser()
self._datasource.read(config_file_path)
except Exception as e:
raise e
@property
def default_config_file(self):
engine_config_file_path = None
try:
engine_config_file_path = os.environ[_ENGINE_CONFIG_FILE_ENV_VAR]
except KeyError:
msg = "'{0}' environment variable was not set.".format(
_ENGINE_CONFIG_FILE_ENV_VAR)
raise ConfigEnvironmentVariableError(msg)
except Exception as exception:
print ("Unexpected exception while attempting to access '{0}' "
"environment variable.".format(_ENGINE_CONFIG_FILE_ENV_VAR))
raise exception
return(engine_config_file_path)
def get(self, item_name, default=None):
try:
return self._datasource.get(self._section_name, item_name)
except ConfigParser.NoOptionError as no_option_err:
if not default:
raise no_option_err
return default
def get_raw(self, item_name, default=None):
'''Performs a get() on SafeConfigParser object without interpolation
'''
try:
return self._datasource.get(self._section_name, item_name,
raw=True)
except ConfigParser.NoOptionError as no_option_err:
if not default:
raise no_option_err
return default
def get_boolean(self, item_name, default=None):
try:
return self._datasource.getboolean(self._section_name,
item_name)
except ConfigParser.NoOptionError as no_option_err:
if not default:
raise no_option_err
return default
#Provided for implementations of cafe, unused by the engine itself
@property
def data_directory(self):
return self.get_raw("data_directory")
#Provided for implementations of cafe, unused by the engine itself
@property
def temp_directory(self):
return self.get_raw("temp_directory")
#Used by the engine for the output of engine and implementation logs
@property
def log_directory(self):
return os.getenv("CLOUDCAFE_LOG_PATH", self.get_raw("log_directory", default="."))
#Used by the engine for the output of engine and implementation logs
@property
def master_log_file_name(self):
return self.get_raw("master_log_file_name", default="engine-master")
#Used by the engine for the output of engine and implementation logs
@property
def use_verbose_logging(self):
return self.get_boolean("use_verbose_logging", False)

16
cafe/engine/models/__init__.py Normal file
View File

@@ -0,0 +1,16 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

261
cafe/engine/models/base.py Normal file
View File

@@ -0,0 +1,261 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from xml.etree import ElementTree
from cafe.common.reporting import cclogging
class CommonToolsMixin(object):
"""Methods used to make building data models easier, common to all types"""
@staticmethod
def _bool_to_string(value, true_string='true', false_string='false'):
"""Returns a string representation of a boolean value, or the value
provided if the value is not an instance of bool
"""
if isinstance(value, bool):
return true_string if value is True else false_string
return value
@staticmethod
def _remove_empty_values(dictionary):
'''Returns a new dictionary based on 'dictionary', minus any keys with
values that are None
'''
return dict((k, v) for k, v in dictionary if v is not None)
class JSON_ToolsMixin(object):
"""Methods used to make building json data models easier"""
pass
class XML_ToolsMixin(object):
"""Methods used to make building xml data models easier"""
@staticmethod
def _set_xml_etree_element(
xml_etree, property_dict, exclude_empty_properties=True):
'''Sets a dictionary of keys and values as properties of the xml etree
element if value is not None. Optionally, add all keys and values as
properties if only if exclude_empty_properties == False.
'''
if exclude_empty_properties:
property_dict = CommonToolsMixin._remove_empty_keys(property_dict)
for key in property_dict:
xml_etree.set(str(key), str(property_dict[key]))
return xml_etree
@staticmethod
def _remove_xml_etree_namespace(doc, namespace):
"""Remove namespace in the passed document in place."""
ns = u'{%s}' % namespace
nsl = len(ns)
for elem in doc.getiterator():
for key in elem.attrib:
if key.startswith(ns):
new_key = key[nsl:]
elem.attrib[new_key] = elem.attrib[key]
del elem.attrib[key]
if elem.tag.startswith(ns):
elem.tag = elem.tag[nsl:]
return doc
class BAD_XML_TOOLS(object):
'''THESE ARE BAD. DON'T USE THEM. They were created in a more innocent
age, and are here for backwards compatability only.
'''
def _auto_value_to_dict(self, value):
ret = None
if isinstance(value, (int, str, unicode, bool)):
ret = value
elif isinstance(value, list):
ret = []
for item in value:
ret.append(self._auto_value_to_dict(item))
elif isinstance(value, dict):
ret = {}
for key in value.keys():
ret[key] = self._auto_value_to_dict(value[key])
elif isinstance(value, BaseMarshallingDomain):
ret = value._obj_to_json()
return ret
def _auto_to_dict(self):
ret = {}
for attr in vars(self).keys():
value = vars(self).get(attr)
if value is not None and attr != '_log':
ret[attr] = self._auto_value_to_dict(value)
if hasattr(self, 'ROOT_TAG'):
return {self.ROOT_TAG: ret}
else:
return ret
def _auto_to_xml(self):
#XML is almost impossible to do without a schema definition because it
#cannot be determined when an instance variable should be an attribute
#of an element or text between that element's tags
ret = ElementTree.Element(self.ROOT_TAG)
for attr in vars(self).keys():
value = vars(self).get(attr)
if value is not None:
assigned = self._auto_value_to_xml(attr, value)
if isinstance(assigned, ElementTree.Element):
ret.append(assigned)
else:
ret.set(attr, str(assigned))
return ret
class BaseModel(object):
__REPR_SEPARATOR__ = '\n'
def __init__(self):
self._log = cclogging.getLogger(
cclogging.get_object_namespace(self.__class__))
def __eq__(self, obj):
try:
if vars(obj) == vars(self):
return True
except:
pass
return False
def __ne__(self, obj):
if obj is None:
return True
if vars(obj) == vars(self):
return False
else:
return True
def __str__(self):
strng = '<{0} object> {1}'.format(
self.__class__.__name__, self.__REPR_SEPARATOR__)
for key in self.__dict__.keys():
if str(key) == '_log':
continue
strng = '{0}{1} = {2}{3}'.format(
strng, str(key), str(self.__dict__[key]),
self.__REPR_SEPARATOR__)
return strng
def __repr__(self):
return self.__str__()
#Splitting the xml and json stuff into mixins cleans up the code but still
#muddies the AutoMarshallingModel namespace. We could create
#tool objects in the AutoMarshallingModel, which would just act as
#sub-namespaces, to keep it clean. --Jose
class AutoMarshallingModel(
BaseModel, CommonToolsMixin, JSON_ToolsMixin, XML_ToolsMixin,
BAD_XML_TOOLS):
"""
@summary: A class used as a base to build and contain the logic necessary
to automatically create serialized requests and automatically
deserialize responses in a format-agnostic way.
"""
_log = cclogging.getLogger(__name__)
def __init__(self):
super(AutoMarshallingModel, self).__init__()
self._log = cclogging.getLogger(
cclogging.get_object_namespace(self.__class__))
def serialize(self, format_type):
serialization_exception = None
try:
serialize_method = '_obj_to_{0}'.format(format_type)
return getattr(self, serialize_method)()
except Exception as serialization_exception:
pass
if serialization_exception:
try:
self._log.error(
'Error occured during serialization of a data model into'
'the "{0}: \n{1}" format'.format(
format_type, serialization_exception))
self._log.exception(serialization_exception)
except Exception as exception:
self._log.exception(exception)
self._log.debug(
"Unable to log information regarding the "
"deserialization exception due to '{0}'".format(
serialization_exception))
return None
@classmethod
def deserialize(cls, serialized_str, format_type):
cls._log = cclogging.getLogger(
cclogging.get_object_namespace(cls))
model_object = None
deserialization_exception = None
if serialized_str and len(serialized_str) > 0:
try:
deserialize_method = '_{0}_to_obj'.format(format_type)
model_object = getattr(cls, deserialize_method)(serialized_str)
except Exception as deserialization_exception:
cls._log.exception(deserialization_exception)
#Try to log string and format_type if deserialization broke
if deserialization_exception is not None:
try:
cls._log.debug(
"Deserialization Error: Attempted to deserialize type"
" using type: {0}".format(format_type.decode(
encoding='UTF-8', errors='ignore')))
cls._log.debug(
"Deserialization Error: Unble to deserialize the "
"following:\n{0}".format(serialized_str.decode(
encoding='UTF-8', errors='ignore')))
except Exception as exception:
cls._log.exception(exception)
cls._log.debug(
"Unable to log information regarding the "
"deserialization exception")
return model_object
#Serialization Functions
def _obj_to_json(self):
raise NotImplementedError
def _obj_to_xml(self):
raise NotImplementedError
#Deserialization Functions
@classmethod
def _xml_to_obj(cls, serialized_str):
raise NotImplementedError
@classmethod
def _json_to_obj(cls, serialized_str):
raise NotImplementedError
class AutoMarshallingListModel(list, AutoMarshallingModel):
"""List-like AutoMarshallingModel used for some special cases"""
def __str__(self):
return list.__str__(self)

37
cafe/engine/models/commandline_response.py Normal file
View File

@@ -0,0 +1,37 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
'''
@summary: Responses directly from the command line
'''
class CommandLineResponse(object):
'''Bare bones object for any Command Line Connector response
@ivar Command: The full original command string for this response
@type Command: C{str}
@ivar StandardOut: The Standard Out generated by this command
@type StandardOut: C{list} of C{str}
@ivar StandardError: The Standard Error generated by this command
@type StandardError: C{list} of C{str}
@ivar ReturnCode: The command's return code
@type ReturnCode: C{int}
'''
def __init__(self):
self.command = ""
self.standard_out = []
self.standard_error = []
self.return_code = None

114
cafe/engine/models/data_interfaces.py Normal file
View File

@@ -0,0 +1,114 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import os
import ConfigParser
from cafe.common.reporting import cclogging
class ConfigDataException(Exception):
pass
class NonExistentConfigPathError(Exception):
pass
class ConfigEnvironmentVariableError(Exception):
pass
#Decorator
def expected_values(*values):
def decorator(fn):
def wrapped():
class UnexpectedConfigOptionValueError(Exception):
pass
value = fn()
if value not in values:
raise UnexpectedConfigOptionValueError(value)
return fn()
return wrapped
return decorator
class BaseConfigSectionInterface(object):
"""
Base class for building an interface for the data contained in a
SafeConfigParser object, as loaded from the config file as defined
by the engine's config file.
This is meant to be a generic interface so that in the future
get() and getboolean() can be reimplemented to provide data from a
database
"""
def __init__(self, config_file_path, section_name):
self._log = cclogging.getLogger(
cclogging.get_object_namespace(self.__class__))
self._datasource = ConfigParser.SafeConfigParser()
self._section_name = section_name
#Check the path
if not os.path.exists(config_file_path):
msg = 'Could not verify the existence of config file at {0}'\
.format(config_file_path)
raise NonExistentConfigPathError(msg)
#Read the file in and turn it into a SafeConfigParser instance
try:
self._datasource.read(config_file_path)
except Exception as exception:
self._log.exception(exception)
raise exception
def get(self, item_name, default=None):
try:
return self._datasource.get(self._section_name, item_name)
except ConfigParser.NoOptionError as e:
self._log.error(str(e))
return default
except ConfigParser.NoSectionError as e:
self._log.error(str(e))
pass
def get_raw(self, item_name, default=None):
'''Performs a get() on SafeConfigParser object without interopolation
'''
try:
return self._datasource.get(self._section_name, item_name,
raw=True)
except ConfigParser.NoOptionError as e:
self._log.error(str(e))
return default
except ConfigParser.NoSectionError as e:
self._log.error(str(e))
pass
def get_boolean(self, item_name, default=None):
try:
return self._datasource.getboolean(self._section_name,
item_name)
except ConfigParser.NoOptionError as e:
self._log.error(str(e))
return default
except ConfigParser.NoSectionError as e:
self._log.error(str(e))
pass

23
cafe/engine/provider.py Normal file
View File

@@ -0,0 +1,23 @@
"""
Copyright 2013 Rackspace
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from cafe.common.reporting import cclogging
class BaseProvider(object):
def __init__(self):
self._log = cclogging.getLogger(
cclogging.get_object_namespace(self.__class__))