From a23f500725df8d5ae83f69eb4da5e47736fbb647 Mon Sep 17 00:00:00 2001 From: Attila Fazekas Date: Tue, 23 Oct 2012 19:32:45 +0200 Subject: [PATCH] Add start of the EC2/S3 API testing to tempest Continues the effort of the https://review.openstack.org/#/c/3064/ * add EC2 keypair and volume tests * add EC2 snapshot from volume test * add EC2 floating ip disasscioation * add EC2 operation on security group * add EC2/S3 image registration * add EC2 instance run test * add Integration test with ssh, console, volume * add S3 object and bucket tests Change-Id: I0dff9b05f215b56456272f22aa1c014cd53b4f4b --- .gitignore | 1 + etc/tempest.conf.sample | 45 ++ etc/tempest.conf.tpl | 45 ++ tempest/common/ssh.py | 9 +- tempest/common/utils/file_utils.py | 25 + tempest/common/utils/linux/remote_client.py | 32 +- tempest/config.py | 81 +++ tempest/exceptions.py | 27 + tempest/openstack.py | 4 + tempest/services/boto/__init__.py | 102 ++++ tempest/services/boto/clients.py | 139 +++++ tempest/testboto.py | 535 ++++++++++++++++++ tempest/tests/boto/__init__.py | 98 ++++ tempest/tests/boto/test_ec2_instance_run.py | 249 ++++++++ tempest/tests/boto/test_ec2_keys.py | 79 +++ tempest/tests/boto/test_ec2_network.py | 49 ++ .../tests/boto/test_ec2_security_groups.py | 78 +++ tempest/tests/boto/test_ec2_volumes.py | 93 +++ tempest/tests/boto/test_s3_buckets.py | 49 ++ tempest/tests/boto/test_s3_ec2_images.py | 144 +++++ tempest/tests/boto/test_s3_objects.py | 58 ++ tempest/tests/boto/utils/__init__.py | 0 tempest/tests/boto/utils/s3.py | 42 ++ tempest/tests/boto/utils/wait.py | 130 +++++ tools/pip-requires | 1 + 25 files changed, 2102 insertions(+), 13 deletions(-) create mode 100644 tempest/common/utils/file_utils.py create mode 100644 tempest/services/boto/__init__.py create mode 100644 tempest/services/boto/clients.py create mode 100644 tempest/testboto.py create mode 100644 tempest/tests/boto/__init__.py create mode 100644 tempest/tests/boto/test_ec2_instance_run.py create mode 100644 tempest/tests/boto/test_ec2_keys.py create mode 100644 tempest/tests/boto/test_ec2_network.py create mode 100644 tempest/tests/boto/test_ec2_security_groups.py create mode 100644 tempest/tests/boto/test_ec2_volumes.py create mode 100644 tempest/tests/boto/test_s3_buckets.py create mode 100644 tempest/tests/boto/test_s3_ec2_images.py create mode 100644 tempest/tests/boto/test_s3_objects.py create mode 100644 tempest/tests/boto/utils/__init__.py create mode 100644 tempest/tests/boto/utils/s3.py create mode 100644 tempest/tests/boto/utils/wait.py diff --git a/.gitignore b/.gitignore index 5b28711174..55096edd8f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,6 @@ include/swift_objects/swift_medium include/swift_objects/swift_large *.log *.swp +*.swo *.egg-info .tox diff --git a/etc/tempest.conf.sample b/etc/tempest.conf.sample index 2987c56184..053c36e0df 100644 --- a/etc/tempest.conf.sample +++ b/etc/tempest.conf.sample @@ -221,3 +221,48 @@ tenant_name = admin # custom Keystone service catalog implementation, you probably want to leave # this value as "object-store" catalog_type = object-store + +[boto] +# This section contains configuration options used when executing tests +# with boto. + +# EC2 URL +ec2_url = http://localhost:8773/services/Cloud +# S3 URL +s3_url = http://localhost:3333 + +# Use keystone ec2-* command to get those values for your test user and tenant +aws_access = +aws_secret = + +#Region +aws_region = RegionOne + +#Image materials for S3 upload +# ALL content of the specified directory will be uploaded to S3 +s3_materials_path = /opt/stack/devstack/files/images/s3-materials/cirros-0.3.0 + +# The manifest.xml files, must be in the s3_materials_path directory +# Subdirectories not allowed! +# The filenames will be used as a Keys in the S3 Buckets + +#ARI Ramdisk manifest. Must be in the above s3_materials_path +ari_manifest = cirros-0.3.0-x86_64-initrd.manifest.xml + +#AMI Machine Image manifest. Must be in the above s3_materials_path +ami_manifest = cirros-0.3.0-x86_64-blank.img.manifest.xml + +#AKI Kernel Image manifest, Must be in the above s3_materials_path +aki_manifest = cirros-0.3.0-x86_64-vmlinuz.manifest.xml + +#Instance type +instance_type = m1.tiny + +#TCP/IP connection timeout +http_socket_timeout = 5 + +# Status change wait timout +build_timeout = 120 + +# Status change wait interval +build_interval = 1 diff --git a/etc/tempest.conf.tpl b/etc/tempest.conf.tpl index 5e2ee7feae..880a3c1ce1 100644 --- a/etc/tempest.conf.tpl +++ b/etc/tempest.conf.tpl @@ -191,3 +191,48 @@ tenant_name = %TENANT_NAME% # custom Keystone service catalog implementation, you probably want to leave # this value as "object-store" catalog_type = %OBJECT_CATALOG_TYPE% + +[boto] +# This section contains configuration options used when executing tests +# with boto. + +# EC2 URL +ec2_url = %BOTO_EC2_URL% +# S3 URL +s3_url = %BOTO_S3_URL% + +# Use keystone ec2-* command to get those values for your test user and tenant +aws_access = %BOTO_AWS_ACCESS% +aws_secret = %BOTO_AWS_SECRET% + +#Region +aws_region = %BOTO_AWS_REGION% + +#Image materials for S3 upload +# ALL content of the specified directory will be uploaded to S3 +s3_materials_path = %BOTO_S3_MATERIALS_PATH% + +# The manifest.xml files, must be in the s3_materials_path directory +# Subdirectories not allowed! +# The filenames will be used as a Keys in the S3 Buckets + +#ARI Ramdisk manifest. Must be in the above s3_materials_path directory +ari_manifest = %BOTO_ARI_MANIFEST% + +#AMI Machine Image manifest. Must be in the above s3_materials_path directory +ami_manifest = %BOTO_AMI_MANIFEST% + +#AKI Kernel Image manifest, Must be in the above s3_materials_path directory +aki_manifest = %BOTO_AKI_MANIFEST% + +#Instance type +instance_type = %BOTO_FLAVOR_NAME% + +#TCP/IP connection timeout +http_socket_timeout = %BOTO_SOCKET_TIMEOUT% + +# Status change wait timout +build_timeout = %BOTO_BUILD_TIMEOUT% + +# Status change wait interval +build_interval = %BOTO_BUILD_INTERVAL% diff --git a/tempest/common/ssh.py b/tempest/common/ssh.py index faf182a8a3..c03a90cb55 100644 --- a/tempest/common/ssh.py +++ b/tempest/common/ssh.py @@ -20,21 +20,26 @@ import socket import warnings import select +from cStringIO import StringIO from tempest import exceptions with warnings.catch_warnings(): warnings.simplefilter("ignore") import paramiko + from paramiko import RSAKey class Client(object): - def __init__(self, host, username, password=None, timeout=300, + def __init__(self, host, username, password=None, timeout=300, pkey=None, channel_timeout=10, look_for_keys=False, key_filename=None): self.host = host self.username = username self.password = password + if isinstance(pkey, basestring): + pkey = RSAKey.from_private_key(StringIO(str(pkey))) + self.pkey = pkey self.look_for_keys = look_for_keys self.key_filename = key_filename self.timeout = int(timeout) @@ -55,7 +60,7 @@ class Client(object): password=self.password, look_for_keys=self.look_for_keys, key_filename=self.key_filename, - timeout=self.timeout) + timeout=self.timeout, pkey=self.pkey) _timeout = False break except socket.error: diff --git a/tempest/common/utils/file_utils.py b/tempest/common/utils/file_utils.py new file mode 100644 index 0000000000..99047ab3fe --- /dev/null +++ b/tempest/common/utils/file_utils.py @@ -0,0 +1,25 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def have_effective_read_access(path): + try: + fh = open(path, "rb") + except IOError: + return False + fh.close() + return True diff --git a/tempest/common/utils/linux/remote_client.py b/tempest/common/utils/linux/remote_client.py index f7b467beb5..ca1557fd40 100644 --- a/tempest/common/utils/linux/remote_client.py +++ b/tempest/common/utils/linux/remote_client.py @@ -4,25 +4,29 @@ from tempest.common import utils from tempest.exceptions import SSHTimeout, ServerUnreachable import time +import re class RemoteClient(): - def __init__(self, server, username, password): + #Note(afazekas): It should always get an address instead of server + def __init__(self, server, username, password=None, pkey=None): ssh_timeout = TempestConfig().compute.ssh_timeout network = TempestConfig().compute.network_for_ssh ip_version = TempestConfig().compute.ip_version_for_ssh - addresses = server['addresses'][network] + if isinstance(server, basestring): + ip_address = server + else: + addresses = server['addresses'][network] + for address in addresses: + if address['version'] == ip_version: + ip_address = address['addr'] + break + else: + raise ServerUnreachable() - for address in addresses: - if address['version'] == ip_version: - ip_address = address['addr'] - break - - if ip_address is None: - raise ServerUnreachable() - - self.ssh_client = Client(ip_address, username, password, ssh_timeout) + self.ssh_client = Client(ip_address, username, password, ssh_timeout, + pkey=pkey) if not self.ssh_client.test_connection_auth(): raise SSHTimeout() @@ -62,3 +66,9 @@ class RemoteClient(): boot_time_string = self.ssh_client.exec_command(cmd) boot_time_string = boot_time_string.replace('\n', '') return time.strptime(boot_time_string, utils.LAST_REBOOT_TIME_FORMAT) + + def write_to_console(self, message): + message = re.sub("([$\\`])", "\\\\\\\\\\1", message) + # usually to /dev/ttyS0 + cmd = 'sudo sh -c "echo \\"%s\\" >/dev/console"' % message + return self.ssh_client.exec_command(cmd) diff --git a/tempest/config.py b/tempest/config.py index c46a0070f1..0ccd4b6e03 100644 --- a/tempest/config.py +++ b/tempest/config.py @@ -37,6 +37,12 @@ class BaseConfig(object): except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): return default_value + def getboolean(self, item_name, default_value=None): + try: + return self.conf.getboolean(self.SECTION_NAME, item_name) + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + return default_value + class IdentityConfig(BaseConfig): @@ -414,6 +420,80 @@ class ObjectStorageConfig(BaseConfig): return self.get("catalog_type", 'object-store') +class BotoConfig(BaseConfig): + """Provides configuration information for connecting to EC2/S3.""" + SECTION_NAME = "boto" + + @property + def ec2_url(self): + """EC2 URL""" + return self.get("ec2_url", "http://localhost:8773/services/Cloud") + + @property + def s3_url(self): + """S3 URL""" + return self.get("s3_url", "http://localhost:8080") + + @property + def aws_secret(self): + """AWS Secret Key""" + return self.get("aws_secret") + + @property + def aws_access(self): + """AWS Access Key""" + return self.get("aws_access") + + @property + def aws_region(self): + """AWS Region""" + return self.get("aws_region", "RegionOne") + + @property + def s3_materials_path(self): + return self.get("s3_materials_path", + "/opt/stack/devstack/files/images/" + "s3-materials/cirros-0.3.0") + + @property + def ari_manifest(self): + """ARI Ramdisk Image manifest""" + return self.get("ari_manifest", + "cirros-0.3.0-x86_64-initrd.manifest.xml") + + @property + def ami_manifest(self): + """AMI Machine Image manifest""" + return self.get("ami_manifest", + "cirros-0.3.0-x86_64-blank.img.manifest.xml") + + @property + def aki_manifest(self): + """AKI Kernel Image manifest""" + return self.get("aki_manifest", + "cirros-0.3.0-x86_64-vmlinuz.manifest.xml") + + @property + def instance_type(self): + """Instance type""" + return self.get("Instance type", "m1.tiny") + + @property + def http_socket_timeout(self): + """boto Http socket timeout""" + return self.get("http_socket_timeout", "3") + + @property + def build_timeout(self): + """status change timeout""" + return float(self.get("build_timeout", "60")) + + @property + def build_interval(self): + """status change test interval""" + return float(self.get("build_interval", 1)) + + # TODO(jaypipes): Move this to a common utils (not data_utils...) def singleton(cls): """Simple wrapper for classes that should only have a single instance""" @@ -463,6 +543,7 @@ class TempestConfig: self.network = NetworkConfig(self._conf) self.volume = VolumeConfig(self._conf) self.object_storage = ObjectStorageConfig(self._conf) + self.boto = BotoConfig(self._conf) def load_config(self, path): """Read configuration from given path and return a config object.""" diff --git a/tempest/exceptions.py b/tempest/exceptions.py index 7154b80fb4..016de69ad4 100644 --- a/tempest/exceptions.py +++ b/tempest/exceptions.py @@ -1,3 +1,21 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + class TempestException(Exception): """ Base Tempest Exception @@ -51,6 +69,11 @@ class AddImageException(TempestException): message = "Image %(image_id) failed to become ACTIVE in the allotted time" +class EC2RegisterImageException(TempestException): + message = ("Image %(image_id) failed to become 'available' " + "in the allotted time") + + class VolumeBuildErrorException(TempestException): message = "Volume %(volume_id)s failed to build and is in ERROR status" @@ -106,3 +129,7 @@ class ServerUnreachable(TempestException): class SQLException(TempestException): message = "SQL error: %(message)s" + + +class TearDownException(TempestException): + message = "%(num)d cleanUp operation failed" diff --git a/tempest/openstack.py b/tempest/openstack.py index 35562b1e76..dc73bd730b 100644 --- a/tempest/openstack.py +++ b/tempest/openstack.py @@ -57,6 +57,8 @@ from tempest.services.volume.xml.volumes_client import VolumesClientXML from tempest.services.object_storage.account_client import AccountClient from tempest.services.object_storage.container_client import ContainerClient from tempest.services.object_storage.object_client import ObjectClient +from tempest.services.boto.clients import APIClientEC2 +from tempest.services.boto.clients import ObjectClientS3 LOG = logging.getLogger(__name__) @@ -186,6 +188,8 @@ class Manager(object): self.account_client = AccountClient(*client_args) self.container_client = ContainerClient(*client_args) self.object_client = ObjectClient(*client_args) + self.ec2api_client = APIClientEC2(*client_args) + self.s3_client = ObjectClientS3(*client_args) class AltManager(Manager): diff --git a/tempest/services/boto/__init__.py b/tempest/services/boto/__init__.py new file mode 100644 index 0000000000..9b9fcebca6 --- /dev/null +++ b/tempest/services/boto/__init__.py @@ -0,0 +1,102 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import boto + +from ConfigParser import DuplicateSectionError + +from tempest.exceptions import InvalidConfiguration +from tempest.exceptions import NotFound + +import re +from types import MethodType +from contextlib import closing + + +class BotoClientBase(object): + + ALLOWED_METHODS = set() + + def __init__(self, config, + username=None, password=None, + auth_url=None, tenant_name=None, + *args, **kwargs): + + self.connection_timeout = config.boto.http_socket_timeout + self.build_timeout = config.boto.build_timeout + # We do not need the "path": "/token" part + if auth_url: + auth_url = re.sub("(.*)" + re.escape(config.identity.path) + "$", + "\\1", auth_url) + self.ks_cred = {"username": username, + "password": password, + "auth_url": auth_url, + "tenant_name": tenant_name} + + def _keystone_aws_get(self): + import keystoneclient.v2_0.client + + keystone = keystoneclient.v2_0.client.Client(**self.ks_cred) + ec2_cred_list = keystone.ec2.list(keystone.auth_user_id) + ec2_cred = None + for cred in ec2_cred_list: + if cred.tenant_id == keystone.auth_tenant_id: + ec2_cred = cred + break + else: + ec2_cred = keystone.ec2.create(keystone.auth_user_id, + keystone.auth_tenant_id) + if not all((ec2_cred, ec2_cred.access, ec2_cred.secret)): + raise NotFound("Unable to get access and secret keys") + return ec2_cred + + def _config_boto_timeout(self, timeout): + try: + boto.config.add_section("Boto") + except DuplicateSectionError: + pass + boto.config.set("Boto", "http_socket_timeout", timeout) + + def __getattr__(self, name): + """Automatically creates methods for the allowed methods set""" + if name in self.ALLOWED_METHODS: + def func(self, *args, **kwargs): + with closing(self.get_connection()) as conn: + return getattr(conn, name)(*args, **kwargs) + + func.__name__ = name + setattr(self, name, MethodType(func, self, self.__class__)) + setattr(self.__class__, name, + MethodType(func, None, self.__class__)) + return getattr(self, name) + else: + raise AttributeError(name) + + def get_connection(self): + self._config_boto_timeout(self.connection_timeout) + if not all((self.connection_data["aws_access_key_id"], + self.connection_data["aws_secret_access_key"])): + if all(self.ks_cred.itervalues()): + ec2_cred = self._keystone_aws_get() + self.connection_data["aws_access_key_id"] = \ + ec2_cred.access + self.connection_data["aws_secret_access_key"] = \ + ec2_cred.secret + else: + raise InvalidConfiguration( + "Unable to get access and secret keys") + return self.connect_method(**self.connection_data) diff --git a/tempest/services/boto/clients.py b/tempest/services/boto/clients.py new file mode 100644 index 0000000000..5fabcae2d7 --- /dev/null +++ b/tempest/services/boto/clients.py @@ -0,0 +1,139 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import boto +from boto.s3.connection import OrdinaryCallingFormat +from boto.ec2.regioninfo import RegionInfo +from tempest.services.boto import BotoClientBase +import urlparse + + +class APIClientEC2(BotoClientBase): + + def connect_method(self, *args, **kwargs): + return boto.connect_ec2(*args, **kwargs) + + def __init__(self, config, *args, **kwargs): + super(APIClientEC2, self).__init__(config, *args, **kwargs) + aws_access = config.boto.aws_access + aws_secret = config.boto.aws_secret + purl = urlparse.urlparse(config.boto.ec2_url) + + region = RegionInfo(name=config.boto.aws_region, + endpoint=purl.hostname) + port = purl.port + if port is None: + if purl.scheme is not "https": + port = 80 + else: + port = 443 + else: + port = int(port) + self.connection_data = {"aws_access_key_id": aws_access, + "aws_secret_access_key": aws_secret, + "is_secure": purl.scheme == "https", + "region": region, + "host": purl.hostname, + "port": port, + "path": purl.path} + + ALLOWED_METHODS = set(('create_key_pair', 'get_key_pair', + 'delete_key_pair', 'import_key_pair', + 'get_all_key_pairs', + 'create_image', 'get_image', + 'register_image', 'deregister_image', + 'get_all_images', 'get_image_attribute', + 'modify_image_attribute', 'reset_image_attribute', + 'get_all_kernels', + 'create_volume', 'delete_volume', + 'get_all_volume_status', 'get_all_volumes', + 'get_volume_attribute', 'modify_volume_attribute' + 'bundle_instance', 'cancel_spot_instance_requests', + 'confirm_product_instanc', + 'get_all_instance_status', 'get_all_instances', + 'get_all_reserved_instances', + 'get_all_spot_instance_requests', + 'get_instance_attribute', 'monitor_instance', + 'monitor_instances', 'unmonitor_instance', + 'unmonitor_instances', + 'purchase_reserved_instance_offering', + 'reboot_instances', 'request_spot_instances', + 'reset_instance_attribute', 'run_instances', + 'start_instances', 'stop_instances', + 'terminate_instances', + 'attach_network_interface', 'attach_volume', + 'detach_network_interface', 'detach_volume', + 'get_console_output', + 'delete_network_interface', 'create_subnet', + 'create_network_interface', 'delete_subnet', + 'get_all_network_interfaces', + 'allocate_address', 'associate_address', + 'disassociate_address', 'get_all_addresses', + 'release_address', + 'create_snapshot', 'delete_snapshot', + 'get_all_snapshots', 'get_snapshot_attribute', + 'modify_snapshot_attribute', + 'reset_snapshot_attribute', 'trim_snapshots', + 'get_all_regions', 'get_all_zones', + 'get_all_security_groups', 'create_security_group', + 'delete_security_group', 'authorize_security_group', + 'authorize_security_group_egress', + 'revoke_security_group', + 'revoke_security_group_egress')) + + def get_good_zone(self): + """ + :rtype: BaseString + :return: Returns with the first available zone name + """ + for zone in self.get_all_zones(): + #NOTE(afazekas): zone.region_name was None + if (zone.state == "available" and + zone.region.name == self.connection_data["region"].name): + return zone.name + else: + raise IndexError("Don't have a good zone") + + +class ObjectClientS3(BotoClientBase): + + def connect_method(self, *args, **kwargs): + return boto.connect_s3(*args, **kwargs) + + def __init__(self, config, *args, **kwargs): + super(ObjectClientS3, self).__init__(config, *args, **kwargs) + aws_access = config.boto.aws_access + aws_secret = config.boto.aws_secret + purl = urlparse.urlparse(config.boto.s3_url) + port = purl.port + if port is None: + if purl.scheme is not "https": + port = 80 + else: + port = 443 + else: + port = int(port) + self.connection_data = {"aws_access_key_id": aws_access, + "aws_secret_access_key": aws_secret, + "is_secure": purl.scheme == "https", + "host": purl.hostname, + "port": port, + "calling_format": OrdinaryCallingFormat()} + + ALLOWED_METHODS = set(('create_bucket', 'delete_bucket', 'generate_url', + 'get_all_buckets', 'get_bucket', 'delete_key', + 'lookup')) diff --git a/tempest/testboto.py b/tempest/testboto.py new file mode 100644 index 0000000000..6c5134679c --- /dev/null +++ b/tempest/testboto.py @@ -0,0 +1,535 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest2 as unittest +import nose +import tempest.tests.boto +from tempest.exceptions import TearDownException +from tempest.tests.boto.utils.wait import state_wait, wait_no_exception +from tempest.tests.boto.utils.wait import re_search_wait, wait_exception +import boto +from boto.s3.key import Key +from boto.s3.bucket import Bucket +from boto.exception import BotoServerError +from contextlib import closing +import re +import logging +import time + +LOG = logging.getLogger(__name__) + + +class BotoExceptionMatcher(object): + STATUS_RE = r'[45]\d\d' + CODE_RE = '.*' # regexp makes sense in group match + + def match(self, exc): + if not isinstance(exc, BotoServerError): + return "%r not an BotoServerError instance" % exc + LOG.info("Status: %s , error_code: %s", exc.status, exc.error_code) + if re.match(self.STATUS_RE, str(exc.status)) is None: + return ("Status code (%s) does not match" + "the expected re pattern \"%s\"" + % (exc.status, self.STATUS_RE)) + if re.match(self.CODE_RE, str(exc.error_code)) is None: + return ("Error code (%s) does not match" + + "the expected re pattern \"%s\"") %\ + (exc.error_code, self.CODE_RE) + + +class ClientError(BotoExceptionMatcher): + STATUS_RE = r'4\d\d' + + +class ServerError(BotoExceptionMatcher): + STATUS_RE = r'5\d\d' + + +def _add_matcher_class(error_cls, error_data, base=BotoExceptionMatcher): + """ + Usable for adding an ExceptionMatcher(s) into the exception tree. + The not leaf elements does wildcard match + """ + # in error_code just literal and '.' characters expected + if not isinstance(error_data, basestring): + (error_code, status_code) = map(str, error_data) + else: + status_code = None + error_code = error_data + parts = error_code.split('.') + basematch = "" + num_parts = len(parts) + max_index = num_parts - 1 + add_cls = error_cls + for i_part in xrange(num_parts): + part = parts[i_part] + leaf = i_part == max_index + if not leaf: + match = basematch + part + "[.].*" + else: + match = basematch + part + + basematch += part + "[.]" + if not hasattr(add_cls, part): + cls_dict = {"CODE_RE": match} + if leaf and status_code is not None: + cls_dict["STATUS_RE"] = status_code + cls = type(part, (base, ), cls_dict) + setattr(add_cls, part, cls()) + add_cls = cls + elif leaf: + raise LookupError("Tries to redefine an error code \"%s\"" % part) + else: + add_cls = getattr(add_cls, part) + + +#TODO(afazekas): classmethod handling +def friendly_function_name_simple(call_able): + name = "" + if hasattr(call_able, "im_class"): + name += call_able.im_class.__name__ + "." + name += call_able.__name__ + return name + + +def friendly_function_call_str(call_able, *args, **kwargs): + string = friendly_function_name_simple(call_able) + string += "(" + ", ".join(map(str, args)) + if len(kwargs): + if len(args): + string += ", " + string += ", ".join("=".join(map(str, (key, value))) + for (key, value) in kwargs.items()) + return string + ")" + + +class BotoTestCase(unittest.TestCase): + """Recommended to use as base class for boto related test""" + @classmethod + def setUpClass(cls): + # The trash contains cleanup functions and paramaters in tuples + # (function, *args, **kwargs) + cls._resource_trash_bin = {} + cls._sequence = -1 + if (hasattr(cls, "EC2") and + tempest.tests.boto.EC2_CAN_CONNECT_ERROR is not None): + raise nose.SkipTest("EC2 " + cls.__name__ + ": " + + tempest.tests.boto.EC2_CAN_CONNECT_ERROR) + if (hasattr(cls, "S3") and + tempest.tests.boto.S3_CAN_CONNECT_ERROR is not None): + raise nose.SkipTest("S3 " + cls.__name__ + ": " + + tempest.tests.boto.S3_CAN_CONNECT_ERROR) + + @classmethod + def addResourceCleanUp(cls, function, *args, **kwargs): + """Adds CleanUp callable, used by tearDownClass. + Recommended to a use (deep)copy on the mutable args""" + cls._sequence = cls._sequence + 1 + cls._resource_trash_bin[cls._sequence] = (function, args, kwargs) + return cls._sequence + + @classmethod + def cancelResourceCleanUp(cls, key): + """Cancel Clean up request""" + del cls._resource_trash_bin[key] + + #TODO(afazekas): Add "with" context handling + def assertBotoError(self, excMatcher, callableObj, + *args, **kwargs): + """Example usage: + self.assertBotoError(self.ec2_error_code.client. + InvalidKeyPair.Duplicate, + self.client.create_keypair, + key_name)""" + try: + callableObj(*args, **kwargs) + except BotoServerError as exc: + error_msg = excMatcher.match(exc) + if error_msg is not None: + raise self.failureException, error_msg + else: + raise self.failureException, "BotoServerError not raised" + + @classmethod + def tearDownClass(cls): + """ Calls the callables added by addResourceCleanUp, + when you overwire this function dont't forget to call this too""" + fail_count = 0 + trash_keys = sorted(cls._resource_trash_bin, reverse=True) + for key in trash_keys: + (function, pos_args, kw_args) = cls._resource_trash_bin[key] + try: + LOG.debug("Cleaning up: %s" % + friendly_function_call_str(function, *pos_args, + **kw_args)) + function(*pos_args, **kw_args) + except BaseException as exc: + fail_count += 1 + LOG.exception(exc) + finally: + del cls._resource_trash_bin[key] + if fail_count: + raise TearDownException(num=fail_count) + + ec2_error_code = BotoExceptionMatcher() + # InsufficientInstanceCapacity can be both server and client error + ec2_error_code.server = ServerError() + ec2_error_code.client = ClientError() + s3_error_code = BotoExceptionMatcher() + s3_error_code.server = ServerError() + s3_error_code.client = ClientError() + valid_image_state = set(('available', 'pending', 'failed')) + valid_instance_state = set(('pending', 'running', 'shutting-down', + 'terminated', 'stopping', 'stopped')) + valid_volume_status = set(('creating', 'available', 'in-use', + 'deleting', 'deleted', 'error')) + valid_snapshot_status = set(('pending', 'completed', 'error')) + + #TODO(afazekas): object base version for resurces supports update + def waitImageState(self, lfunction, wait_for): + state = state_wait(lfunction, wait_for, self.valid_image_state) + self.assertIn(state, self.valid_image_state) + return state + + def waitInstanceState(self, lfunction, wait_for): + state = state_wait(lfunction, wait_for, self.valid_instance_state) + self.assertIn(state, self.valid_instance_state) + return state + + def waitVolumeStatus(self, lfunction, wait_for): + state = state_wait(lfunction, wait_for, self.valid_volume_status) + self.assertIn(state, self.valid_volume_status) + return state + + def waitSnapshotStatus(self, lfunction, wait_for): + state = state_wait(lfunction, wait_for, self.valid_snapshot_status) + self.assertIn(state, self.valid_snapshot_status) + return state + + def assertImageStateWait(self, lfunction, wait_for): + state = self.waitImageState(lfunction, wait_for) + self.assertIn(state, wait_for) + + def assertInstanceStateWait(self, lfunction, wait_for): + state = self.waitInstanceState(lfunction, wait_for) + self.assertIn(state, wait_for) + + def assertVolumeStatusWait(self, lfunction, wait_for): + state = self.waitVolumeStatus(lfunction, wait_for) + self.assertIn(state, wait_for) + + def assertSnapshotStatusWait(self, lfunction, wait_for): + state = self.waitSnapshotStatus(lfunction, wait_for) + self.assertIn(state, wait_for) + + def assertAddressDissasociatedWait(self, address): + + def _disassociate(): + cli = self.ec2_client + addresses = cli.get_all_addresses(addresses=(address.public_ip,)) + if len(addresses) != 1: + return "INVALID" + if addresses[0].instance_id: + LOG.info("%s associated to %s", + address.public_ip, + addresses[0].instance_id) + return "ASSOCIATED" + return "DISASSOCIATED" + + state = state_wait(_disassociate, "DISASSOCIATED", + set(("ASSOCIATED", "DISASSOCIATED"))) + self.assertEqual(state, "DISASSOCIATED") + + def assertAddressReleasedWait(self, address): + + def _address_delete(): + #NOTE(afazekas): the filter gives back IP + # even if it is not associated to my tenant + if (address.public_ip not in map(lambda a: a.public_ip, + self.ec2_client.get_all_addresses())): + return "DELETED" + return "NOTDELETED" + + state = state_wait(_address_delete, "DELETED") + self.assertEqual(state, "DELETED") + + def assertReSearch(self, regexp, string): + if re.search(regexp, string) is None: + raise self.failureException("regexp: '%s' not found in '%s'" % + (regexp, string)) + + def assertNotReSearch(self, regexp, string): + if re.search(regexp, string) is not None: + raise self.failureException("regexp: '%s' found in '%s'" % + (regexp, string)) + + def assertReMatch(self, regexp, string): + if re.match(regexp, string) is None: + raise self.failureException("regexp: '%s' not matches on '%s'" % + (regexp, string)) + + def assertNotReMatch(self, regexp, string): + if re.match(regexp, string) is not None: + raise self.failureException("regexp: '%s' matches on '%s'" % + (regexp, string)) + + @classmethod + def destroy_bucket(cls, connection_data, bucket): + """Destroys the bucket and its content, just for teardown""" + exc_num = 0 + try: + with closing(boto.connect_s3(**connection_data)) as conn: + if isinstance(bucket, basestring): + bucket = conn.lookup(bucket) + assert isinstance(bucket, Bucket) + for obj in bucket.list(): + try: + bucket.delete_key(obj.key) + obj.close() + except BaseException as exc: + LOG.exception(exc) + exc_num += 1 + conn.delete_bucket(bucket) + except BaseException as exc: + LOG.exception(exc) + exc_num += 1 + if exc_num: + raise TearDownException(num=exc_num) + + @classmethod + def destroy_reservation(cls, reservation): + """Terminate instances in a reservation, just for teardown""" + exc_num = 0 + + def _instance_state(): + try: + instance.update(validate=True) + except ValueError: + return "terminated" + return instance.state + + for instance in reservation.instances: + try: + instance.terminate() + re_search_wait(_instance_state, "terminated") + except BaseException as exc: + LOG.exception(exc) + exc_num += 1 + if exc_num: + raise TearDownException(num=exc_num) + + #NOTE(afazekas): The incorrect ErrorCodes makes very, very difficult + # to write better teardown + + @classmethod + def destroy_security_group_wait(cls, group): + """Delete group. + Use just for teardown! + """ + #NOTE(afazekas): should wait/try until all related instance terminates + #2. looks like it is locked even if the instance not listed + time.sleep(1) + group.delete() + + @classmethod + def destroy_volume_wait(cls, volume): + """Delete volume, tryies to detach first. + Use just for teardown! + """ + exc_num = 0 + snaps = volume.snapshots() + if len(snaps): + LOG.critical("%s Volume has %s snapshot(s)", volume.id, + map(snps.id, snaps)) + + #Note(afazekas): detaching/attching not valid EC2 status + def _volume_state(): + volume.update(validate=True) + try: + if volume.status != "available": + volume.detach(force=True) + except BaseException as exc: + LOG.exception(exc) + #exc_num += 1 "nonlocal" not in python2 + return volume.status + + try: + re_search_wait(_volume_state, "available") # not validates status + LOG.info(_volume_state()) + volume.delete() + except BaseException as exc: + LOG.exception(exc) + exc_num += 1 + if exc_num: + raise TearDownException(num=exc_num) + + @classmethod + def destroy_snapshot_wait(cls, snapshot): + """delete snaphot, wait until not exists""" + snapshot.delete() + + def _update(): + snapshot.update(validate=True) + + wait_exception(_update) + + +# you can specify tuples if you want to specify the status pattern +for code in ('AddressLimitExceeded', 'AttachmentLimitExceeded', 'AuthFailure', + 'Blocked', 'CustomerGatewayLimitExceeded', 'DependencyViolation', + 'DiskImageSizeTooLarge', 'FilterLimitExceeded', + 'Gateway.NotAttached', 'IdempotentParameterMismatch', + 'IncorrectInstanceState', 'IncorrectState', + 'InstanceLimitExceeded', 'InsufficientInstanceCapacity', + 'InsufficientReservedInstancesCapacity', + 'InternetGatewayLimitExceeded', 'InvalidAMIAttributeItemValue', + 'InvalidAMIID.Malformed', 'InvalidAMIID.NotFound', + 'InvalidAMIID.Unavailable', 'InvalidAssociationID.NotFound', + 'InvalidAttachment.NotFound', 'InvalidConversionTaskId', + 'InvalidCustomerGateway.DuplicateIpAddress', + 'InvalidCustomerGatewayID.NotFound', 'InvalidDevice.InUse', + 'InvalidDhcpOptionsID.NotFound', 'InvalidFormat', + 'InvalidFilter', 'InvalidGatewayID.NotFound', + 'InvalidGroup.Duplicate', 'InvalidGroupId.Malformed', + 'InvalidGroup.InUse', 'InvalidGroup.NotFound', + 'InvalidGroup.Reserved', 'InvalidInstanceID.Malformed', + 'InvalidInstanceID.NotFound', + 'InvalidInternetGatewayID.NotFound', 'InvalidIPAddress.InUse', + 'InvalidKeyPair.Duplicate', 'InvalidKeyPair.Format', + 'InvalidKeyPair.NotFound', 'InvalidManifest', + 'InvalidNetworkAclEntry.NotFound', + 'InvalidNetworkAclID.NotFound', 'InvalidParameterCombination', + 'InvalidParameterValue', 'InvalidPermission.Duplicate', + 'InvalidPermission.Malformed', 'InvalidReservationID.Malformed', + 'InvalidReservationID.NotFound', 'InvalidRoute.NotFound', + 'InvalidRouteTableID.NotFound', + 'InvalidSecurity.RequestHasExpired', + 'InvalidSnapshotID.Malformed', 'InvalidSnapshot.NotFound', + 'InvalidUserID.Malformed', 'InvalidReservedInstancesId', + 'InvalidReservedInstancesOfferingId', + 'InvalidSubnetID.NotFound', 'InvalidVolumeID.Duplicate', + 'InvalidVolumeID.Malformed', 'InvalidVolumeID.ZoneMismatch', + 'InvalidVolume.NotFound', 'InvalidVpcID.NotFound', + 'InvalidVpnConnectionID.NotFound', + 'InvalidVpnGatewayID.NotFound', + 'InvalidZone.NotFound', 'LegacySecurityGroup', + 'MissingParameter', 'NetworkAclEntryAlreadyExists', + 'NetworkAclEntryLimitExceeded', 'NetworkAclLimitExceeded', + 'NonEBSInstance', 'PendingSnapshotLimitExceeded', + 'PendingVerification', 'OptInRequired', 'RequestLimitExceeded', + 'ReservedInstancesLimitExceeded', 'Resource.AlreadyAssociated', + 'ResourceLimitExceeded', 'RouteAlreadyExists', + 'RouteLimitExceeded', 'RouteTableLimitExceeded', + 'RulesPerSecurityGroupLimitExceeded', + 'SecurityGroupLimitExceeded', + 'SecurityGroupsPerInstanceLimitExceeded', + 'SnapshotLimitExceeded', 'SubnetLimitExceeded', + 'UnknownParameter', 'UnsupportedOperation', + 'VolumeLimitExceeded', 'VpcLimitExceeded', + 'VpnConnectionLimitExceeded', + 'VpnGatewayAttachmentLimitExceeded', 'VpnGatewayLimitExceeded'): + _add_matcher_class(BotoTestCase.ec2_error_code.client, + code, base=ClientError) + +for code in ('InsufficientAddressCapacity', 'InsufficientInstanceCapacity', + 'InsufficientReservedInstanceCapacity', 'InternalError', + 'Unavailable'): + _add_matcher_class(BotoTestCase.ec2_error_code.server, + code, base=ServerError) + + +for code in (('AccessDenied', 403), + ('AccountProblem', 403), + ('AmbiguousGrantByEmailAddress', 400), + ('BadDigest', 400), + ('BucketAlreadyExists', 409), + ('BucketAlreadyOwnedByYou', 409), + ('BucketNotEmpty', 409), + ('CredentialsNotSupported', 400), + ('CrossLocationLoggingProhibited', 403), + ('EntityTooSmall', 400), + ('EntityTooLarge', 400), + ('ExpiredToken', 400), + ('IllegalVersioningConfigurationException', 400), + ('IncompleteBody', 400), + ('IncorrectNumberOfFilesInPostRequest', 400), + ('InlineDataTooLarge', 400), + ('InvalidAccessKeyId', 403), + 'InvalidAddressingHeader', + ('InvalidArgument', 400), + ('InvalidBucketName', 400), + ('InvalidBucketState', 409), + ('InvalidDigest', 400), + ('InvalidLocationConstraint', 400), + ('InvalidPart', 400), + ('InvalidPartOrder', 400), + ('InvalidPayer', 403), + ('InvalidPolicyDocument', 400), + ('InvalidRange', 416), + ('InvalidRequest', 400), + ('InvalidSecurity', 403), + ('InvalidSOAPRequest', 400), + ('InvalidStorageClass', 400), + ('InvalidTargetBucketForLogging', 400), + ('InvalidToken', 400), + ('InvalidURI', 400), + ('KeyTooLong', 400), + ('MalformedACLError', 400), + ('MalformedPOSTRequest', 400), + ('MalformedXML', 400), + ('MaxMessageLengthExceeded', 400), + ('MaxPostPreDataLengthExceededError', 400), + ('MetadataTooLarge', 400), + ('MethodNotAllowed', 405), + ('MissingAttachment'), + ('MissingContentLength', 411), + ('MissingRequestBodyError', 400), + ('MissingSecurityElement', 400), + ('MissingSecurityHeader', 400), + ('NoLoggingStatusForKey', 400), + ('NoSuchBucket', 404), + ('NoSuchKey', 404), + ('NoSuchLifecycleConfiguration', 404), + ('NoSuchUpload', 404), + ('NoSuchVersion', 404), + ('NotSignedUp', 403), + ('NotSuchBucketPolicy', 404), + ('OperationAborted', 409), + ('PermanentRedirect', 301), + ('PreconditionFailed', 412), + ('Redirect', 307), + ('RequestIsNotMultiPartContent', 400), + ('RequestTimeout', 400), + ('RequestTimeTooSkewed', 403), + ('RequestTorrentOfBucketError', 400), + ('SignatureDoesNotMatch', 403), + ('TemporaryRedirect', 307), + ('TokenRefreshRequired', 400), + ('TooManyBuckets', 400), + ('UnexpectedContent', 400), + ('UnresolvableGrantByEmailAddress', 400), + ('UserKeyMustBeSpecified', 400)): + _add_matcher_class(BotoTestCase.s3_error_code.client, + code, base=ClientError) + + +for code in (('InternalError', 500), + ('NotImplemented', 501), + ('ServiceUnavailable', 503), + ('SlowDown', 503)): + _add_matcher_class(BotoTestCase.s3_error_code.server, + code, base=ServerError) diff --git a/tempest/tests/boto/__init__.py b/tempest/tests/boto/__init__.py new file mode 100644 index 0000000000..3d5ea6c73a --- /dev/null +++ b/tempest/tests/boto/__init__.py @@ -0,0 +1,98 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import tempest.config +from tempest.common.utils.file_utils import have_effective_read_access +import os +import tempest.openstack +import re +import keystoneclient.exceptions +import boto.exception +import logging +import urlparse + +A_I_IMAGES_READY = False # ari,ami,aki +S3_CAN_CONNECT_ERROR = "Unknown Error" +EC2_CAN_CONNECT_ERROR = "Unknown Error" + + +def setup_package(): + global A_I_IMAGES_READY + global S3_CAN_CONNECT_ERROR + global EC2_CAN_CONNECT_ERROR + secret_matcher = re.compile("[A-Za-z0-9+/]{32,}") # 40 in other system + id_matcher = re.compile("[A-Za-z0-9]{20,}") + + def all_read(*args): + return all(map(have_effective_read_access, args)) + + config = tempest.config.TempestConfig() + materials_path = config.boto.s3_materials_path + ami_path = materials_path + os.sep + config.boto.ami_manifest + aki_path = materials_path + os.sep + config.boto.aki_manifest + ari_path = materials_path + os.sep + config.boto.ari_manifest + + A_I_IMAGES_READY = all_read(ami_path, aki_path, ari_path) + boto_logger = logging.getLogger('boto') + level = boto_logger.level + boto_logger.setLevel(logging.CRITICAL) # suppress logging for these + + def _cred_sub_check(connection_data): + if not id_matcher.match(connection_data["aws_access_key_id"]): + raise Exception("Invalid AWS access Key") + if not secret_matcher.match(connection_data["aws_secret_access_key"]): + raise Exception("Invalid AWS secret Key") + raise Exception("Unknown (Authentication?) Error") + openstack = tempest.openstack.Manager() + try: + if urlparse.urlparse(config.boto.ec2_url).hostname is None: + raise Exception("Failed to get hostname from the ec2_url") + ec2client = openstack.ec2api_client + try: + ec2client.get_all_regions() + except boto.exception.BotoServerError as exc: + if exc.error_code is None: + raise Exception("EC2 target does not looks EC2 service") + _cred_sub_check(ec2client.connection_data) + + except keystoneclient.exceptions.Unauthorized: + EC2_CAN_CONNECT_ERROR = "AWS credentials not set," +\ + " faild to get them even by keystoneclient" + except Exception as exc: + logging.exception(exc) + EC2_CAN_CONNECT_ERROR = str(exc) + else: + EC2_CAN_CONNECT_ERROR = None + + try: + if urlparse.urlparse(config.boto.s3_url).hostname is None: + raise Exception("Failed to get hostname from the s3_url") + s3client = openstack.s3_client + try: + s3client.get_bucket("^INVALID*#()@INVALID.") + except boto.exception.BotoServerError as exc: + if exc.status == 403: + _cred_sub_check(s3client.connection_data) + except Exception as exc: + logging.exception(exc) + S3_CAN_CONNECT_ERROR = str(exc) + except keystoneclient.exceptions.Unauthorized: + S3_CAN_CONNECT_ERROR = "AWS credentials not set," +\ + " faild to get them even by keystoneclient" + else: + S3_CAN_CONNECT_ERROR = None + boto_logger.setLevel(level) diff --git a/tempest/tests/boto/test_ec2_instance_run.py b/tempest/tests/boto/test_ec2_instance_run.py new file mode 100644 index 0000000000..e5c61fb20c --- /dev/null +++ b/tempest/tests/boto/test_ec2_instance_run.py @@ -0,0 +1,249 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import nose +from nose.plugins.attrib import attr +import unittest2 as unittest +from tempest.testboto import BotoTestCase +from tempest.tests.boto.utils.s3 import s3_upload_dir +import tempest.tests.boto +from tempest.common.utils.data_utils import rand_name +from tempest.exceptions import EC2RegisterImageException +from tempest.tests.boto.utils.wait import state_wait, re_search_wait +from tempest import openstack +from tempest.common.utils.linux.remote_client import RemoteClient +from boto.s3.key import Key +from contextlib import closing +import logging + + +LOG = logging.getLogger(__name__) + + +@attr("S3", "EC2") +class InstanceRunTest(BotoTestCase): + + @classmethod + def setUpClass(cls): + super(InstanceRunTest, cls).setUpClass() + if not tempest.tests.boto.A_I_IMAGES_READY: + raise nose.SkipTest("".join(("EC2 ", cls.__name__, + ": requires ami/aki/ari manifest"))) + cls.os = openstack.Manager() + cls.s3_client = cls.os.s3_client + cls.ec2_client = cls.os.ec2api_client + config = cls.os.config + cls.zone = cls.ec2_client.get_good_zone() + cls.materials_path = config.boto.s3_materials_path + ami_manifest = config.boto.ami_manifest + aki_manifest = config.boto.aki_manifest + ari_manifest = config.boto.ari_manifest + cls.instance_type = config.boto.instance_type + cls.bucket_name = rand_name("s3bucket-") + cls.keypair_name = rand_name("keypair-") + cls.keypair = cls.ec2_client.create_key_pair(cls.keypair_name) + cls.addResourceCleanUp(cls.ec2_client.delete_key_pair, + cls.keypair_name) + bucket = cls.s3_client.create_bucket(cls.bucket_name) + cls.addResourceCleanUp(cls.destroy_bucket, + cls.s3_client.connection_data, + cls.bucket_name) + s3_upload_dir(bucket, cls.materials_path) + cls.images = {"ami": + {"name": rand_name("ami-name-"), + "location": cls.bucket_name + "/" + ami_manifest}, + "aki": + {"name": rand_name("aki-name-"), + "location": cls.bucket_name + "/" + aki_manifest}, + "ari": + {"name": rand_name("ari-name-"), + "location": cls.bucket_name + "/" + ari_manifest}} + for image in cls.images.itervalues(): + image["image_id"] = cls.ec2_client.register_image( + name=image["name"], + image_location=image["location"]) + cls.addResourceCleanUp(cls.ec2_client.deregister_image, + image["image_id"]) + + for image in cls.images.itervalues(): + def _state(): + retr = cls.ec2_client.get_image(image["image_id"]) + return retr.state + state = state_wait(_state, "available") + if state != "available": + for _image in cls.images.itervalues(): + ec2_client.deregister_image(_image["image_id"]) + raise RegisterImageException(image_id=image["image_id"]) + + @attr(type='smoke') + def test_run_stop_terminate_instance(self): + """EC2 run, stop and terminate instance""" + image_ami = self.ec2_client.get_image(self.images["ami"] + ["image_id"]) + reservation = image_ami.run(kernel_id=self.images["aki"]["image_id"], + ramdisk_id=self.images["ari"]["image_id"], + instance_type=self.instance_type) + rcuk = self.addResourceCleanUp(self.destroy_reservation, reservation) + + def _state(): + instance.update(validate=True) + return instance.state + + for instance in reservation.instances: + LOG.info("state: %s", instance.state) + if instance.state != "running": + self.assertInstanceStateWait(_state, "running") + + for instance in reservation.instances: + instance.stop() + LOG.info("state: %s", instance.state) + if instance.state != "stopped": + self.assertInstanceStateWait(_state, "stopped") + + for instance in reservation.instances: + instance.terminate() + self.cancelResourceCleanUp(rcuk) + + @attr(type='smoke') + def test_run_terminate_instance(self): + """EC2 run, terminate immediately""" + image_ami = self.ec2_client.get_image(self.images["ami"] + ["image_id"]) + reservation = image_ami.run(kernel_id=self.images["aki"]["image_id"], + ramdisk_id=self.images["ari"]["image_id"], + instance_type=self.instance_type) + + for instance in reservation.instances: + instance.terminate() + + instance.update(validate=True) + self.assertNotEqual(instance.state, "running") + + #NOTE(afazekas): doctored test case, + # with normal validation it would fail + @attr("slow", type='smoke') + def test_integration_1(self): + """EC2 1. integration test (not strict)""" + image_ami = self.ec2_client.get_image(self.images["ami"]["image_id"]) + sec_group_name = rand_name("securitygroup-") + group_desc = sec_group_name + " security group description " + security_group = self.ec2_client.create_security_group(sec_group_name, + group_desc) + self.addResourceCleanUp(self.destroy_security_group_wait, + security_group) + self.ec2_client.authorize_security_group(sec_group_name, + ip_protocol="icmp", + cidr_ip="0.0.0.0/0", + from_port=-1, + to_port=-1) + self.ec2_client.authorize_security_group(sec_group_name, + ip_protocol="tcp", + cidr_ip="0.0.0.0/0", + from_port=22, + to_port=22) + reservation = image_ami.run(kernel_id=self.images["aki"]["image_id"], + ramdisk_id=self.images["ari"]["image_id"], + instance_type=self.instance_type, + key_name=self.keypair_name, + security_groups=(sec_group_name,)) + self.addResourceCleanUp(self.destroy_reservation, + reservation) + volume = self.ec2_client.create_volume(1, self.zone) + self.addResourceCleanUp(self.destroy_volume_wait, volume) + instance = reservation.instances[0] + + def _instance_state(): + instance.update(validate=True) + return instance.state + + def _volume_state(): + volume.update(validate=True) + return volume.status + + LOG.info("state: %s", instance.state) + if instance.state != "running": + self.assertInstanceStateWait(_instance_state, "running") + + address = self.ec2_client.allocate_address() + rcuk_a = self.addResourceCleanUp(address.delete) + address.associate(instance.id) + + rcuk_da = self.addResourceCleanUp(address.disassociate) + #TODO(afazekas): ping test. dependecy/permission ? + + self.assertVolumeStatusWait(_volume_state, "available") + #NOTE(afazekas): it may be reports availble before it is available + + ssh = RemoteClient(address.public_ip, + self.os.config.compute.ssh_user, + pkey=self.keypair.material) + text = rand_name("Pattern text for console output -") + resp = ssh.write_to_console(text) + self.assertFalse(resp) + + def _output(): + output = instance.get_console_output() + return output.output + + re_search_wait(_output, text) + part_lines = ssh.get_partitions().split('\n') + # "attaching" invalid EC2 state ! #1074901 + volume.attach(instance.id, "/dev/vdh") + + #self.assertVolumeStatusWait(_volume_state, "in-use") # #1074901 + re_search_wait(_volume_state, "in-use") + + #NOTE(afazekas): Different Hypervisor backends names + # differently the devices, + # now we just test is the partition number increased/decrised + + def _part_state(): + current = ssh.get_partitions().split('\n') + if current > part_lines: + return 'INCREASE' + if current < part_lines: + return 'DECREASE' + return 'EQUAL' + + state_wait(_part_state, 'INCREASE') + part_lines = ssh.get_partitions().split('\n') + + #TODO(afazekas): Resource compare to the flavor settings + + volume.detach() # "detaching" invalid EC2 status #1074901 + + #self.assertVolumeStatusWait(_volume_state, "available") + re_search_wait(_volume_state, "available") + LOG.info("Volume %s state: %s", volume.id, volume.status) + + state_wait(_part_state, 'DECREASE') + + instance.stop() + address.disassociate() + self.assertAddressDissasociatedWait(address) + self.cancelResourceCleanUp(rcuk_da) + address.release() + self.assertAddressReleasedWait(address) + self.cancelResourceCleanUp(rcuk_a) + + LOG.info("state: %s", instance.state) + if instance.state != "stopped": + self.assertInstanceStateWait(_instance_state, "stopped") + #TODO(afazekas): move steps from teardown to the test case + + +#TODO(afazekas): Snapshot/volume read/write test case diff --git a/tempest/tests/boto/test_ec2_keys.py b/tempest/tests/boto/test_ec2_keys.py new file mode 100644 index 0000000000..79d0b2bf88 --- /dev/null +++ b/tempest/tests/boto/test_ec2_keys.py @@ -0,0 +1,79 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from nose.plugins.attrib import attr +import unittest2 as unittest +from tempest.testboto import BotoTestCase +from tempest.common.utils.data_utils import rand_name +from tempest import openstack + + +def compare_key_pairs(a, b): + return (a.name == b.name and + a.fingerprint == b.fingerprint) + + +@attr("EC2") +class EC2KeysTest(BotoTestCase): + + @classmethod + def setUpClass(cls): + super(EC2KeysTest, cls).setUpClass() + cls.os = openstack.Manager() + cls.client = cls.os.ec2api_client + + @attr(type='smoke') + def test_create_ec2_keypair(self): + """EC2 create KeyPair""" + key_name = rand_name("keypair-") + self.addResourceCleanUp(self.client.delete_key_pair, key_name) + keypair = self.client.create_key_pair(key_name) + self.assertTrue(compare_key_pairs(keypair, + self.client.get_key_pair(key_name))) + + @attr(type='smoke') + @unittest.skip("Skipped until the Bug #1072318 is resolved") + def test_delete_ec2_keypair(self): + """EC2 delete KeyPair""" + key_name = rand_name("keypair-") + self.client.create_key_pair(key_name) + self.client.delete_key_pair(key_name) + self.assertEqual(None, self.client.get_key_pair(key_name)) + + @attr(type='smoke') + def test_get_ec2_keypair(self): + """EC2 get KeyPair""" + key_name = rand_name("keypair-") + self.addResourceCleanUp(self.client.delete_key_pair, key_name) + keypair = self.client.create_key_pair(key_name) + self.assertTrue(compare_key_pairs(keypair, + self.client.get_key_pair(key_name))) + + @attr(type='smoke') + @unittest.skip("Skipped until the Bug #1072762 is resolved") + def test_duplicate_ec2_keypair(self): + """EC2 duplicate KeyPair""" + key_name = rand_name("keypair-") + self.addResourceCleanUp(self.client.delete_key_pair, key_name) + keypair = self.client.create_key_pair(key_name) + self.assertEC2ResponseError(self.error_code.client. + InvalidKeyPair.Duplicate, + self.client.create_key_pair, + key_name) + self.assertTrue(compare_key_pairs(keypair, + self.client.get_key_pair(key_name))) diff --git a/tempest/tests/boto/test_ec2_network.py b/tempest/tests/boto/test_ec2_network.py new file mode 100644 index 0000000000..accf6779fa --- /dev/null +++ b/tempest/tests/boto/test_ec2_network.py @@ -0,0 +1,49 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nose.plugins.attrib import attr +import unittest2 as unittest +from tempest.testboto import BotoTestCase +from tempest import openstack + + +@attr("EC2") +class EC2NetworkTest(BotoTestCase): + + @classmethod + def setUpClass(cls): + super(EC2NetworkTest, cls).setUpClass() + cls.os = openstack.Manager() + cls.client = cls.os.ec2api_client + +#Note(afazekas): these tests for things duable without an instance + @unittest.skip("Skipped until the Bug #1080406 is resolved") + @attr(type='smoke') + def test_disassociate_not_associated_floating_ip(self): + """EC2 disassociate not associated floating ip""" + ec2_codes = self.ec2_error_code + address = self.client.allocate_address() + public_ip = address.public_ip + rcuk = self.addResourceCleanUp(self.client.release_address, public_ip) + addresses_get = self.client.get_all_addresses(addresses=(public_ip,)) + self.assertEqual(len(addresses_get), 1) + self.assertEqual(addresses_get[0].public_ip, public_ip) + self.assertBotoError(ec2_codes.client.InvalidAssociationID.NotFound, + address.disassociate) + self.client.release_address(public_ip) + self.cancelResourceCleanUp(rcuk) + assertAddressReleasedWait(address) diff --git a/tempest/tests/boto/test_ec2_security_groups.py b/tempest/tests/boto/test_ec2_security_groups.py new file mode 100644 index 0000000000..4e978e19df --- /dev/null +++ b/tempest/tests/boto/test_ec2_security_groups.py @@ -0,0 +1,78 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nose.plugins.attrib import attr +import unittest2 as unittest +from tempest.testboto import BotoTestCase +from tempest.common.utils.data_utils import rand_name +from tempest import openstack + + +@attr("EC2") +class EC2SecurityGroupTest(BotoTestCase): + + @classmethod + def setUpClass(cls): + super(EC2SecurityGroupTest, cls).setUpClass() + cls.os = openstack.Manager() + cls.client = cls.os.ec2api_client + + @attr(type='smoke') + def test_create_authorize_security_group(self): + """EC2 Create, authorize/revoke security group""" + group_name = rand_name("securty_group-") + group_description = group_name + " security group description " + group = self.client.create_security_group(group_name, + group_description) + self.addResourceCleanUp(self.client.delete_security_group, group_name) + groups_get = self.client.get_all_security_groups(groupnames= + (group_name,)) + self.assertEqual(len(groups_get), 1) + group_get = groups_get[0] + self.assertEqual(group.name, group_get.name) + self.assertEqual(group.name, group_get.name) + #ping (icmp_echo) and other icmp allowed from everywhere + # from_port and to_port act as icmp type + success = self.client.authorize_security_group(group_name, + ip_protocol="icmp", + cidr_ip="0.0.0.0/0", + from_port=-1, + to_port=-1) + self.assertTrue(success) + #allow standard ssh port from anywhere + success = self.client.authorize_security_group(group_name, + ip_protocol="tcp", + cidr_ip="0.0.0.0/0", + from_port=22, + to_port=22) + self.assertTrue(success) + #TODO(afazekas): Duplicate tests + group_get = self.client.get_all_security_groups(groupnames= + (group_name,))[0] + #remove listed rules + for ip_permission in group_get.rules: + for cidr in ip_permission.grants: + self.assertTrue(self.client.revoke_security_group(group_name, + ip_protocol=ip_permission.ip_protocol, + cidr_ip=cidr, + from_port=ip_permission.from_port, + to_port=ip_permission.to_port)) + + group_get = self.client.get_all_security_groups(groupnames= + (group_name,))[0] + #all rules shuld be removed now + self.assertEqual(0, len(group_get.rules)) diff --git a/tempest/tests/boto/test_ec2_volumes.py b/tempest/tests/boto/test_ec2_volumes.py new file mode 100644 index 0000000000..8b7e6be6eb --- /dev/null +++ b/tempest/tests/boto/test_ec2_volumes.py @@ -0,0 +1,93 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from nose.plugins.attrib import attr +from tempest.testboto import BotoTestCase +from tempest import openstack +import unittest2 as unittest +import logging +import time + +LOG = logging.getLogger(__name__) + + +def compare_volumes(a, b): + return (a.id == b.id and + a.size == b.size) + + +@attr("EC2") +class EC2VolumesTest(BotoTestCase): + + @classmethod + def setUpClass(cls): + super(EC2VolumesTest, cls).setUpClass() + cls.os = openstack.Manager() + cls.client = cls.os.ec2api_client + cls.zone = cls.client.get_good_zone() + +#NOTE(afazekas): as admin it can trigger the Bug #1074901 + @attr(type='smoke') + def test_create_get_delete(self): + """EC2 Create, get, delete Volume""" + volume = self.client.create_volume(1, self.zone) + cuk = self.addResourceCleanUp(self.client.delete_volume, volume.id) + self.assertIn(volume.status, self.valid_volume_status) + retrieved = self.client.get_all_volumes((volume.id,)) + self.assertEqual(1, len(retrieved)) + self.assertTrue(compare_volumes(volume, retrieved[0])) + + def _status(): + volume.update(validate=True) + return volume.status + + self.assertVolumeStatusWait(_status, "available") + self.client.delete_volume(volume.id) + self.cancelResourceCleanUp(cuk) + + @unittest.skip("Skipped until the Bug #1080284 is resolved") + def test_create_volme_from_snapshot(self): + """EC2 Create volume from snapshot""" + volume = self.client.create_volume(1, self.zone) + self.addResourceCleanUp(self.client.delete_volume, volume.id) + + def _status(): + volume.update(validate=True) + return volume.status + + self.assertVolumeStatusWait(_status, "available") + snap = self.client.create_snapshot(volume.id) + self.addResourceCleanUp(self.destroy_snapshot_wait, snap) + + def _snap_status(): + snap.update(validate=True) + return snap.status + + #self.assertVolumeStatusWait(_snap_status, "available") # not a volume + self.assertSnapshotStatusWait(_snap_status, "completed") + + svol = self.client.create_volume(1, self.zone, snapshot=snap) + cuk = self.addResourceCleanUp(svol.delete) + + def _snap_vol_status(): + svol.update(validate=True) + return svol.status + + self.assertVolumeStatusWait(_snap_vol_status, "available") + svol.delete() + self.cancelResourceCleanUp(cuk) diff --git a/tempest/tests/boto/test_s3_buckets.py b/tempest/tests/boto/test_s3_buckets.py new file mode 100644 index 0000000000..56cf52cb9b --- /dev/null +++ b/tempest/tests/boto/test_s3_buckets.py @@ -0,0 +1,49 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nose.plugins.attrib import attr +import unittest2 as unittest +from tempest.testboto import BotoTestCase +from tempest.common.utils.data_utils import rand_name +from tempest import openstack + + +@attr("S3") +class S3BucketsTest(BotoTestCase): + + @classmethod + def setUpClass(cls): + super(S3BucketsTest, cls).setUpClass() + cls.os = openstack.Manager() + cls.client = cls.os.s3_client + cls.config = cls.os.config + + @unittest.skip("Skipped until the Bug #1076965 is resolved") + @attr(type='smoke') + def test_create_and_get_delete_bucket(self): + """S3 Create, get and delete bucket""" + bucket_name = rand_name("s3bucket-") + cleanup_key = self.addResourceCleanUp(self.client.delete_bucket, + bucket_name) + bucket = self.client.create_bucket(bucket_name) + self.assertTrue(bucket.name == bucket_name) + bucket = self.client.get_bucket(bucket_name) + self.assertTrue(bucket.name == bucket_name) + self.client.delete_bucket(bucket_name) + self.assertBotoError(self.s3_error_code.client.NoSuchBucket, + self.client.get_bucket, bucket_name) + self.cancelResourceCleanUp(cleanup_key) diff --git a/tempest/tests/boto/test_s3_ec2_images.py b/tempest/tests/boto/test_s3_ec2_images.py new file mode 100644 index 0000000000..eeb7039763 --- /dev/null +++ b/tempest/tests/boto/test_s3_ec2_images.py @@ -0,0 +1,144 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nose.plugins.attrib import attr +import unittest2 as unittest +from tempest import openstack +from tempest.testboto import BotoTestCase +import tempest.tests.boto +from tempest.tests.boto.utils.wait import state_wait +from tempest.tests.boto.utils.s3 import s3_upload_dir +from tempest.common.utils.data_utils import rand_name +from contextlib import closing +from boto.s3.key import Key +import logging +import nose +import os + + +@attr("S3", "EC2") +class S3ImagesTest(BotoTestCase): + + @classmethod + def setUpClass(cls): + super(S3ImagesTest, cls).setUpClass() + if not tempest.tests.boto.A_I_IMAGES_READY: + raise nose.SkipTest("".join(("EC2 ", cls.__name__, + ": requires ami/aki/ari manifest"))) + cls.os = openstack.Manager() + cls.s3_client = cls.os.s3_client + cls.images_client = cls.os.ec2api_client + config = cls.os.config + cls.materials_path = config.boto.s3_materials_path + cls.ami_manifest = config.boto.ami_manifest + cls.aki_manifest = config.boto.aki_manifest + cls.ari_manifest = config.boto.ari_manifest + cls.ami_path = cls.materials_path + os.sep + cls.ami_manifest + cls.aki_path = cls.materials_path + os.sep + cls.aki_manifest + cls.ari_path = cls.materials_path + os.sep + cls.ari_manifest + cls.bucket_name = rand_name("bucket-") + bucket = cls.s3_client.create_bucket(cls.bucket_name) + cls.addResourceCleanUp(cls.destroy_bucket, + cls.s3_client.connection_data, + cls.bucket_name) + s3_upload_dir(bucket, cls.materials_path) + + #Note(afazekas): Without the normal status change test! + # otherwise I would skip it too + @attr(type='smoke') + def test_register_get_deregister_ami_image(self): + """Register and deregister ami image""" + image = {"name": rand_name("ami-name-"), + "location": self.bucket_name + "/" + self.ami_manifest, + "type": "ami"} + image["image_id"] = self.images_client.register_image( + name=image["name"], + image_location=image["location"]) + #Note(afazekas): delete_snapshot=True might trigger boto lib? bug + image["cleanUp"] = self.addResourceCleanUp( + self.images_client.deregister_image, + image["image_id"]) + self.assertEqual(image["image_id"][0:3], image["type"]) + retrieved_image = self.images_client.get_image(image["image_id"]) + self.assertTrue(retrieved_image.name == image["name"]) + self.assertTrue(retrieved_image.id == image["image_id"]) + state = retrieved_image.state + if state != "available": + def _state(): + retr = self.images_client.get_image(image["image_id"]) + return retr.state + state = state_wait(_state, "available") + self.assertEqual("available", state) + self.images_client.deregister_image(image["image_id"]) + #TODO(afazekas): double deregister ? + self.cancelResourceCleanUp(image["cleanUp"]) + + @unittest.skip("Skipped until the Bug #1074904 is resolved") + def test_register_get_deregister_aki_image(self): + """Register and deregister aki image""" + image = {"name": rand_name("aki-name-"), + "location": self.bucket_name + "/" + self.ari_manifest, + "type": "aki"} + image["image_id"] = self.images_client.register_image( + name=image["name"], + image_location=image["location"]) + image["cleanUp"] = self.addResourceCleanUp( + self.images_client.deregister_image, + image["image_id"]) + self.assertEqual(image["image_id"][0:3], image["type"]) + retrieved_image = self.images_client.get_image(image["image_id"]) + self.assertTrue(retrieved_image.name == image["name"]) + self.assertTrue(retrieved_image.id == image["image_id"]) + self.assertIn(retrieved_image.state, self.valid_image_state) + if retrieved_image.state != "available": + def _state(): + retr = self.images_client.get_image(image["image_id"]) + return retr.state + self.assertImageStateWait(_state, "available") + self.images_client.deregister_image(image["image_id"]) + #TODO(afazekas): verify deregister in a better way + retrieved_image = self.images_client.get_image(image["image_id"]) + self.assertIn(retrieved_image.state, self.valid_image_state) + self.cancelResourceCleanUp(image["cleanUp"]) + + @unittest.skip("Skipped until the Bug #1074908 and #1074904 is resolved") + def test_register_get_deregister_ari_image(self): + """Register and deregister ari image""" + image = {"name": rand_name("ari-name-"), + "location": "/" + self.bucket_name + "/" + self.ari_manifest, + "type": "ari"} + image["image_id"] = self.images_client.register_image( + name=image["name"], + image_location=image["location"]) + image["cleanUp"] = self.addResourceCleanUp( + self.images_client.deregister_image, + image["image_id"]) + self.assertEqual(image["image_id"][0:3], image["type"]) + retrieved_image = self.images_client.get_image(image["image_id"]) + self.assertIn(retrieved_image.state, self.valid_image_state) + if retrieved_image.state != "available": + def _state(): + retr = self.images_client.get_image(image["image_id"]) + return retr.state + self.assertImageStateWait(_state, "available") + self.assertIn(retrieved_image.state, self.valid_image_state) + self.assertTrue(retrieved_image.name == image["name"]) + self.assertTrue(retrieved_image.id == image["image_id"]) + self.images_client.deregister_image(image["image_id"]) + self.cancelResourceCleanUp(image["cleanUp"]) + +#TODO(afazekas): less copy-paste style diff --git a/tempest/tests/boto/test_s3_objects.py b/tempest/tests/boto/test_s3_objects.py new file mode 100644 index 0000000000..c31ad6e372 --- /dev/null +++ b/tempest/tests/boto/test_s3_objects.py @@ -0,0 +1,58 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nose.plugins.attrib import attr +import unittest2 as unittest +from tempest.testboto import BotoTestCase +from tempest.common.utils.data_utils import rand_name +from tempest import openstack +from tempest.tests import boto +from boto.s3.key import Key +from contextlib import closing + + +@attr("S3") +class S3BucketsTest(BotoTestCase): + + @classmethod + def setUpClass(cls): + super(S3BucketsTest, cls).setUpClass() + cls.os = openstack.Manager() + cls.client = cls.os.s3_client + cls.config = cls.os.config + + @unittest.skip("Skipped until the Bug #1076534 is resolved") + @attr(type='smoke') + def test_create_get_delete_object(self): + """S3 Create, get and delete object""" + bucket_name = rand_name("s3bucket-") + object_name = rand_name("s3object-") + content = 'x' * 42 + bucket = self.client.create_bucket(bucket_name) + self.addResourceCleanUp(self.destroy_bucket, + self.client.connection_data, + bucket_name) + + self.assertTrue(bucket.name == bucket_name) + with closing(Key(bucket)) as key: + key.key = object_name + key.set_contents_from_string(content) + readback = key.get_contents_as_string() + self.assertTrue(readback == content) + bucket.delete_key(key) + self.assertBotoError(self.s3_error_code.client.NoSuchKey, + key.get_contents_as_string) diff --git a/tempest/tests/boto/utils/__init__.py b/tempest/tests/boto/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tempest/tests/boto/utils/s3.py b/tempest/tests/boto/utils/s3.py new file mode 100644 index 0000000000..70d9263213 --- /dev/null +++ b/tempest/tests/boto/utils/s3.py @@ -0,0 +1,42 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import boto +from boto.s3.key import Key +from contextlib import closing +import os +import re +import logging + + +LOG = logging.getLogger(__name__) + + +def s3_upload_dir(bucket, path, prefix="", connection_data=None): + if isinstance(bucket, basestring): + with closing(boto.connect_s3(**connection_data)) as conn: + bucket = conn.lookup(bucket) + for root, dirs, files in os.walk(path): + for fil in files: + with closing(Key(bucket)) as key: + source = root + os.sep + fil + target = re.sub("^" + re.escape(path) + "?/", prefix, source) + if os.sep != '/': + target = re.sub(re.escape(os.sep), '/', target) + key.key = target + LOG.info("Uploading %s to %s/%s", source, bucket.name, target) + key.set_contents_from_filename(source) diff --git a/tempest/tests/boto/utils/wait.py b/tempest/tests/boto/utils/wait.py new file mode 100644 index 0000000000..38b6ba1d46 --- /dev/null +++ b/tempest/tests/boto/utils/wait.py @@ -0,0 +1,130 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import tempest.config +import time +from unittest2 import TestCase +import logging +import re +from boto.exception import BotoServerError + +LOG = logging.getLogger(__name__) + +_boto_config = tempest.config.TempestConfig().boto + +default_timeout = _boto_config.build_timeout + +default_check_interval = _boto_config.build_interval + + +def state_wait(lfunction, final_set=set(), valid_set=None): + #TODO(afazekas): evaluate using ABC here + if not isinstance(final_set, set): + final_set = set((final_set,)) + if not isinstance(valid_set, set) and valid_set is not None: + valid_set = set((valid_set,)) + start_time = time.time() + old_status = status = lfunction() + while True: + if status != old_status: + LOG.info('State transition "%s" ==> "%s" %d second', old_status, + status, time.time() - start_time) + if status in final_set: + return status + if valid_set is not None and status not in valid_set: + return status + dtime = time.time() - start_time + if dtime > default_timeout: + raise TestCase.failureException("State change timeout exceeded!" + '(%ds) While waiting' + 'for %s at "%s"' % + (dtime, + final_set, status)) + time.sleep(default_check_interval) + old_status = status + status = lfunction() + + +def re_search_wait(lfunction, regexp): + """Stops waiting on success""" + start_time = time.time() + while True: + text = lfunction() + result = re.search(regexp, text) + if result is not None: + LOG.info('Pattern "%s" found in %d second in "%s"', + regexp, + time.time() - start_time, + text) + return result + dtime = time.time() - start_time + if dtime > default_timeout: + raise TestCase.failureException('Pattern find timeout exceeded!' + '(%ds) While waiting for' + '"%s" pattern in "%s"' % + (dtime, + regexp, text)) + time.sleep(default_check_interval) + + +def wait_no_exception(lfunction, exc_class=None, exc_matcher=None): + """Stops waiting on success""" + start_time = time.time() + if exc_matcher is not None: + exc_class = BotoServerError + + if exc_class is None: + exc_class = BaseException + while True: + result = None + try: + result = lfunction() + LOG.info('No Exception in %d second', + time.time() - start_time) + return result + except exc_class as exc: + if exc_matcher is not None: + res = exc_matcher.match(exc) + if res is not None: + LOG.info(res) + raise exc + # Let the other exceptions propagate + dtime = time.time() - start_time + if dtime > default_timeout: + raise TestCase.failureException("Wait timeout exceeded! (%ds)" % + dtime) + time.sleep(default_check_interval) + + +#NOTE(afazekas): EC2/boto normally raise exception instead of empty list +def wait_exception(lfunction): + """Returns with the exception or raises one""" + start_time = time.time() + while True: + try: + lfunction() + except BaseException as exc: + LOG.info('Exception in %d second', + time.time() - start_time) + return exc + dtime = time.time() - start_time + if dtime > default_timeout: + raise TestCase.failureException("Wait timeout exceeded! (%ds)" % + dtime) + time.sleep(default_check_interval) + +#TODO(afazekas): consider strategy design pattern.. diff --git a/tools/pip-requires b/tools/pip-requires index d3f9db750c..9c861d93b9 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -4,3 +4,4 @@ httplib2>=0.7.0 pika unittest2 lxml +boto>=2.2.1