diff --git a/glance_store/_drivers/http.py b/glance_store/_drivers/http.py index f256cdbe..e74746ac 100644 --- a/glance_store/_drivers/http.py +++ b/glance_store/_drivers/http.py @@ -287,7 +287,8 @@ class Store(glance_store.driver.Store): self.conf, uri=url, image_id=image_id, - store_specs=store_specs) + store_specs=store_specs, + backend=self.backend_group) @staticmethod def _check_store_uri(conn, loc): @@ -317,9 +318,15 @@ class Store(glance_store.driver.Store): def _get_response(self, location, verb): if not hasattr(self, 'session'): self.session = requests.Session() - ca_bundle = self.conf.glance_store.https_ca_certificates_file - disable_https = self.conf.glance_store.https_insecure + + if self.backend_group: + store_conf = getattr(self.conf, self.backend_group) + else: + store_conf = self.conf.glance_store + + ca_bundle = store_conf.https_ca_certificates_file + disable_https = store_conf.https_insecure self.session.verify = ca_bundle if ca_bundle else not disable_https - self.session.proxies = self.conf.glance_store.http_proxy_information + self.session.proxies = store_conf.http_proxy_information return self.session.request(verb, location.get_uri(), stream=True, allow_redirects=False) diff --git a/glance_store/_drivers/sheepdog.py b/glance_store/_drivers/sheepdog.py index da2cea62..3ca92a6d 100644 --- a/glance_store/_drivers/sheepdog.py +++ b/glance_store/_drivers/sheepdog.py @@ -230,9 +230,14 @@ class StoreLocation(glance_store.location.StoreLocation): self.addr = pieces[0] # This is used for backwards compatibility. else: + if self.backend_group: + store_conf = getattr(self.conf, self.backend_group) + else: + store_conf = self.conf.glance_store + self.image = pieces[0] - self.port = self.conf.glance_store.sheepdog_store_port - self.addr = self.conf.glance_store.sheepdog_store_address + self.port = store_conf.sheepdog_store_port + self.addr = store_conf.sheepdog_store_address class ImageIterator(object): @@ -272,15 +277,19 @@ class Store(glance_store.driver.Store): this method. If the store was not able to successfully configure itself, it should raise `exceptions.BadStoreConfiguration` """ + if self.backend_group: + store_conf = getattr(self.conf, self.backend_group) + else: + store_conf = self.conf.glance_store try: - chunk_size = self.conf.glance_store.sheepdog_store_chunk_size + chunk_size = store_conf.sheepdog_store_chunk_size self.chunk_size = chunk_size * units.Mi self.READ_CHUNKSIZE = self.chunk_size self.WRITE_CHUNKSIZE = self.READ_CHUNKSIZE - self.addr = self.conf.glance_store.sheepdog_store_address - self.port = self.conf.glance_store.sheepdog_store_port + self.addr = store_conf.sheepdog_store_address + self.port = store_conf.sheepdog_store_port except cfg.ConfigFileValueError as e: reason = _("Error in store configuration: %s") % e LOG.error(reason) @@ -362,7 +371,7 @@ class Store(glance_store.driver.Store): 'image': image_id, 'addr': self.addr, 'port': self.port - }, self.conf) + }, self.conf, backend_group=self.backend_group) image.create(image_size) @@ -389,7 +398,11 @@ class Store(glance_store.driver.Store): with excutils.save_and_reraise_exception(): image.delete() - return (location.get_uri(), offset, checksum.hexdigest(), {}) + metadata = {} + if self.backend_group: + metadata['backend'] = u"%s" % self.backend_group + + return (location.get_uri(), offset, checksum.hexdigest(), metadata) @capabilities.check def delete(self, location, context=None): diff --git a/glance_store/_drivers/swift/buffered.py b/glance_store/_drivers/swift/buffered.py index 02c0594e..8dec9fe0 100644 --- a/glance_store/_drivers/swift/buffered.py +++ b/glance_store/_drivers/swift/buffered.py @@ -90,15 +90,21 @@ class BufferedReader(object): to ensure there is enough disk space available. """ - def __init__(self, fd, checksum, total, verifier=None): + def __init__(self, fd, checksum, total, verifier=None, backend_group=None): self.fd = fd self.total = total self.checksum = checksum self.verifier = verifier + self.backend_group = backend_group # maintain a pointer to use to update checksum and verifier self.update_position = 0 - buffer_dir = CONF.glance_store.swift_upload_buffer_dir + if self.backend_group: + buffer_dir = getattr(CONF, + self.backend_group).swift_upload_buffer_dir + else: + buffer_dir = CONF.glance_store.swift_upload_buffer_dir + self._tmpfile = tempfile.TemporaryFile(dir=buffer_dir) self._buffered = False diff --git a/glance_store/_drivers/swift/connection_manager.py b/glance_store/_drivers/swift/connection_manager.py index 883f8d56..56fa2957 100644 --- a/glance_store/_drivers/swift/connection_manager.py +++ b/glance_store/_drivers/swift/connection_manager.py @@ -83,9 +83,15 @@ class SwiftConnectionManager(object): auth_ref = self.client.session.auth.auth_ref # if connection token is going to expire soon (keystone checks # is token is going to expire or expired already) - if auth_ref.will_expire_soon( - self.store.conf.glance_store.swift_store_expire_soon_interval - ): + if self.store.backend_group: + interval = getattr( + self.store.conf, self.store.backend_group + ).swift_store_expire_soon_interval + else: + store_conf = self.store.conf.glance_store + interval = store_conf.swift_store_expire_soon_interval + + if auth_ref.will_expire_soon(interval): LOG.info(_LI("Requesting new token for swift connection.")) # request new token with session and client provided by store auth_token = self.client.session.get_auth_headers().get( diff --git a/glance_store/_drivers/swift/store.py b/glance_store/_drivers/swift/store.py index e5839067..b4c11c9b 100644 --- a/glance_store/_drivers/swift/store.py +++ b/glance_store/_drivers/swift/store.py @@ -495,7 +495,13 @@ def swift_retry_iter(resp_iter, length, store, location, manager): retries = 0 bytes_read = 0 - while retries <= store.conf.glance_store.swift_store_retry_get_count: + if store.backend_group: + rcount = getattr(store.conf, + store.backend_group).swift_store_retry_get_count + else: + rcount = store.conf.glance_store.swift_store_retry_get_count + + while retries <= rcount: try: for chunk in resp_iter: yield chunk @@ -506,20 +512,18 @@ def swift_retry_iter(resp_iter, length, store, location, manager): % encodeutils.exception_to_unicode(e)) if bytes_read != length: - if retries == store.conf.glance_store.swift_store_retry_get_count: + if retries == rcount: # terminate silently and let higher level decide LOG.error(_LE("Stopping Swift retries after %d " "attempts") % retries) break else: retries += 1 - glance_conf = store.conf.glance_store - retry_count = glance_conf.swift_store_retry_get_count LOG.info(_LI("Retrying Swift connection " "(%(retries)d/%(max_retries)d) with " "range=%(start)d-%(end)d"), {'retries': retries, - 'max_retries': retry_count, + 'max_retries': rcount, 'start': bytes_read, 'end': length}) (_resp_headers, resp_iter) = store._get_object(location, @@ -578,7 +582,11 @@ class StoreLocation(location.StoreLocation): if not credentials_included: # Used only in case of an add # Get the current store from config - store = self.conf.glance_store.default_swift_reference + if self.backend_group: + store = getattr(self.conf, + self.backend_group).default_swift_reference + else: + store = self.conf.glance_store.default_swift_reference return '%s://%s/%s/%s' % ('swift+config', store, container, obj) if self.scheme == 'swift+config': @@ -593,7 +601,8 @@ class StoreLocation(location.StoreLocation): def _get_conf_value_from_account_ref(self, netloc): try: - ref_params = sutils.SwiftParams(self.conf).params + ref_params = sutils.SwiftParams( + self.conf, backend=self.backend_group).params self.user = ref_params[netloc]['user'] self.key = ref_params[netloc]['key'] netloc = ref_params[netloc]['auth_address'] @@ -726,11 +735,21 @@ class StoreLocation(location.StoreLocation): return ''.join([auth_scheme, self.auth_or_store_url]) -def Store(conf): +def Store(conf, backend=None): + group = 'glance_store' + if backend: + group = backend + multi_tenant = getattr(conf, backend).swift_store_multi_tenant + default_store = conf.glance_store.default_backend + else: + default_store = conf.glance_store.default_store + multi_tenant = conf.glance_store.swift_store_multi_tenant + # NOTE(dharinic): Multi-tenant store cannot work with swift config - if conf.glance_store.swift_store_multi_tenant: - if (conf.glance_store.default_store == 'swift+config' or - sutils.is_multiple_swift_store_accounts_enabled(conf)): + if multi_tenant: + if (default_store == 'swift+config' or + sutils.is_multiple_swift_store_accounts_enabled( + conf, backend=backend)): msg = _("Swift multi-tenant store cannot be configured to " "work with swift+config. The options " "'swift_store_multi_tenant' and " @@ -742,13 +761,13 @@ def Store(conf): reason=msg) try: conf.register_opts(_SWIFT_OPTS + sutils.swift_opts + - buffered.BUFFERING_OPTS, group='glance_store') + buffered.BUFFERING_OPTS, group=group) except cfg.DuplicateOptError: pass - if conf.glance_store.swift_store_multi_tenant: - return MultiTenantStore(conf) - return SingleTenantStore(conf) + if multi_tenant: + return MultiTenantStore(conf, backend=backend) + return SingleTenantStore(conf, backend=backend) Store.OPTIONS = _SWIFT_OPTS + sutils.swift_opts + buffered.BUFFERING_OPTS @@ -771,7 +790,11 @@ class BaseStore(driver.Store): return ('swift+https', 'swift', 'swift+http', 'swift+config') def configure(self, re_raise_bsc=False): - glance_conf = self.conf.glance_store + if self.backend_group: + glance_conf = getattr(self.conf, self.backend_group) + else: + glance_conf = self.conf.glance_store + _obj_size = self._option_get('swift_store_large_object_size') self.large_object_size = _obj_size * ONE_MB _chunk_size = self._option_get('swift_store_large_object_chunk_size') @@ -821,10 +844,14 @@ class BaseStore(driver.Store): @capabilities.check def get(self, location, connection=None, offset=0, chunk_size=None, context=None): + if self.backend_group: + glance_conf = getattr(self.conf, self.backend_group) + else: + glance_conf = self.conf.glance_store + location = location.store_location # initialize manager to receive valid connections - allow_retry = \ - self.conf.glance_store.swift_store_retry_get_count > 0 + allow_retry = glance_conf.swift_store_retry_get_count > 0 with self.get_manager(location, context, allow_reauth=allow_retry) as manager: (resp_headers, resp_body) = self._get_object(location, @@ -855,7 +882,11 @@ class BaseStore(driver.Store): return 0 def _option_get(self, param): - result = getattr(self.conf.glance_store, param) + if self.backend_group: + result = getattr(getattr(self.conf, self.backend_group), param) + else: + result = getattr(self.conf.glance_store, param) + if not result: reason = (_("Could not find %(param)s in configuration options.") % param) @@ -940,8 +971,9 @@ class BaseStore(driver.Store): chunk_name = "%s-%05d" % (location.obj, chunk_id) - with self.reader_class(image_file, checksum, - chunk_size, verifier) as reader: + with self.reader_class( + image_file, checksum, chunk_size, verifier, + backend_group=self.backend_group) as reader: if reader.is_zero_size is True: LOG.debug('Not writing zero-length chunk.') break @@ -1004,12 +1036,18 @@ class BaseStore(driver.Store): # image data. We *really* should consider NOT returning # the location attribute from GET /images/ and # GET /images/details - if sutils.is_multiple_swift_store_accounts_enabled(self.conf): + if sutils.is_multiple_swift_store_accounts_enabled( + self.conf, backend=self.backend_group): include_creds = False else: include_creds = True + + metadata = {} + if self.backend_group: + metadata['backend'] = u"%s" % self.backend_group + return (location.get_uri(credentials_included=include_creds), - image_size, obj_etag, {}) + image_size, obj_etag, metadata) except swiftclient.ClientException as e: if e.http_status == http_client.CONFLICT: msg = _("Swift already has an image at this location") @@ -1086,11 +1124,15 @@ class BaseStore(driver.Store): :param container: Name of container to create :param connection: Connection to swift service """ + if self.backend_group: + store_conf = getattr(self.conf, self.backend_group) + else: + store_conf = self.conf.glance_store try: connection.head_container(container) except swiftclient.ClientException as e: if e.http_status == http_client.NOT_FOUND: - if self.conf.glance_store.swift_store_create_container_on_put: + if store_conf.swift_store_create_container_on_put: try: msg = (_LI("Creating swift container %(container)s") % {'container': container}) @@ -1167,9 +1209,11 @@ class BaseStore(driver.Store): class SingleTenantStore(BaseStore): EXAMPLE_URL = "swift://:@//" - def __init__(self, conf): - super(SingleTenantStore, self).__init__(conf) - self.ref_params = sutils.SwiftParams(self.conf).params + def __init__(self, conf, backend=None): + super(SingleTenantStore, self).__init__(conf, backend=backend) + self.backend_group = backend + self.ref_params = sutils.SwiftParams(self.conf, + backend=backend).params def configure(self, re_raise_bsc=False): # set configuration before super so configure_add can override @@ -1182,7 +1226,15 @@ class SingleTenantStore(BaseStore): super(SingleTenantStore, self).configure(re_raise_bsc=re_raise_bsc) def configure_add(self): - default_ref = self.conf.glance_store.default_swift_reference + if self.backend_group: + default_ref = getattr(self.conf, + self.backend_group).default_swift_reference + self.container = getattr(self.conf, + self.backend_group).swift_store_container + else: + default_ref = self.conf.glance_store.default_swift_reference + self.container = self.conf.glance_store.swift_store_container + default_swift_reference = self.ref_params.get(default_ref) if default_swift_reference: self.auth_address = default_swift_reference.get('auth_address') @@ -1195,7 +1247,7 @@ class SingleTenantStore(BaseStore): self.scheme = 'swift+http' else: self.scheme = 'swift+https' - self.container = self.conf.glance_store.swift_store_container + self.auth_version = default_swift_reference.get('auth_version') self.user = default_swift_reference.get('user') self.key = default_swift_reference.get('key') @@ -1220,7 +1272,8 @@ class SingleTenantStore(BaseStore): 'auth_or_store_url': self.auth_address, 'user': self.user, 'key': self.key} - return StoreLocation(specs, self.conf) + return StoreLocation(specs, self.conf, + backend_group=self.backend_group) def get_container_name(self, image_id, default_image_container): """ @@ -1238,8 +1291,14 @@ class SingleTenantStore(BaseStore): :param default_image_container: container name from ``swift_store_container`` """ - seed_num_chars = \ - self.conf.glance_store.swift_store_multiple_containers_seed + if self.backend_group: + seed_num_chars = getattr( + self.conf, + self.backend_group).swift_store_multiple_containers_seed + else: + seed_num_chars = \ + self.conf.glance_store.swift_store_multiple_containers_seed + if seed_num_chars is None \ or seed_num_chars < 0 or seed_num_chars > 32: reason = _("An integer value between 0 and 32 is required for" @@ -1345,7 +1404,12 @@ class MultiTenantStore(BaseStore): EXAMPLE_URL = "swift:////" def _get_endpoint(self, context): - self.container = self.conf.glance_store.swift_store_container + if self.backend_group: + self.container = getattr(self.conf, + self.backend_group).swift_store_container + else: + self.container = self.conf.glance_store.swift_store_container + if context is None: reason = _("Multi-tenant Swift storage requires a context.") raise exceptions.BadStoreConfiguration(store_name="swift", @@ -1418,7 +1482,8 @@ class MultiTenantStore(BaseStore): 'container': self.container + '_' + str(image_id), 'obj': str(image_id), 'auth_or_store_url': ep} - return StoreLocation(specs, self.conf) + return StoreLocation(specs, self.conf, + backend_group=self.backend_group) def get_connection(self, location, context=None): return swiftclient.Connection( @@ -1430,8 +1495,14 @@ class MultiTenantStore(BaseStore): def init_client(self, location, context=None): # read client parameters from config files - ref_params = sutils.SwiftParams(self.conf).params - default_ref = self.conf.glance_store.default_swift_reference + ref_params = sutils.SwiftParams(self.conf, + backend=self.backend_group).params + if self.backend_group: + default_ref = getattr(self.conf, + self.backend_group).default_swift_reference + else: + default_ref = self.conf.glance_store.default_swift_reference + default_swift_reference = ref_params.get(default_ref) if not default_swift_reference: reason = _("default_swift_reference %s is " @@ -1503,7 +1574,13 @@ class MultiTenantStore(BaseStore): def get_manager(self, store_location, context=None, allow_reauth=False): # if global toggle is turned off then do not allow re-authentication # with trusts - if not self.conf.glance_store.swift_store_use_trusts: + if self.backend_group: + use_trusts = getattr(self.conf, + self.backend_group).swift_store_use_trusts + else: + use_trusts = self.conf.glance_store.swift_store_use_trusts + + if not use_trusts: allow_reauth = False return connection_manager.MultiTenantConnectionManager(self, @@ -1513,11 +1590,13 @@ class MultiTenantStore(BaseStore): class ChunkReader(object): - def __init__(self, fd, checksum, total, verifier=None): + def __init__(self, fd, checksum, total, verifier=None, + backend_group=None): self.fd = fd self.checksum = checksum self.total = total self.verifier = verifier + self.backend_group = backend_group self.bytes_read = 0 self.is_zero_size = False self.byteone = fd.read(1) diff --git a/glance_store/_drivers/swift/utils.py b/glance_store/_drivers/swift/utils.py index a73aaaaa..e499cade 100644 --- a/glance_store/_drivers/swift/utils.py +++ b/glance_store/_drivers/swift/utils.py @@ -111,30 +111,39 @@ else: LOG = logging.getLogger(__name__) -def is_multiple_swift_store_accounts_enabled(conf): - if conf.glance_store.swift_store_config_file is None: +def is_multiple_swift_store_accounts_enabled(conf, backend=None): + if backend: + cfg_file = getattr(conf, backend).swift_store_config_file + else: + cfg_file = conf.glance_store.swift_store_config_file + + if cfg_file is None: return False return True class SwiftParams(object): - def __init__(self, conf): + def __init__(self, conf, backend=None): self.conf = conf - if is_multiple_swift_store_accounts_enabled(self.conf): + self.backend_group = backend + if is_multiple_swift_store_accounts_enabled( + self.conf, backend=backend): self.params = self._load_config() else: self.params = self._form_default_params() def _form_default_params(self): default = {} - + if self.backend_group: + glance_store = getattr(self.conf, self.backend_group) + else: + glance_store = self.conf.glance_store if ( - self.conf.glance_store.swift_store_user and - self.conf.glance_store.swift_store_key and - self.conf.glance_store.swift_store_auth_address + glance_store.swift_store_user and + glance_store.swift_store_key and + glance_store.swift_store_auth_address ): - glance_store = self.conf.glance_store default['user'] = glance_store.swift_store_user default['key'] = glance_store.swift_store_key default['auth_address'] = glance_store.swift_store_auth_address @@ -147,14 +156,18 @@ class SwiftParams(object): return {} def _load_config(self): - try: + if self.backend_group: + scf = getattr(self.conf, + self.backend_group).swift_store_config_file + else: scf = self.conf.glance_store.swift_store_config_file + try: conf_file = self.conf.find_file(scf) CONFIG.read(conf_file) except Exception as e: msg = (_("swift config file " "%(conf)s:%(exc)s not found"), - {'conf': self.conf.glance_store.swift_store_config_file, + {'conf': scf, 'exc': e}) LOG.error(msg) raise exceptions.BadStoreConfiguration(store_name='swift', @@ -177,7 +190,12 @@ class SwiftParams(object): try: reference['auth_version'] = CONFIG.get(ref, 'auth_version') except configparser.NoOptionError: - av = self.conf.glance_store.swift_store_auth_version + if self.backend_group: + av = getattr( + self.conf, + self.backend_group).swift_store_auth_version + else: + av = self.conf.glance_store.swift_store_auth_version reference['auth_version'] = av account_params[ref] = reference diff --git a/glance_store/_drivers/vmware_datastore.py b/glance_store/_drivers/vmware_datastore.py index 6ac31241..e8df7d19 100644 --- a/glance_store/_drivers/vmware_datastore.py +++ b/glance_store/_drivers/vmware_datastore.py @@ -284,10 +284,12 @@ class StoreLocation(location.StoreLocation): vsphere://server_host/folder/file_path?dcPath=dc_path&dsName=ds_name """ - def __init__(self, store_specs, conf): - super(StoreLocation, self).__init__(store_specs, conf) + def __init__(self, store_specs, conf, backend_group=None): + super(StoreLocation, self).__init__(store_specs, conf, + backend_group=backend_group) self.datacenter_path = None self.datastore_name = None + self.backend_group = backend_group def process_specs(self): self.scheme = self.specs.get('scheme', STORE_SCHEME) @@ -359,8 +361,8 @@ class Store(glance_store.Store): OPTIONS = _VMWARE_OPTS WRITE_CHUNKSIZE = units.Mi - def __init__(self, conf): - super(Store, self).__init__(conf) + def __init__(self, conf, backend=None): + super(Store, self).__init__(conf, backend=backend) self.datastores = {} def reset_session(self): @@ -375,13 +377,18 @@ class Store(glance_store.Store): return (STORE_SCHEME,) def _sanity_check(self): - if self.conf.glance_store.vmware_api_retry_count <= 0: + if self.backend_group: + store_conf = getattr(self.conf, self.backend_group) + else: + store_conf = self.conf.glance_store + + if store_conf.vmware_api_retry_count <= 0: msg = _('vmware_api_retry_count should be greater than zero') LOG.error(msg) raise exceptions.BadStoreConfiguration( store_name='vmware_datastore', reason=msg) - if self.conf.glance_store.vmware_task_poll_interval <= 0: + if store_conf.vmware_task_poll_interval <= 0: msg = _('vmware_task_poll_interval should be greater than zero') LOG.error(msg) raise exceptions.BadStoreConfiguration( @@ -393,10 +400,16 @@ class Store(glance_store.Store): self.server_host = self._option_get('vmware_server_host') self.server_username = self._option_get('vmware_server_username') self.server_password = self._option_get('vmware_server_password') - self.api_retry_count = self.conf.glance_store.vmware_api_retry_count - self.tpoll_interval = self.conf.glance_store.vmware_task_poll_interval - self.ca_file = self.conf.glance_store.vmware_ca_file - self.api_insecure = self.conf.glance_store.vmware_insecure + + if self.backend_group: + store_conf = getattr(self.conf, self.backend_group) + else: + store_conf = self.conf.glance_store + + self.api_retry_count = store_conf.vmware_api_retry_count + self.tpoll_interval = store_conf.vmware_task_poll_interval + self.ca_file = store_conf.vmware_ca_file + self.api_insecure = store_conf.vmware_insecure if api is None: msg = _("Missing dependencies: oslo_vmware") raise exceptions.BadStoreConfiguration( @@ -492,7 +505,13 @@ class Store(glance_store.Store): def configure_add(self): datastores = self._option_get('vmware_datastores') self.datastores = self._build_datastore_weighted_map(datastores) - self.store_image_dir = self.conf.glance_store.vmware_store_image_dir + + if self.backend_group: + store_conf = getattr(self.conf, self.backend_group) + else: + store_conf = self.conf.glance_store + + self.store_image_dir = store_conf.vmware_store_image_dir def select_datastore(self, image_size): """Select a datastore with free space larger than image size.""" @@ -513,7 +532,12 @@ class Store(glance_store.Store): raise exceptions.StorageFull() def _option_get(self, param): - result = getattr(self.conf.glance_store, param) + if self.backend_group: + store_conf = getattr(self.conf, self.backend_group) + else: + store_conf = self.conf.glance_store + + result = getattr(store_conf, param) if not result: reason = (_("Could not find %(param)s in configuration " "options.") % {'param': param}) @@ -562,7 +586,8 @@ class Store(glance_store.Store): 'image_dir': self.store_image_dir, 'datacenter_path': ds.datacenter.path, 'datastore_name': ds.name, - 'image_id': image_id}, self.conf) + 'image_id': image_id}, self.conf, + backend_group=self.backend_group) # NOTE(arnaud): use a decorator when the config is not tied to self cookie = self._build_vim_cookie_header(True) headers = dict(headers) @@ -609,8 +634,12 @@ class Store(glance_store.Store): LOG.error(msg) raise exceptions.BackendException(msg) + metadata = {} + if self.backend_group: + metadata['backend'] = u"%s" % self.backend_group + return (loc.get_uri(), image_file.size, - image_file.checksum.hexdigest(), {}) + image_file.checksum.hexdigest(), metadata) @capabilities.check def get(self, location, offset=0, chunk_size=None, context=None): @@ -760,7 +789,8 @@ class Store(glance_store.Store): self.conf, uri=vsphere_url, image_id=image_id, - store_specs=store_specs) + store_specs=store_specs, + backend=self.backend_group) def new_session(insecure=False, ca_file=None, total_retries=None): diff --git a/glance_store/location.py b/glance_store/location.py index 4084179a..c389a42c 100644 --- a/glance_store/location.py +++ b/glance_store/location.py @@ -108,7 +108,7 @@ def get_location_from_uri_and_backend(uri, backend, conf=CONF): raise exceptions.UnknownScheme(scheme=backend) return Location(pieces.scheme, scheme_info['location_class'], - conf, uri=uri) + conf, uri=uri, backend=backend) def register_scheme_backend_map(scheme_map): @@ -148,7 +148,7 @@ class Location(object): """ def __init__(self, store_name, store_location_class, conf, - uri=None, image_id=None, store_specs=None): + uri=None, image_id=None, store_specs=None, backend=None): """ Create a new Location object. @@ -161,12 +161,15 @@ class Location(object): :param store_specs: Dictionary of information about the location of the image that is dependent on the backend store + :param backend: Name of store backend """ self.store_name = store_name self.image_id = image_id self.store_specs = store_specs or {} self.conf = conf - self.store_location = store_location_class(self.store_specs, conf) + self.backend_group = backend + self.store_location = store_location_class( + self.store_specs, conf, backend_group=backend) if uri: self.store_location.parse_uri(uri) @@ -187,9 +190,10 @@ class StoreLocation(object): Base class that must be implemented by each store """ - def __init__(self, store_specs, conf): + def __init__(self, store_specs, conf, backend_group=None): self.conf = conf self.specs = store_specs + self.backend_group = backend_group if self.specs: self.process_specs() diff --git a/glance_store/tests/unit/test_connection_manager.py b/glance_store/tests/unit/test_connection_manager.py index 5c444f48..4529019a 100644 --- a/glance_store/tests/unit/test_connection_manager.py +++ b/glance_store/tests/unit/test_connection_manager.py @@ -45,6 +45,7 @@ class TestConnectionManager(base.StoreBaseTest): conf=self.conf, auth_version='3') + store.backend_group = None store.init_client.return_value = self.client return store diff --git a/glance_store/tests/unit/test_multistore_sheepdog.py b/glance_store/tests/unit/test_multistore_sheepdog.py new file mode 100644 index 00000000..999eba11 --- /dev/null +++ b/glance_store/tests/unit/test_multistore_sheepdog.py @@ -0,0 +1,217 @@ +# Copyright 2018 RedHat Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +from oslo_concurrency import processutils +from oslo_config import cfg +from oslo_utils import units +import six + +import glance_store as store +from glance_store._drivers import sheepdog +from glance_store import exceptions +from glance_store import location +from glance_store.tests import base +from glance_store.tests.unit import test_store_capabilities as test_cap + + +class TestSheepdogMultiStore(base.MultiStoreBaseTest, + test_cap.TestStoreCapabilitiesChecking): + + # NOTE(flaper87): temporary until we + # can move to a fully-local lib. + # (Swift store's fault) + _CONF = cfg.ConfigOpts() + + def setUp(self): + """Establish a clean test environment.""" + super(TestSheepdogMultiStore, self).setUp() + enabled_backends = { + "sheepdog1": "sheepdog", + "sheepdog2": "sheepdog", + } + self.conf = self._CONF + self.conf(args=[]) + self.conf.register_opt(cfg.DictOpt('enabled_backends')) + self.config(enabled_backends=enabled_backends) + store.register_store_opts(self.conf) + self.config(default_backend='sheepdog1', group='glance_store') + + # mock sheepdog commands + def _fake_execute(*cmd, **kwargs): + pass + + execute = mock.patch.object(processutils, 'execute').start() + execute.side_effect = _fake_execute + self.addCleanup(execute.stop) + + # Ensure stores + locations cleared + location.SCHEME_TO_CLS_BACKEND_MAP = {} + + store.create_multi_stores(self.conf) + self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP', + dict()) + self.addCleanup(self.conf.reset) + + self.store = sheepdog.Store(self.conf, backend='sheepdog1') + self.store.configure() + self.store_specs = {'image': '6bd59e6e-c410-11e5-ab67-0a73f1fda51b', + 'addr': '127.0.0.1', + 'port': 7000} + + @mock.patch.object(sheepdog.SheepdogImage, 'write') + @mock.patch.object(sheepdog.SheepdogImage, 'create') + @mock.patch.object(sheepdog.SheepdogImage, 'exist') + def test_add_image(self, mock_exist, mock_create, mock_write): + data = six.BytesIO(b'xx') + mock_exist.return_value = False + + (uri, size, checksum, loc) = self.store.add('fake_image_id', data, 2) + self.assertEqual("sheepdog1", loc["backend"]) + + mock_exist.assert_called_once_with() + mock_create.assert_called_once_with(2) + mock_write.assert_called_once_with(b'xx', 0, 2) + + @mock.patch.object(sheepdog.SheepdogImage, 'write') + @mock.patch.object(sheepdog.SheepdogImage, 'create') + @mock.patch.object(sheepdog.SheepdogImage, 'exist') + def test_add_image_to_different_backend(self, mock_exist, + mock_create, mock_write): + self.store = sheepdog.Store(self.conf, backend='sheepdog2') + self.store.configure() + + data = six.BytesIO(b'xx') + mock_exist.return_value = False + + (uri, size, checksum, loc) = self.store.add('fake_image_id', data, 2) + self.assertEqual("sheepdog2", loc["backend"]) + + mock_exist.assert_called_once_with() + mock_create.assert_called_once_with(2) + mock_write.assert_called_once_with(b'xx', 0, 2) + + @mock.patch.object(sheepdog.SheepdogImage, 'write') + @mock.patch.object(sheepdog.SheepdogImage, 'exist') + def test_add_bad_size_with_image(self, mock_exist, mock_write): + data = six.BytesIO(b'xx') + mock_exist.return_value = False + + self.assertRaises(exceptions.Forbidden, self.store.add, + 'fake_image_id', data, 'test') + + mock_exist.assert_called_once_with() + self.assertEqual(mock_write.call_count, 0) + + @mock.patch.object(sheepdog.SheepdogImage, 'delete') + @mock.patch.object(sheepdog.SheepdogImage, 'write') + @mock.patch.object(sheepdog.SheepdogImage, 'create') + @mock.patch.object(sheepdog.SheepdogImage, 'exist') + def test_cleanup_when_add_image_exception(self, mock_exist, mock_create, + mock_write, mock_delete): + data = six.BytesIO(b'xx') + mock_exist.return_value = False + mock_write.side_effect = exceptions.BackendException + + self.assertRaises(exceptions.BackendException, self.store.add, + 'fake_image_id', data, 2) + + mock_exist.assert_called_once_with() + mock_create.assert_called_once_with(2) + mock_write.assert_called_once_with(b'xx', 0, 2) + mock_delete.assert_called_once_with() + + def test_add_duplicate_image(self): + def _fake_run_command(command, data, *params): + if command == "list -r": + return "= fake_volume 0 1000" + + with mock.patch.object(sheepdog.SheepdogImage, '_run_command') as cmd: + cmd.side_effect = _fake_run_command + data = six.BytesIO(b'xx') + self.assertRaises(exceptions.Duplicate, self.store.add, + 'fake_image_id', data, 2) + + def test_get(self): + def _fake_run_command(command, data, *params): + if command == "list -r": + return "= fake_volume 0 1000" + + with mock.patch.object(sheepdog.SheepdogImage, '_run_command') as cmd: + cmd.side_effect = _fake_run_command + loc = location.Location('test_sheepdog_store', + sheepdog.StoreLocation, + self.conf, store_specs=self.store_specs, + backend='sheepdog1') + ret = self.store.get(loc) + self.assertEqual(1000, ret[1]) + + def test_partial_get(self): + loc = location.Location('test_sheepdog_store', sheepdog.StoreLocation, + self.conf, store_specs=self.store_specs, + backend='sheepdog1') + self.assertRaises(exceptions.StoreRandomGetNotSupported, + self.store.get, loc, chunk_size=1) + + def test_get_size(self): + def _fake_run_command(command, data, *params): + if command == "list -r": + return "= fake_volume 0 1000" + + with mock.patch.object(sheepdog.SheepdogImage, '_run_command') as cmd: + cmd.side_effect = _fake_run_command + loc = location.Location('test_sheepdog_store', + sheepdog.StoreLocation, + self.conf, store_specs=self.store_specs, + backend='sheepdog1') + ret = self.store.get_size(loc) + self.assertEqual(1000, ret) + + def test_delete(self): + called_commands = [] + + def _fake_run_command(command, data, *params): + called_commands.append(command) + if command == "list -r": + return "= fake_volume 0 1000" + + with mock.patch.object(sheepdog.SheepdogImage, '_run_command') as cmd: + cmd.side_effect = _fake_run_command + loc = location.Location('test_sheepdog_store', + sheepdog.StoreLocation, + self.conf, store_specs=self.store_specs, + backend='sheepdog1') + self.store.delete(loc) + self.assertEqual(['list -r', 'delete'], called_commands) + + def test_add_with_verifier(self): + """Test that 'verifier.update' is called when verifier is provided.""" + verifier = mock.MagicMock(name='mock_verifier') + self.store.chunk_size = units.Ki + image_id = 'fake_image_id' + file_size = units.Ki # 1K + file_contents = b"*" * file_size + image_file = six.BytesIO(file_contents) + + def _fake_run_command(command, data, *params): + pass + + with mock.patch.object(sheepdog.SheepdogImage, '_run_command') as cmd: + cmd.side_effect = _fake_run_command + (uri, size, checksum, loc) = self.store.add( + image_id, image_file, file_size, verifier=verifier) + self.assertEqual("sheepdog1", loc["backend"]) + + verifier.update.assert_called_with(file_contents) diff --git a/glance_store/tests/unit/test_multistore_vmware.py b/glance_store/tests/unit/test_multistore_vmware.py new file mode 100644 index 00000000..c6f16905 --- /dev/null +++ b/glance_store/tests/unit/test_multistore_vmware.py @@ -0,0 +1,645 @@ +# Copyright 2018 RedHat Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Tests the Multiple VMware Datastore backend store""" + +import hashlib +import uuid + +import mock +from oslo_config import cfg +from oslo_utils import units +from oslo_vmware import api +from oslo_vmware import exceptions as vmware_exceptions +from oslo_vmware.objects import datacenter as oslo_datacenter +from oslo_vmware.objects import datastore as oslo_datastore +import six + +import glance_store as store +import glance_store._drivers.vmware_datastore as vm_store +from glance_store import exceptions +from glance_store import location +from glance_store.tests import base +from glance_store.tests.unit import test_store_capabilities +from glance_store.tests import utils + + +FAKE_UUID = str(uuid.uuid4()) + +FIVE_KB = 5 * units.Ki + +VMWARE_DS = { + 'debug': True, + 'vmware_server_host': '127.0.0.1', + 'vmware_server_username': 'username', + 'vmware_server_password': 'password', + 'vmware_store_image_dir': '/openstack_glance', + 'vmware_insecure': 'True', + 'vmware_datastores': ['a:b:0'], +} + + +def format_location(host_ip, folder_name, image_id, datastores): + """ + Helper method that returns a VMware Datastore store URI given + the component pieces. + """ + scheme = 'vsphere' + (datacenter_path, datastore_name, weight) = datastores[0].split(':') + return ("%s://%s/folder%s/%s?dcPath=%s&dsName=%s" + % (scheme, host_ip, folder_name, + image_id, datacenter_path, datastore_name)) + + +def fake_datastore_obj(*args, **kwargs): + dc_obj = oslo_datacenter.Datacenter(ref='fake-ref', + name='fake-name') + dc_obj.path = args[0] + return oslo_datastore.Datastore(ref='fake-ref', + datacenter=dc_obj, + name=args[1]) + + +class TestMultiStore(base.MultiStoreBaseTest, + test_store_capabilities.TestStoreCapabilitiesChecking): + + # NOTE(flaper87): temporary until we + # can move to a fully-local lib. + # (Swift store's fault) + _CONF = cfg.ConfigOpts() + + @mock.patch.object(vm_store.Store, '_get_datastore') + @mock.patch('oslo_vmware.api.VMwareAPISession') + def setUp(self, mock_api_session, mock_get_datastore): + """Establish a clean test environment.""" + super(TestMultiStore, self).setUp() + enabled_backends = { + "vmware1": "vmware", + "vmware2": "vmware" + } + self.conf = self._CONF + self.conf(args=[]) + self.conf.register_opt(cfg.DictOpt('enabled_backends')) + self.config(enabled_backends=enabled_backends) + store.register_store_opts(self.conf) + self.config(default_backend='vmware1', group='glance_store') + + # set vmware related config options + self.config(group='vmware1', + vmware_server_username='admin', + vmware_server_password='admin', + vmware_server_host='127.0.0.1', + vmware_insecure='True', + vmware_datastores=['a:b:0'], + vmware_store_image_dir='/openstack_glance') + + self.config(group='vmware2', + vmware_server_username='admin', + vmware_server_password='admin', + vmware_server_host='127.0.0.1', + vmware_insecure='True', + vmware_datastores=['a:b:1'], + vmware_store_image_dir='/openstack_glance_1') + # Ensure stores + locations cleared + location.SCHEME_TO_CLS_BACKEND_MAP = {} + + store.create_multi_stores(self.conf) + self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP', + dict()) + self.addCleanup(self.conf.reset) + + vm_store.Store.CHUNKSIZE = 2 + + mock_get_datastore.side_effect = fake_datastore_obj + + self.store = vm_store.Store(self.conf, backend="vmware1") + self.store.configure() + + def _mock_http_connection(self): + return mock.patch('six.moves.http_client.HTTPConnection') + + @mock.patch('oslo_vmware.api.VMwareAPISession') + def test_get(self, mock_api_session): + """Test a "normal" retrieval of an image in chunks.""" + expected_image_size = 31 + expected_returns = ['I am a teapot, short and stout\n'] + loc = location.get_location_from_uri_and_backend( + "vsphere://127.0.0.1/folder/openstack_glance/%s" + "?dsName=ds1&dcPath=dc1" % FAKE_UUID, "vmware1", conf=self.conf) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response() + (image_file, image_size) = self.store.get(loc) + self.assertEqual(expected_image_size, image_size) + chunks = [c for c in image_file] + self.assertEqual(expected_returns, chunks) + + @mock.patch('oslo_vmware.api.VMwareAPISession') + def test_get_non_existing(self, mock_api_session): + """ + Test that trying to retrieve an image that doesn't exist + raises an error + """ + loc = location.get_location_from_uri_and_backend( + "vsphere://127.0.0.1/folder/openstack_glan" + "ce/%s?dsName=ds1&dcPath=dc1" % FAKE_UUID, "vmware1", + conf=self.conf) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response(status_code=404) + self.assertRaises(exceptions.NotFound, self.store.get, loc) + + @mock.patch.object(vm_store.Store, '_build_vim_cookie_header') + @mock.patch.object(vm_store.Store, 'select_datastore') + @mock.patch.object(vm_store._Reader, 'size') + @mock.patch.object(api, 'VMwareAPISession') + def test_add(self, fake_api_session, fake_size, fake_select_datastore, + fake_cookie): + """Test that we can add an image via the VMware backend.""" + fake_select_datastore.return_value = self.store.datastores[0][0] + expected_image_id = str(uuid.uuid4()) + expected_size = FIVE_KB + expected_contents = b"*" * expected_size + hash_code = hashlib.md5(expected_contents) + expected_checksum = hash_code.hexdigest() + fake_size.__get__ = mock.Mock(return_value=expected_size) + expected_cookie = 'vmware_soap_session=fake-uuid' + fake_cookie.return_value = expected_cookie + expected_headers = {'Content-Length': six.text_type(expected_size), + 'Cookie': expected_cookie} + with mock.patch('hashlib.md5') as md5: + md5.return_value = hash_code + expected_location = format_location( + VMWARE_DS['vmware_server_host'], + VMWARE_DS['vmware_store_image_dir'], + expected_image_id, + VMWARE_DS['vmware_datastores']) + image = six.BytesIO(expected_contents) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response() + location, size, checksum, metadata = self.store.add( + expected_image_id, image, expected_size) + _, kwargs = HttpConn.call_args + self.assertEqual(expected_headers, kwargs['headers']) + self.assertEqual("vmware1", metadata["backend"]) + + self.assertEqual(utils.sort_url_by_qs_keys(expected_location), + utils.sort_url_by_qs_keys(location)) + self.assertEqual(expected_size, size) + self.assertEqual(expected_checksum, checksum) + + @mock.patch.object(vm_store.Store, 'select_datastore') + @mock.patch.object(vm_store._Reader, 'size') + @mock.patch('oslo_vmware.api.VMwareAPISession') + def test_add_size_zero(self, mock_api_session, fake_size, + fake_select_datastore): + """ + Test that when specifying size zero for the image to add, + the actual size of the image is returned. + """ + fake_select_datastore.return_value = self.store.datastores[0][0] + expected_image_id = str(uuid.uuid4()) + expected_size = FIVE_KB + expected_contents = b"*" * expected_size + hash_code = hashlib.md5(expected_contents) + expected_checksum = hash_code.hexdigest() + fake_size.__get__ = mock.Mock(return_value=expected_size) + with mock.patch('hashlib.md5') as md5: + md5.return_value = hash_code + expected_location = format_location( + VMWARE_DS['vmware_server_host'], + VMWARE_DS['vmware_store_image_dir'], + expected_image_id, + VMWARE_DS['vmware_datastores']) + image = six.BytesIO(expected_contents) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response() + location, size, checksum, metadata = self.store.add( + expected_image_id, image, 0) + self.assertEqual("vmware1", metadata["backend"]) + + self.assertEqual(utils.sort_url_by_qs_keys(expected_location), + utils.sort_url_by_qs_keys(location)) + self.assertEqual(expected_size, size) + self.assertEqual(expected_checksum, checksum) + + @mock.patch.object(vm_store.Store, 'select_datastore') + @mock.patch('glance_store._drivers.vmware_datastore._Reader') + def test_add_with_verifier(self, fake_reader, fake_select_datastore): + """Test that the verifier is passed to the _Reader during add.""" + verifier = mock.MagicMock(name='mock_verifier') + image_id = str(uuid.uuid4()) + size = FIVE_KB + contents = b"*" * size + image = six.BytesIO(contents) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response() + location, size, checksum, metadata = self.store.add( + image_id, image, size, verifier=verifier) + self.assertEqual("vmware1", metadata["backend"]) + + fake_reader.assert_called_with(image, verifier) + + @mock.patch.object(vm_store.Store, 'select_datastore') + @mock.patch('glance_store._drivers.vmware_datastore._Reader') + def test_add_with_verifier_size_zero(self, fake_reader, fake_select_ds): + """Test that the verifier is passed to the _ChunkReader during add.""" + verifier = mock.MagicMock(name='mock_verifier') + image_id = str(uuid.uuid4()) + size = FIVE_KB + contents = b"*" * size + image = six.BytesIO(contents) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response() + location, size, checksum, metadata = self.store.add( + image_id, image, 0, verifier=verifier) + self.assertEqual("vmware1", metadata["backend"]) + + fake_reader.assert_called_with(image, verifier) + + @mock.patch('oslo_vmware.api.VMwareAPISession') + def test_delete(self, mock_api_session): + """Test we can delete an existing image in the VMware store.""" + loc = location.get_location_from_uri_and_backend( + "vsphere://127.0.0.1/folder/openstack_glance/%s?" + "dsName=ds1&dcPath=dc1" % FAKE_UUID, "vmware1", conf=self.conf) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response() + vm_store.Store._service_content = mock.Mock() + self.store.delete(loc) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response(status_code=404) + self.assertRaises(exceptions.NotFound, self.store.get, loc) + + @mock.patch('oslo_vmware.api.VMwareAPISession') + def test_delete_non_existing(self, mock_api_session): + """ + Test that trying to delete an image that doesn't exist raises an error + """ + loc = location.get_location_from_uri_and_backend( + "vsphere://127.0.0.1/folder/openstack_glance/%s?" + "dsName=ds1&dcPath=dc1" % FAKE_UUID, + "vmware1", conf=self.conf) + with mock.patch.object(self.store.session, + 'wait_for_task') as mock_task: + mock_task.side_effect = vmware_exceptions.FileNotFoundException + self.assertRaises(exceptions.NotFound, self.store.delete, loc) + + @mock.patch('oslo_vmware.api.VMwareAPISession') + def test_get_size(self, mock_api_session): + """ + Test we can get the size of an existing image in the VMware store + """ + loc = location.get_location_from_uri_and_backend( + "vsphere://127.0.0.1/folder/openstack_glance/%s" + "?dsName=ds1&dcPath=dc1" % FAKE_UUID, "vmware1", conf=self.conf) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response() + image_size = self.store.get_size(loc) + self.assertEqual(image_size, 31) + + @mock.patch('oslo_vmware.api.VMwareAPISession') + def test_get_size_non_existing(self, mock_api_session): + """ + Test that trying to retrieve an image size that doesn't exist + raises an error + """ + loc = location.get_location_from_uri_and_backend( + "vsphere://127.0.0.1/folder/openstack_glan" + "ce/%s?dsName=ds1&dcPath=dc1" % FAKE_UUID, + "vmware1", conf=self.conf) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response(status_code=404) + self.assertRaises(exceptions.NotFound, self.store.get_size, loc) + + def test_reader_full(self): + content = b'XXX' + image = six.BytesIO(content) + expected_checksum = hashlib.md5(content).hexdigest() + reader = vm_store._Reader(image) + ret = reader.read() + self.assertEqual(content, ret) + self.assertEqual(expected_checksum, reader.checksum.hexdigest()) + self.assertEqual(len(content), reader.size) + + def test_reader_partial(self): + content = b'XXX' + image = six.BytesIO(content) + expected_checksum = hashlib.md5(b'X').hexdigest() + reader = vm_store._Reader(image) + ret = reader.read(1) + self.assertEqual(b'X', ret) + self.assertEqual(expected_checksum, reader.checksum.hexdigest()) + self.assertEqual(1, reader.size) + + def test_reader_with_verifier(self): + content = b'XXX' + image = six.BytesIO(content) + verifier = mock.MagicMock(name='mock_verifier') + reader = vm_store._Reader(image, verifier) + reader.read() + verifier.update.assert_called_with(content) + + def test_sanity_check_multiple_datastores(self): + self.config(group='vmware1', vmware_api_retry_count=1) + self.config(group='vmware1', vmware_task_poll_interval=1) + self.config(group='vmware1', vmware_datastores=['a:b:0', 'a:d:0']) + try: + self.store._sanity_check() + except exceptions.BadStoreConfiguration: + self.fail() + + def test_parse_datastore_info_and_weight_less_opts(self): + datastore = 'a' + self.assertRaises(exceptions.BadStoreConfiguration, + self.store._parse_datastore_info_and_weight, + datastore) + + def test_parse_datastore_info_and_weight_invalid_weight(self): + datastore = 'a:b:c' + self.assertRaises(exceptions.BadStoreConfiguration, + self.store._parse_datastore_info_and_weight, + datastore) + + def test_parse_datastore_info_and_weight_empty_opts(self): + datastore = 'a: :0' + self.assertRaises(exceptions.BadStoreConfiguration, + self.store._parse_datastore_info_and_weight, + datastore) + datastore = ':b:0' + self.assertRaises(exceptions.BadStoreConfiguration, + self.store._parse_datastore_info_and_weight, + datastore) + + def test_parse_datastore_info_and_weight(self): + datastore = 'a:b:100' + parts = self.store._parse_datastore_info_and_weight(datastore) + self.assertEqual('a', parts[0]) + self.assertEqual('b', parts[1]) + self.assertEqual('100', parts[2]) + + def test_parse_datastore_info_and_weight_default_weight(self): + datastore = 'a:b' + parts = self.store._parse_datastore_info_and_weight(datastore) + self.assertEqual('a', parts[0]) + self.assertEqual('b', parts[1]) + self.assertEqual(0, parts[2]) + + @mock.patch.object(vm_store.Store, 'select_datastore') + @mock.patch.object(api, 'VMwareAPISession') + def test_unexpected_status(self, mock_api_session, mock_select_datastore): + expected_image_id = str(uuid.uuid4()) + expected_size = FIVE_KB + expected_contents = b"*" * expected_size + image = six.BytesIO(expected_contents) + self.session = mock.Mock() + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response(status_code=401) + self.assertRaises(exceptions.BackendException, + self.store.add, + expected_image_id, image, expected_size) + + @mock.patch.object(vm_store.Store, 'select_datastore') + @mock.patch.object(api, 'VMwareAPISession') + def test_unexpected_status_no_response_body(self, mock_api_session, + mock_select_datastore): + expected_image_id = str(uuid.uuid4()) + expected_size = FIVE_KB + expected_contents = b"*" * expected_size + image = six.BytesIO(expected_contents) + self.session = mock.Mock() + with self._mock_http_connection() as HttpConn: + HttpConn.return_value = utils.fake_response(status_code=500, + no_response_body=True) + self.assertRaises(exceptions.BackendException, + self.store.add, + expected_image_id, image, expected_size) + + @mock.patch.object(api, 'VMwareAPISession') + def test_reset_session(self, mock_api_session): + self.store.reset_session() + self.assertTrue(mock_api_session.called) + + @mock.patch.object(api, 'VMwareAPISession') + def test_build_vim_cookie_header_active(self, mock_api_session): + self.store.session.is_current_session_active = mock.Mock() + self.store.session.is_current_session_active.return_value = True + self.store._build_vim_cookie_header(True) + self.assertFalse(mock_api_session.called) + + @mock.patch.object(api, 'VMwareAPISession') + def test_build_vim_cookie_header_expired(self, mock_api_session): + self.store.session.is_current_session_active = mock.Mock() + self.store.session.is_current_session_active.return_value = False + self.store._build_vim_cookie_header(True) + self.assertTrue(mock_api_session.called) + + @mock.patch.object(api, 'VMwareAPISession') + def test_build_vim_cookie_header_expired_noverify(self, mock_api_session): + self.store.session.is_current_session_active = mock.Mock() + self.store.session.is_current_session_active.return_value = False + self.store._build_vim_cookie_header() + self.assertFalse(mock_api_session.called) + + @mock.patch.object(vm_store.Store, 'select_datastore') + @mock.patch.object(api, 'VMwareAPISession') + def test_add_ioerror(self, mock_api_session, mock_select_datastore): + mock_select_datastore.return_value = self.store.datastores[0][0] + expected_image_id = str(uuid.uuid4()) + expected_size = FIVE_KB + expected_contents = b"*" * expected_size + image = six.BytesIO(expected_contents) + self.session = mock.Mock() + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.request.side_effect = IOError + self.assertRaises(exceptions.BackendException, + self.store.add, + expected_image_id, image, expected_size) + + def test_qs_sort_with_literal_question_mark(self): + url = 'scheme://example.com/path?key2=val2&key1=val1?sort=true' + exp_url = 'scheme://example.com/path?key1=val1%3Fsort%3Dtrue&key2=val2' + self.assertEqual(exp_url, + utils.sort_url_by_qs_keys(url)) + + @mock.patch.object(vm_store.Store, '_get_datastore') + @mock.patch.object(api, 'VMwareAPISession') + def test_build_datastore_weighted_map(self, mock_api_session, mock_ds_obj): + datastores = ['a:b:100', 'c:d:100', 'e:f:200'] + mock_ds_obj.side_effect = fake_datastore_obj + ret = self.store._build_datastore_weighted_map(datastores) + ds = ret[200] + self.assertEqual('e', ds[0].datacenter.path) + self.assertEqual('f', ds[0].name) + ds = ret[100] + self.assertEqual(2, len(ds)) + + @mock.patch.object(vm_store.Store, '_get_datastore') + @mock.patch.object(api, 'VMwareAPISession') + def test_build_datastore_weighted_map_equal_weight(self, mock_api_session, + mock_ds_obj): + datastores = ['a:b:200', 'a:b:200'] + mock_ds_obj.side_effect = fake_datastore_obj + ret = self.store._build_datastore_weighted_map(datastores) + ds = ret[200] + self.assertEqual(2, len(ds)) + + @mock.patch.object(vm_store.Store, '_get_datastore') + @mock.patch.object(api, 'VMwareAPISession') + def test_build_datastore_weighted_map_empty_list(self, mock_api_session, + mock_ds_ref): + datastores = [] + ret = self.store._build_datastore_weighted_map(datastores) + self.assertEqual({}, ret) + + @mock.patch.object(vm_store.Store, '_get_datastore') + @mock.patch.object(vm_store.Store, '_get_freespace') + def test_select_datastore_insufficient_freespace(self, mock_get_freespace, + mock_ds_ref): + datastores = ['a:b:100', 'c:d:100', 'e:f:200'] + image_size = 10 + self.store.datastores = ( + self.store._build_datastore_weighted_map(datastores)) + freespaces = [5, 5, 5] + + def fake_get_fp(*args, **kwargs): + return freespaces.pop(0) + mock_get_freespace.side_effect = fake_get_fp + self.assertRaises(exceptions.StorageFull, + self.store.select_datastore, image_size) + + @mock.patch.object(vm_store.Store, '_get_datastore') + @mock.patch.object(vm_store.Store, '_get_freespace') + def test_select_datastore_insufficient_fs_one_ds(self, mock_get_freespace, + mock_ds_ref): + # Tests if fs is updated with just one datastore. + datastores = ['a:b:100'] + image_size = 10 + self.store.datastores = ( + self.store._build_datastore_weighted_map(datastores)) + freespaces = [5] + + def fake_get_fp(*args, **kwargs): + return freespaces.pop(0) + mock_get_freespace.side_effect = fake_get_fp + self.assertRaises(exceptions.StorageFull, + self.store.select_datastore, image_size) + + @mock.patch.object(vm_store.Store, '_get_datastore') + @mock.patch.object(vm_store.Store, '_get_freespace') + def test_select_datastore_equal_freespace(self, mock_get_freespace, + mock_ds_obj): + datastores = ['a:b:100', 'c:d:100', 'e:f:200'] + image_size = 10 + mock_ds_obj.side_effect = fake_datastore_obj + self.store.datastores = ( + self.store._build_datastore_weighted_map(datastores)) + freespaces = [11, 11, 11] + + def fake_get_fp(*args, **kwargs): + return freespaces.pop(0) + mock_get_freespace.side_effect = fake_get_fp + + ds = self.store.select_datastore(image_size) + self.assertEqual('e', ds.datacenter.path) + self.assertEqual('f', ds.name) + + @mock.patch.object(vm_store.Store, '_get_datastore') + @mock.patch.object(vm_store.Store, '_get_freespace') + def test_select_datastore_contention(self, mock_get_freespace, + mock_ds_obj): + datastores = ['a:b:100', 'c:d:100', 'e:f:200'] + image_size = 10 + mock_ds_obj.side_effect = fake_datastore_obj + self.store.datastores = ( + self.store._build_datastore_weighted_map(datastores)) + freespaces = [5, 11, 12] + + def fake_get_fp(*args, **kwargs): + return freespaces.pop(0) + mock_get_freespace.side_effect = fake_get_fp + ds = self.store.select_datastore(image_size) + self.assertEqual('c', ds.datacenter.path) + self.assertEqual('d', ds.name) + + def test_select_datastore_empty_list(self): + datastores = [] + self.store.datastores = ( + self.store._build_datastore_weighted_map(datastores)) + self.assertRaises(exceptions.StorageFull, + self.store.select_datastore, 10) + + @mock.patch('oslo_vmware.api.VMwareAPISession') + def test_get_datacenter_ref(self, mock_api_session): + datacenter_path = 'Datacenter1' + self.store._get_datacenter(datacenter_path) + self.store.session.invoke_api.assert_called_with( + self.store.session.vim, + 'FindByInventoryPath', + self.store.session.vim.service_content.searchIndex, + inventoryPath=datacenter_path) + + @mock.patch('oslo_vmware.api.VMwareAPISession') + def test_http_get_redirect(self, mock_api_session): + # Add two layers of redirects to the response stack, which will + # return the default 200 OK with the expected data after resolving + # both redirects. + redirect1 = {"location": "https://example.com?dsName=ds1&dcPath=dc1"} + redirect2 = {"location": "https://example.com?dsName=ds2&dcPath=dc2"} + responses = [utils.fake_response(), + utils.fake_response(status_code=302, headers=redirect1), + utils.fake_response(status_code=301, headers=redirect2)] + + def getresponse(*args, **kwargs): + return responses.pop() + + expected_image_size = 31 + expected_returns = ['I am a teapot, short and stout\n'] + loc = location.get_location_from_uri_and_backend( + "vsphere://127.0.0.1/folder/openstack_glance/%s" + "?dsName=ds1&dcPath=dc1" % FAKE_UUID, "vmware1", conf=self.conf) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.side_effect = getresponse + (image_file, image_size) = self.store.get(loc) + self.assertEqual(expected_image_size, image_size) + chunks = [c for c in image_file] + self.assertEqual(expected_returns, chunks) + + @mock.patch('oslo_vmware.api.VMwareAPISession') + def test_http_get_max_redirects(self, mock_api_session): + redirect = {"location": "https://example.com?dsName=ds1&dcPath=dc1"} + responses = ([utils.fake_response(status_code=302, headers=redirect)] + * (vm_store.MAX_REDIRECTS + 1)) + + def getresponse(*args, **kwargs): + return responses.pop() + + loc = location.get_location_from_uri_and_backend( + "vsphere://127.0.0.1/folder/openstack_glance/%s" + "?dsName=ds1&dcPath=dc1" % FAKE_UUID, "vmware1", conf=self.conf) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.side_effect = getresponse + self.assertRaises(exceptions.MaxRedirectsExceeded, self.store.get, + loc) + + @mock.patch('oslo_vmware.api.VMwareAPISession') + def test_http_get_redirect_invalid(self, mock_api_session): + redirect = {"location": "https://example.com?dsName=ds1&dcPath=dc1"} + + loc = location.get_location_from_uri_and_backend( + "vsphere://127.0.0.1/folder/openstack_glance/%s" + "?dsName=ds1&dcPath=dc1" % FAKE_UUID, "vmware1", conf=self.conf) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response(status_code=307, + headers=redirect) + self.assertRaises(exceptions.BadStoreUri, self.store.get, loc) diff --git a/glance_store/tests/unit/test_swift_store_multibackend.py b/glance_store/tests/unit/test_swift_store_multibackend.py new file mode 100644 index 00000000..defbb3c4 --- /dev/null +++ b/glance_store/tests/unit/test_swift_store_multibackend.py @@ -0,0 +1,2206 @@ +# Copyright 2018 RedHat Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Tests the Swift backend store""" + +import copy +import fixtures +import hashlib +import mock +import tempfile +import uuid + +from oslo_config import cfg +from oslo_utils import encodeutils +from oslo_utils import units +from oslotest import moxstubout +import requests_mock +import six +from six import moves +from six.moves import http_client +# NOTE(jokke): simplified transition to py3, behaves like py2 xrange +from six.moves import range +import swiftclient + +from glance_store._drivers.swift import connection_manager as manager +from glance_store._drivers.swift import store as swift +from glance_store._drivers.swift import utils as sutils +from glance_store import capabilities +from glance_store import exceptions +from glance_store import location +import glance_store.multi_backend as store +from glance_store.tests import base +from glance_store.tests.unit import test_store_capabilities + + +CONF = cfg.CONF + +FAKE_UUID = lambda: str(uuid.uuid4()) +FAKE_UUID2 = lambda: str(uuid.uuid4()) + +Store = swift.Store +FIVE_KB = 5 * units.Ki +FIVE_GB = 5 * units.Gi +MAX_SWIFT_OBJECT_SIZE = FIVE_GB +SWIFT_PUT_OBJECT_CALLS = 0 +SWIFT_CONF = {'swift_store_auth_address': 'localhost:8080', + 'swift_store_container': 'glance', + 'swift_store_user': 'user', + 'swift_store_key': 'key', + 'swift_store_retry_get_count': 1, + 'default_swift_reference': 'ref1' + } + + +# We stub out as little as possible to ensure that the code paths +# between swift and swiftclient are tested +# thoroughly +def stub_out_swiftclient(stubs, swift_store_auth_version): + fixture_containers = ['glance'] + fixture_container_headers = {} + fixture_headers = { + 'glance/%s' % FAKE_UUID: { + 'content-length': FIVE_KB, + 'etag': 'c2e5db72bd7fd153f53ede5da5a06de3' + }, + 'glance/%s' % FAKE_UUID2: {'x-static-large-object': 'true', }, + } + fixture_objects = {'glance/%s' % FAKE_UUID: six.BytesIO(b"*" * FIVE_KB), + 'glance/%s' % FAKE_UUID2: six.BytesIO(b"*" * FIVE_KB), } + + def fake_head_container(url, token, container, **kwargs): + if container not in fixture_containers: + msg = "No container %s found" % container + status = http_client.NOT_FOUND + raise swiftclient.ClientException(msg, http_status=status) + return fixture_container_headers + + def fake_put_container(url, token, container, **kwargs): + fixture_containers.append(container) + + def fake_post_container(url, token, container, headers, **kwargs): + for key, value in headers.items(): + fixture_container_headers[key] = value + + def fake_put_object(url, token, container, name, contents, **kwargs): + # PUT returns the ETag header for the newly-added object + # Large object manifest... + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS += 1 + CHUNKSIZE = 64 * units.Ki + fixture_key = "%s/%s" % (container, name) + if fixture_key not in fixture_headers: + if kwargs.get('headers'): + etag = kwargs['headers']['ETag'] + manifest = kwargs.get('headers').get('X-Object-Manifest') + fixture_headers[fixture_key] = {'manifest': True, + 'etag': etag, + 'x-object-manifest': manifest} + fixture_objects[fixture_key] = None + return etag + if hasattr(contents, 'read'): + fixture_object = six.BytesIO() + read_len = 0 + chunk = contents.read(CHUNKSIZE) + checksum = hashlib.md5() + while chunk: + fixture_object.write(chunk) + read_len += len(chunk) + checksum.update(chunk) + chunk = contents.read(CHUNKSIZE) + etag = checksum.hexdigest() + else: + fixture_object = six.BytesIO(contents) + read_len = len(contents) + etag = hashlib.md5(fixture_object.getvalue()).hexdigest() + if read_len > MAX_SWIFT_OBJECT_SIZE: + msg = ('Image size:%d exceeds Swift max:%d' % + (read_len, MAX_SWIFT_OBJECT_SIZE)) + raise swiftclient.ClientException( + msg, http_status=http_client.REQUEST_ENTITY_TOO_LARGE) + fixture_objects[fixture_key] = fixture_object + fixture_headers[fixture_key] = { + 'content-length': read_len, + 'etag': etag} + return etag + else: + msg = ("Object PUT failed - Object with key %s already exists" + % fixture_key) + raise swiftclient.ClientException(msg, + http_status=http_client.CONFLICT) + + def fake_get_object(conn, container, name, **kwargs): + # GET returns the tuple (list of headers, file object) + fixture_key = "%s/%s" % (container, name) + if fixture_key not in fixture_headers: + msg = "Object GET failed" + status = http_client.NOT_FOUND + raise swiftclient.ClientException(msg, http_status=status) + + byte_range = None + headers = kwargs.get('headers', dict()) + if headers is not None: + headers = dict((k.lower(), v) for k, v in headers.items()) + if 'range' in headers: + byte_range = headers.get('range') + + fixture = fixture_headers[fixture_key] + if 'manifest' in fixture: + # Large object manifest... we return a file containing + # all objects with prefix of this fixture key + chunk_keys = sorted([k for k in fixture_headers.keys() + if k.startswith(fixture_key) and + k != fixture_key]) + result = six.BytesIO() + for key in chunk_keys: + result.write(fixture_objects[key].getvalue()) + else: + result = fixture_objects[fixture_key] + + if byte_range is not None: + start = int(byte_range.split('=')[1].strip('-')) + result = six.BytesIO(result.getvalue()[start:]) + fixture_headers[fixture_key]['content-length'] = len( + result.getvalue()) + + return fixture_headers[fixture_key], result + + def fake_head_object(url, token, container, name, **kwargs): + # HEAD returns the list of headers for an object + try: + fixture_key = "%s/%s" % (container, name) + return fixture_headers[fixture_key] + except KeyError: + msg = "Object HEAD failed - Object does not exist" + status = http_client.NOT_FOUND + raise swiftclient.ClientException(msg, http_status=status) + + def fake_delete_object(url, token, container, name, **kwargs): + # DELETE returns nothing + fixture_key = "%s/%s" % (container, name) + if fixture_key not in fixture_headers: + msg = "Object DELETE failed - Object does not exist" + status = http_client.NOT_FOUND + raise swiftclient.ClientException(msg, http_status=status) + else: + del fixture_headers[fixture_key] + del fixture_objects[fixture_key] + + def fake_http_connection(*args, **kwargs): + return None + + def fake_get_auth(url, user, key, auth_version, **kwargs): + if url is None: + return None, None + if 'http' in url and '://' not in url: + raise ValueError('Invalid url %s' % url) + # Check the auth version against the configured value + if swift_store_auth_version != auth_version: + msg = 'AUTHENTICATION failed (version mismatch)' + raise swiftclient.ClientException(msg) + return None, None + + stubs.Set(swiftclient.client, + 'head_container', fake_head_container) + stubs.Set(swiftclient.client, + 'put_container', fake_put_container) + stubs.Set(swiftclient.client, + 'post_container', fake_post_container) + stubs.Set(swiftclient.client, + 'put_object', fake_put_object) + stubs.Set(swiftclient.client, + 'delete_object', fake_delete_object) + stubs.Set(swiftclient.client, + 'head_object', fake_head_object) + stubs.Set(swiftclient.client.Connection, + 'get_object', fake_get_object) + stubs.Set(swiftclient.client, + 'get_auth', fake_get_auth) + stubs.Set(swiftclient.client, + 'http_connection', fake_http_connection) + + +class SwiftTests(object): + + def mock_keystone_client(self): + # mock keystone client functions to avoid dependency errors + swift.ks_v3 = mock.MagicMock() + swift.ks_session = mock.MagicMock() + swift.ks_client = mock.MagicMock() + + @property + def swift_store_user(self): + return 'tenant:user1' + + def test_get_size(self): + """ + Test that we can get the size of an object in the swift store + """ + uri = "swift://%s:key@auth_address/glance/%s" % ( + self.swift_store_user, FAKE_UUID) + loc = location.get_location_from_uri_and_backend( + uri, "swift1", conf=self.conf) + image_size = self.store.get_size(loc) + self.assertEqual(5120, image_size) + + @mock.patch.object(store, + 'get_store_from_store_identifier') + def test_get_size_with_multi_tenant_on(self, mock_get): + """Test that single tenant uris work with multi tenant on.""" + mock_get.return_value = self.store + uri = ("swift://%s:key@auth_address/glance/%s" % + (self.swift_store_user, FAKE_UUID)) + self.config(group="swift1", swift_store_config_file=None) + self.config(group="swift1", swift_store_multi_tenant=True) + # NOTE(markwash): ensure the image is found + ctxt = mock.MagicMock() + size = store.get_size_from_uri_and_backend( + uri, "swift1", context=ctxt) + self.assertEqual(5120, size) + + def test_multi_tenant_with_swift_config(self): + """ + Test that Glance does not start when a config file is set on + multi-tenant mode + """ + schemes = ['swift', 'swift+config'] + for s in schemes: + self.config(group='glance_store', default_backend="swift1") + self.config(group="swift1", + swift_store_config_file='not/none', + swift_store_multi_tenant=True) + self.assertRaises(exceptions.BadStoreConfiguration, + Store, self.conf, backend="swift1") + + def test_get(self): + """Test a "normal" retrieval of an image in chunks.""" + uri = "swift://%s:key@auth_address/glance/%s" % ( + self.swift_store_user, FAKE_UUID) + loc = location.get_location_from_uri_and_backend( + uri, "swift1", conf=self.conf) + (image_swift, image_size) = self.store.get(loc) + self.assertEqual(5120, image_size) + + expected_data = b"*" * FIVE_KB + data = b"" + + for chunk in image_swift: + data += chunk + self.assertEqual(expected_data, data) + + def test_get_with_retry(self): + """ + Test a retrieval where Swift does not get the full image in a single + request. + """ + uri = "swift://%s:key@auth_address/glance/%s" % ( + self.swift_store_user, FAKE_UUID) + loc = location.get_location_from_uri_and_backend( + uri, "swift1", conf=self.conf) + ctxt = mock.MagicMock() + (image_swift, image_size) = self.store.get(loc, context=ctxt) + resp_full = b''.join([chunk for chunk in image_swift.wrapped]) + resp_half = resp_full[:len(resp_full) // 2] + resp_half = six.BytesIO(resp_half) + manager = self.store.get_manager(loc.store_location, ctxt) + + image_swift.wrapped = swift.swift_retry_iter(resp_half, image_size, + self.store, + loc.store_location, + manager) + self.assertEqual(5120, image_size) + + expected_data = b"*" * FIVE_KB + data = b"" + + for chunk in image_swift: + data += chunk + self.assertEqual(expected_data, data) + + def test_get_with_http_auth(self): + """ + Test a retrieval from Swift with an HTTP authurl. This is + specified either via a Location header with swift+http:// or using + http:// in the swift_store_auth_address config value + """ + loc = location.get_location_from_uri_and_backend( + "swift+http://%s:key@auth_address/glance/%s" % + (self.swift_store_user, FAKE_UUID), "swift1", conf=self.conf) + + ctxt = mock.MagicMock() + (image_swift, image_size) = self.store.get(loc, context=ctxt) + self.assertEqual(5120, image_size) + + expected_data = b"*" * FIVE_KB + data = b"" + + for chunk in image_swift: + data += chunk + self.assertEqual(expected_data, data) + + def test_get_non_existing(self): + """ + Test that trying to retrieve a swift that doesn't exist + raises an error + """ + loc = location.get_location_from_uri_and_backend( + "swift://%s:key@authurl/glance/noexist" % (self.swift_store_user), + "swift1", conf=self.conf) + self.assertRaises(exceptions.NotFound, + self.store.get, + loc) + + def test_buffered_reader_opts(self): + self.config(group="swift1", swift_buffer_on_upload=True) + self.config(group="swift1", swift_upload_buffer_dir=self.test_dir) + try: + self.store = Store(self.conf, backend="swift1") + except exceptions.BadStoreConfiguration: + self.fail("Buffered Reader exception raised when it " + "should not have been") + + def test_buffered_reader_with_invalid_path(self): + self.config(group="swift1", swift_buffer_on_upload=True) + self.config(group="swift1", swift_upload_buffer_dir="/some/path") + self.store = Store(self.conf, backend="swift1") + self.assertRaises(exceptions.BadStoreConfiguration, + self.store.configure) + + def test_buffered_reader_with_no_path_given(self): + self.config(group="swift1", swift_buffer_on_upload=True) + self.store = Store(self.conf, backend="swift1") + self.assertRaises(exceptions.BadStoreConfiguration, + self.store.configure) + + @mock.patch('glance_store._drivers.swift.utils' + '.is_multiple_swift_store_accounts_enabled', + mock.Mock(return_value=False)) + def test_add(self): + """Test that we can add an image via the swift backend.""" + moves.reload_module(swift) + self.mock_keystone_client() + self.store = Store(self.conf, backend="swift1") + self.store.configure() + expected_swift_size = FIVE_KB + expected_swift_contents = b"*" * expected_swift_size + expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() + expected_image_id = str(uuid.uuid4()) + loc = "swift+https://tenant%%3Auser1:key@localhost:8080/glance/%s" + expected_location = loc % (expected_image_id) + image_swift = six.BytesIO(expected_swift_contents) + + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + + loc, size, checksum, metadata = self.store.add( + expected_image_id, image_swift, expected_swift_size) + + self.assertEqual("swift1", metadata["backend"]) + self.assertEqual(expected_location, loc) + self.assertEqual(expected_swift_size, size) + self.assertEqual(expected_checksum, checksum) + # Expecting a single object to be created on Swift i.e. no chunking. + self.assertEqual(1, SWIFT_PUT_OBJECT_CALLS) + + loc = location.get_location_from_uri_and_backend( + expected_location, "swift1", conf=self.conf) + (new_image_swift, new_image_size) = self.store.get(loc) + new_image_contents = b''.join([chunk for chunk in new_image_swift]) + new_image_swift_size = len(new_image_swift) + + self.assertEqual(expected_swift_contents, new_image_contents) + self.assertEqual(expected_swift_size, new_image_swift_size) + + def test_add_multi_store(self): + + conf = copy.deepcopy(SWIFT_CONF) + conf['default_swift_reference'] = 'store_2' + self.config(group="swift1", **conf) + moves.reload_module(swift) + self.mock_keystone_client() + self.store = Store(self.conf, backend="swift1") + self.store.configure() + + expected_swift_size = FIVE_KB + expected_swift_contents = b"*" * expected_swift_size + expected_image_id = str(uuid.uuid4()) + image_swift = six.BytesIO(expected_swift_contents) + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + loc = 'swift+config://store_2/glance/%s' + + expected_location = loc % (expected_image_id) + + location, size, checksum, arg = self.store.add(expected_image_id, + image_swift, + expected_swift_size) + self.assertEqual("swift1", arg['backend']) + self.assertEqual(expected_location, location) + + @mock.patch('glance_store._drivers.swift.utils' + '.is_multiple_swift_store_accounts_enabled', + mock.Mock(return_value=False)) + def test_multi_tenant_image_add_uses_users_context(self): + expected_swift_size = FIVE_KB + expected_swift_contents = b"*" * expected_swift_size + expected_image_id = str(uuid.uuid4()) + expected_container = 'container_' + expected_image_id + loc = 'swift+https://some_endpoint/%s/%s' + expected_location = loc % (expected_container, expected_image_id) + image_swift = six.BytesIO(expected_swift_contents) + + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + + self.config(group='swift1', swift_store_container='container') + self.config(group='swift1', swift_store_create_container_on_put=True) + self.config(group='swift1', swift_store_multi_tenant=True) + service_catalog = [ + { + 'endpoint_links': [], + 'endpoints': [ + { + 'adminURL': 'https://some_admin_endpoint', + 'region': 'RegionOne', + 'internalURL': 'https://some_internal_endpoint', + 'publicURL': 'https://some_endpoint', + }, + ], + 'type': 'object-store', + 'name': 'Object Storage Service', + } + ] + ctxt = mock.MagicMock( + user='user', tenant='tenant', auth_token='123', + service_catalog=service_catalog) + store = swift.MultiTenantStore(self.conf, backend='swift1') + store.configure() + loc, size, checksum, metadata = store.add(expected_image_id, + image_swift, + expected_swift_size, + context=ctxt) + + self.assertEqual("swift1", metadata['backend']) + # ensure that image add uses user's context + self.assertEqual(expected_location, loc) + + @mock.patch('glance_store._drivers.swift.utils' + '.is_multiple_swift_store_accounts_enabled', + mock.Mock(return_value=True)) + def test_add_auth_url_variations(self): + """ + Test that we can add an image via the swift backend with + a variety of different auth_address values + """ + conf = copy.deepcopy(SWIFT_CONF) + self.config(group="swift1", **conf) + + variations = { + 'store_4': 'swift+config://store_4/glance/%s', + 'store_5': 'swift+config://store_5/glance/%s', + 'store_6': 'swift+config://store_6/glance/%s' + } + + for variation, expected_location in variations.items(): + image_id = str(uuid.uuid4()) + expected_location = expected_location % image_id + expected_swift_size = FIVE_KB + expected_swift_contents = b"*" * expected_swift_size + expected_checksum = \ + hashlib.md5(expected_swift_contents).hexdigest() + + image_swift = six.BytesIO(expected_swift_contents) + + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + conf['default_swift_reference'] = variation + self.config(group="swift1", **conf) + moves.reload_module(swift) + self.mock_keystone_client() + self.store = Store(self.conf, backend="swift1") + self.store.configure() + loc, size, checksum, metadata = self.store.add(image_id, + image_swift, + expected_swift_size) + + self.assertEqual("swift1", metadata['backend']) + self.assertEqual(expected_location, loc) + self.assertEqual(expected_swift_size, size) + self.assertEqual(expected_checksum, checksum) + self.assertEqual(1, SWIFT_PUT_OBJECT_CALLS) + + loc = location.get_location_from_uri_and_backend( + expected_location, "swift1", conf=self.conf) + (new_image_swift, new_image_size) = self.store.get(loc) + new_image_contents = b''.join([chunk for chunk in new_image_swift]) + new_image_swift_size = len(new_image_swift) + + self.assertEqual(expected_swift_contents, new_image_contents) + self.assertEqual(expected_swift_size, new_image_swift_size) + + def test_add_no_container_no_create(self): + """ + Tests that adding an image with a non-existing container + raises an appropriate exception + """ + conf = copy.deepcopy(SWIFT_CONF) + conf['swift_store_user'] = 'tenant:user' + conf['swift_store_create_container_on_put'] = False + conf['swift_store_container'] = 'noexist' + self.config(group="swift1", **conf) + moves.reload_module(swift) + self.mock_keystone_client() + + self.store = Store(self.conf, backend='swift1') + self.store.configure() + + image_swift = six.BytesIO(b"nevergonnamakeit") + + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + + # We check the exception text to ensure the container + # missing text is found in it, otherwise, we would have + # simply used self.assertRaises here + exception_caught = False + try: + self.store.add(str(uuid.uuid4()), image_swift, 0) + except exceptions.BackendException as e: + exception_caught = True + self.assertIn("container noexist does not exist in Swift", + encodeutils.exception_to_unicode(e)) + self.assertTrue(exception_caught) + self.assertEqual(0, SWIFT_PUT_OBJECT_CALLS) + + @mock.patch('glance_store._drivers.swift.utils' + '.is_multiple_swift_store_accounts_enabled', + mock.Mock(return_value=True)) + def test_add_no_container_and_create(self): + """ + Tests that adding an image with a non-existing container + creates the container automatically if flag is set + """ + expected_swift_size = FIVE_KB + expected_swift_contents = b"*" * expected_swift_size + expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() + expected_image_id = str(uuid.uuid4()) + loc = 'swift+config://ref1/noexist/%s' + expected_location = loc % (expected_image_id) + image_swift = six.BytesIO(expected_swift_contents) + + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + conf = copy.deepcopy(SWIFT_CONF) + conf['swift_store_user'] = 'tenant:user' + conf['swift_store_create_container_on_put'] = True + conf['swift_store_container'] = 'noexist' + self.config(group="swift1", **conf) + moves.reload_module(swift) + self.mock_keystone_client() + self.store = Store(self.conf, backend="swift1") + self.store.configure() + loc, size, checksum, metadata = self.store.add(expected_image_id, + image_swift, + expected_swift_size) + + self.assertEqual("swift1", metadata['backend']) + self.assertEqual(expected_location, loc) + self.assertEqual(expected_swift_size, size) + self.assertEqual(expected_checksum, checksum) + self.assertEqual(1, SWIFT_PUT_OBJECT_CALLS) + + loc = location.get_location_from_uri_and_backend( + expected_location, "swift1", conf=self.conf) + (new_image_swift, new_image_size) = self.store.get(loc) + new_image_contents = b''.join([chunk for chunk in new_image_swift]) + new_image_swift_size = len(new_image_swift) + + self.assertEqual(expected_swift_contents, new_image_contents) + self.assertEqual(expected_swift_size, new_image_swift_size) + + @mock.patch('glance_store._drivers.swift.utils' + '.is_multiple_swift_store_accounts_enabled', + mock.Mock(return_value=True)) + def test_add_no_container_and_multiple_containers_create(self): + """ + Tests that adding an image with a non-existing container while using + multi containers will create the container automatically if flag is set + """ + expected_swift_size = FIVE_KB + expected_swift_contents = b"*" * expected_swift_size + expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() + expected_image_id = str(uuid.uuid4()) + container = 'randomname_' + expected_image_id[:2] + loc = 'swift+config://ref1/%s/%s' + expected_location = loc % (container, expected_image_id) + image_swift = six.BytesIO(expected_swift_contents) + + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + conf = copy.deepcopy(SWIFT_CONF) + conf['swift_store_user'] = 'tenant:user' + conf['swift_store_create_container_on_put'] = True + conf['swift_store_container'] = 'randomname' + conf['swift_store_multiple_containers_seed'] = 2 + self.config(group="swift1", **conf) + moves.reload_module(swift) + self.mock_keystone_client() + + self.store = Store(self.conf, backend="swift1") + self.store.configure() + loc, size, checksum, metadata = self.store.add(expected_image_id, + image_swift, + expected_swift_size) + + self.assertEqual("swift1", metadata['backend']) + self.assertEqual(expected_location, loc) + self.assertEqual(expected_swift_size, size) + self.assertEqual(expected_checksum, checksum) + self.assertEqual(1, SWIFT_PUT_OBJECT_CALLS) + + loc = location.get_location_from_uri_and_backend( + expected_location, "swift1", conf=self.conf) + (new_image_swift, new_image_size) = self.store.get(loc) + new_image_contents = b''.join([chunk for chunk in new_image_swift]) + new_image_swift_size = len(new_image_swift) + + self.assertEqual(expected_swift_contents, new_image_contents) + self.assertEqual(expected_swift_size, new_image_swift_size) + + @mock.patch('glance_store._drivers.swift.utils' + '.is_multiple_swift_store_accounts_enabled', + mock.Mock(return_value=True)) + def test_add_no_container_and_multiple_containers_no_create(self): + """ + Tests that adding an image with a non-existing container while using + multiple containers raises an appropriate exception + """ + conf = copy.deepcopy(SWIFT_CONF) + conf['swift_store_user'] = 'tenant:user' + conf['swift_store_create_container_on_put'] = False + conf['swift_store_container'] = 'randomname' + conf['swift_store_multiple_containers_seed'] = 2 + self.config(group="swift1", **conf) + moves.reload_module(swift) + self.mock_keystone_client() + + expected_image_id = str(uuid.uuid4()) + expected_container = 'randomname_' + expected_image_id[:2] + + self.store = Store(self.conf, backend="swift1") + self.store.configure() + + image_swift = six.BytesIO(b"nevergonnamakeit") + + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + + # We check the exception text to ensure the container + # missing text is found in it, otherwise, we would have + # simply used self.assertRaises here + exception_caught = False + try: + self.store.add(expected_image_id, image_swift, 0) + except exceptions.BackendException as e: + exception_caught = True + expected_msg = "container %s does not exist in Swift" + expected_msg = expected_msg % expected_container + self.assertIn(expected_msg, encodeutils.exception_to_unicode(e)) + self.assertTrue(exception_caught) + self.assertEqual(0, SWIFT_PUT_OBJECT_CALLS) + + @mock.patch('glance_store._drivers.swift.utils' + '.is_multiple_swift_store_accounts_enabled', + mock.Mock(return_value=True)) + def test_add_with_verifier(self): + """Test that the verifier is updated when verifier is provided.""" + swift_size = FIVE_KB + base_byte = b"12345678" + swift_contents = base_byte * (swift_size // 8) + image_id = str(uuid.uuid4()) + image_swift = six.BytesIO(swift_contents) + + self.store = Store(self.conf, backend="swift1") + self.store.configure() + orig_max_size = self.store.large_object_size + orig_temp_size = self.store.large_object_chunk_size + custom_size = units.Ki + verifier = mock.MagicMock(name='mock_verifier') + + try: + self.store.large_object_size = custom_size + self.store.large_object_chunk_size = custom_size + self.store.add(image_id, image_swift, swift_size, + verifier=verifier) + finally: + self.store.large_object_chunk_size = orig_temp_size + self.store.large_object_size = orig_max_size + + # Confirm verifier update called expected number of times + self.assertEqual(2 * swift_size / custom_size, + verifier.update.call_count) + + # define one chunk of the contents + swift_contents_piece = base_byte * (custom_size // 8) + + # confirm all expected calls to update have occurred + calls = [mock.call(swift_contents_piece), + mock.call(b''), + mock.call(swift_contents_piece), + mock.call(b''), + mock.call(swift_contents_piece), + mock.call(b''), + mock.call(swift_contents_piece), + mock.call(b''), + mock.call(swift_contents_piece), + mock.call(b'')] + verifier.update.assert_has_calls(calls) + + @mock.patch('glance_store._drivers.swift.utils' + '.is_multiple_swift_store_accounts_enabled', + mock.Mock(return_value=True)) + def test_add_with_verifier_small(self): + """Test that the verifier is updated for smaller images.""" + swift_size = FIVE_KB + base_byte = b"12345678" + swift_contents = base_byte * (swift_size // 8) + image_id = str(uuid.uuid4()) + image_swift = six.BytesIO(swift_contents) + + self.store = Store(self.conf, backend="swift1") + self.store.configure() + orig_max_size = self.store.large_object_size + orig_temp_size = self.store.large_object_chunk_size + custom_size = 6 * units.Ki + verifier = mock.MagicMock(name='mock_verifier') + + try: + self.store.large_object_size = custom_size + self.store.large_object_chunk_size = custom_size + self.store.add(image_id, image_swift, swift_size, + verifier=verifier) + finally: + self.store.large_object_chunk_size = orig_temp_size + self.store.large_object_size = orig_max_size + + # Confirm verifier update called expected number of times + self.assertEqual(2, verifier.update.call_count) + + # define one chunk of the contents + swift_contents_piece = base_byte * (swift_size // 8) + + # confirm all expected calls to update have occurred + calls = [mock.call(swift_contents_piece), + mock.call(b'')] + verifier.update.assert_has_calls(calls) + + @mock.patch('glance_store._drivers.swift.utils' + '.is_multiple_swift_store_accounts_enabled', + mock.Mock(return_value=False)) + def test_multi_container_doesnt_impact_multi_tenant_add(self): + expected_swift_size = FIVE_KB + expected_swift_contents = b"*" * expected_swift_size + expected_image_id = str(uuid.uuid4()) + expected_container = 'container_' + expected_image_id + loc = 'swift+https://some_endpoint/%s/%s' + expected_location = loc % (expected_container, expected_image_id) + image_swift = six.BytesIO(expected_swift_contents) + + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + + self.config(group="swift1", swift_store_container='container') + self.config(group="swift1", swift_store_create_container_on_put=True) + self.config(group="swift1", swift_store_multiple_containers_seed=2) + service_catalog = [ + { + 'endpoint_links': [], + 'endpoints': [ + { + 'adminURL': 'https://some_admin_endpoint', + 'region': 'RegionOne', + 'internalURL': 'https://some_internal_endpoint', + 'publicURL': 'https://some_endpoint', + }, + ], + 'type': 'object-store', + 'name': 'Object Storage Service', + } + ] + ctxt = mock.MagicMock( + user='user', tenant='tenant', auth_token='123', + service_catalog=service_catalog) + store = swift.MultiTenantStore(self.conf, backend="swift1") + store.configure() + location, size, checksum, metadata = store.add(expected_image_id, + image_swift, + expected_swift_size, + context=ctxt) + + self.assertEqual("swift1", metadata['backend']) + self.assertEqual(expected_location, location) + + @mock.patch('glance_store._drivers.swift.utils' + '.is_multiple_swift_store_accounts_enabled', + mock.Mock(return_value=True)) + def test_add_large_object(self): + """ + Tests that adding a very large image. We simulate the large + object by setting store.large_object_size to a small number + and then verify that there have been a number of calls to + put_object()... + """ + expected_swift_size = FIVE_KB + expected_swift_contents = b"*" * expected_swift_size + expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() + expected_image_id = str(uuid.uuid4()) + loc = 'swift+config://ref1/glance/%s' + expected_location = loc % (expected_image_id) + image_swift = six.BytesIO(expected_swift_contents) + + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + + self.store = Store(self.conf, backend="swift1") + self.store.configure() + orig_max_size = self.store.large_object_size + orig_temp_size = self.store.large_object_chunk_size + try: + self.store.large_object_size = units.Ki + self.store.large_object_chunk_size = units.Ki + loc, size, checksum, metadata = self.store.add(expected_image_id, + image_swift, + expected_swift_size) + finally: + self.store.large_object_chunk_size = orig_temp_size + self.store.large_object_size = orig_max_size + + self.assertEqual("swift1", metadata['backend']) + self.assertEqual(expected_location, loc) + self.assertEqual(expected_swift_size, size) + self.assertEqual(expected_checksum, checksum) + # Expecting 6 objects to be created on Swift -- 5 chunks and 1 + # manifest. + self.assertEqual(6, SWIFT_PUT_OBJECT_CALLS) + + loc = location.get_location_from_uri_and_backend( + expected_location, "swift1", conf=self.conf) + (new_image_swift, new_image_size) = self.store.get(loc) + new_image_contents = b''.join([chunk for chunk in new_image_swift]) + new_image_swift_size = len(new_image_contents) + + self.assertEqual(expected_swift_contents, new_image_contents) + self.assertEqual(expected_swift_size, new_image_swift_size) + + def test_add_large_object_zero_size(self): + """ + Tests that adding an image to Swift which has both an unknown size and + exceeds Swift's maximum limit of 5GB is correctly uploaded. + + We avoid the overhead of creating a 5GB object for this test by + temporarily setting MAX_SWIFT_OBJECT_SIZE to 1KB, and then adding + an object of 5KB. + + Bug lp:891738 + """ + # Set up a 'large' image of 5KB + expected_swift_size = FIVE_KB + expected_swift_contents = b"*" * expected_swift_size + expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() + expected_image_id = str(uuid.uuid4()) + loc = 'swift+config://ref1/glance/%s' + expected_location = loc % (expected_image_id) + image_swift = six.BytesIO(expected_swift_contents) + + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + + # Temporarily set Swift MAX_SWIFT_OBJECT_SIZE to 1KB and add our image, + # explicitly setting the image_length to 0 + + self.store = Store(self.conf, backend="swift1") + self.store.configure() + orig_max_size = self.store.large_object_size + orig_temp_size = self.store.large_object_chunk_size + global MAX_SWIFT_OBJECT_SIZE + orig_max_swift_object_size = MAX_SWIFT_OBJECT_SIZE + try: + MAX_SWIFT_OBJECT_SIZE = units.Ki + self.store.large_object_size = units.Ki + self.store.large_object_chunk_size = units.Ki + loc, size, checksum, metadata = self.store.add(expected_image_id, + image_swift, + 0) + finally: + self.store.large_object_chunk_size = orig_temp_size + self.store.large_object_size = orig_max_size + MAX_SWIFT_OBJECT_SIZE = orig_max_swift_object_size + + self.assertEqual("swift1", metadata['backend']) + self.assertEqual(expected_location, loc) + self.assertEqual(expected_swift_size, size) + self.assertEqual(expected_checksum, checksum) + # Expecting 6 calls to put_object -- 5 chunks, and the manifest. + self.assertEqual(6, SWIFT_PUT_OBJECT_CALLS) + + loc = location.get_location_from_uri_and_backend( + expected_location, "swift1", conf=self.conf) + (new_image_swift, new_image_size) = self.store.get(loc) + new_image_contents = b''.join([chunk for chunk in new_image_swift]) + new_image_swift_size = len(new_image_contents) + + self.assertEqual(expected_swift_contents, new_image_contents) + self.assertEqual(expected_swift_size, new_image_swift_size) + + def test_add_already_existing(self): + """ + Tests that adding an image with an existing identifier + raises an appropriate exception + """ + self.store = Store(self.conf, backend="swift1") + self.store.configure() + image_swift = six.BytesIO(b"nevergonnamakeit") + self.assertRaises(exceptions.Duplicate, + self.store.add, + FAKE_UUID, image_swift, 0) + + def _option_required(self, key): + conf = self.getConfig() + conf[key] = None + + try: + self.config(group="swift1", **conf) + self.store = Store(self.conf, backend="swift1") + return not self.store.is_capable( + capabilities.BitMasks.WRITE_ACCESS) + except Exception: + return False + + def test_no_store_credentials(self): + """ + Tests that options without a valid credentials disables the add method + """ + self.store = Store(self.conf, backend="swift1") + self.store.ref_params = {'ref1': {'auth_address': + 'authurl.com', 'user': '', + 'key': ''}} + self.store.configure() + self.assertFalse(self.store.is_capable( + capabilities.BitMasks.WRITE_ACCESS)) + + def test_no_auth_address(self): + """ + Tests that options without auth address disables the add method + """ + self.store = Store(self.conf, backend="swift1") + self.store.ref_params = {'ref1': {'auth_address': + '', 'user': 'user1', + 'key': 'key1'}} + self.store.configure() + self.assertFalse(self.store.is_capable( + capabilities.BitMasks.WRITE_ACCESS)) + + def test_delete(self): + """ + Test we can delete an existing image in the swift store + """ + conf = copy.deepcopy(SWIFT_CONF) + self.config(group="swift1", **conf) + moves.reload_module(swift) + self.mock_keystone_client() + self.store = Store(self.conf, backend="swift1") + self.store.configure() + + uri = "swift://%s:key@authurl/glance/%s" % ( + self.swift_store_user, FAKE_UUID) + loc = location.get_location_from_uri_and_backend( + uri, "swift1", conf=self.conf) + self.store.delete(loc) + + self.assertRaises(exceptions.NotFound, self.store.get, loc) + + @mock.patch.object(swiftclient.client, 'delete_object') + def test_delete_slo(self, mock_del_obj): + """ + Test we can delete an existing image stored as SLO, static large object + """ + conf = copy.deepcopy(SWIFT_CONF) + self.config(group="swift1", **conf) + moves.reload_module(swift) + self.store = Store(self.conf, backend="swift1") + self.store.configure() + + uri = "swift://%s:key@authurl/glance/%s" % (self.swift_store_user, + FAKE_UUID2) + loc = location.get_location_from_uri_and_backend( + uri, "swift1", conf=self.conf) + self.store.delete(loc) + + self.assertEqual(1, mock_del_obj.call_count) + _, kwargs = mock_del_obj.call_args + self.assertEqual('multipart-manifest=delete', + kwargs.get('query_string')) + + @mock.patch.object(swiftclient.client, 'delete_object') + def test_delete_nonslo_not_deleted_as_slo(self, mock_del_obj): + """ + Test that non-SLOs are not being deleted the SLO way + """ + conf = copy.deepcopy(SWIFT_CONF) + self.config(group="swift1", **conf) + moves.reload_module(swift) + self.mock_keystone_client() + self.store = Store(self.conf, backend="swift1") + self.store.configure() + + uri = "swift://%s:key@authurl/glance/%s" % (self.swift_store_user, + FAKE_UUID) + loc = location.get_location_from_uri_and_backend( + uri, "swift1", conf=self.conf) + self.store.delete(loc) + + self.assertEqual(1, mock_del_obj.call_count) + _, kwargs = mock_del_obj.call_args + self.assertIsNone(kwargs.get('query_string')) + + def test_delete_with_reference_params(self): + """ + Test we can delete an existing image in the swift store + """ + conf = copy.deepcopy(SWIFT_CONF) + self.config(group="swift1", **conf) + moves.reload_module(swift) + # mock client because v3 uses it to receive auth_info + self.mock_keystone_client() + self.store = Store(self.conf, backend="swift1") + self.store.configure() + + uri = "swift+config://ref1/glance/%s" % (FAKE_UUID) + loc = location.get_location_from_uri_and_backend( + uri, "swift1", conf=self.conf) + self.store.delete(loc) + + self.assertRaises(exceptions.NotFound, self.store.get, loc) + + def test_delete_non_existing(self): + """ + Test that trying to delete a swift that doesn't exist + raises an error + """ + conf = copy.deepcopy(SWIFT_CONF) + self.config(group="swift1", **conf) + moves.reload_module(swift) + self.store = Store(self.conf, backend="swift1") + self.store.configure() + + loc = location.get_location_from_uri_and_backend( + "swift://%s:key@authurl/glance/noexist" % (self.swift_store_user), + "swift1", conf=self.conf) + self.assertRaises(exceptions.NotFound, self.store.delete, loc) + + def test_delete_with_some_segments_failing(self): + """ + Tests that delete of a segmented object recovers from error(s) while + deleting one or more segments. + To test this we add a segmented object first and then delete it, while + simulating errors on one or more segments. + """ + + test_image_id = str(uuid.uuid4()) + + def fake_head_object(container, object_name): + object_manifest = '/'.join([container, object_name]) + '-' + return {'x-object-manifest': object_manifest} + + def fake_get_container(container, **kwargs): + # Returning 5 fake segments + return None, [{'name': '%s-%03d' % (test_image_id, x)} + for x in range(1, 6)] + + def fake_delete_object(container, object_name): + # Simulate error on 1st and 3rd segments + global SWIFT_DELETE_OBJECT_CALLS + SWIFT_DELETE_OBJECT_CALLS += 1 + if object_name.endswith('-001') or object_name.endswith('-003'): + raise swiftclient.ClientException('Object DELETE failed') + else: + pass + + conf = copy.deepcopy(SWIFT_CONF) + self.config(group="swift1", **conf) + moves.reload_module(swift) + self.store = Store(self.conf, backend="swift1") + self.store.configure() + + loc_uri = "swift+https://%s:key@localhost:8080/glance/%s" + loc_uri = loc_uri % (self.swift_store_user, test_image_id) + loc = location.get_location_from_uri_and_backend( + loc_uri, "swift1", conf=self.conf) + + conn = self.store.get_connection(loc.store_location) + conn.delete_object = fake_delete_object + conn.head_object = fake_head_object + conn.get_container = fake_get_container + + global SWIFT_DELETE_OBJECT_CALLS + SWIFT_DELETE_OBJECT_CALLS = 0 + + self.store.delete(loc, connection=conn) + # Expecting 6 delete calls, 5 for the segments and 1 for the manifest + self.assertEqual(6, SWIFT_DELETE_OBJECT_CALLS) + + def test_read_acl_public(self): + """ + Test that we can set a public read acl. + """ + self.config(group="swift1", swift_store_config_file=None) + self.config(group="swift1", swift_store_multi_tenant=True) + store = Store(self.conf, backend="swift1") + store.configure() + uri = "swift+http://storeurl/glance/%s" % FAKE_UUID + loc = location.get_location_from_uri_and_backend( + uri, "swift1", conf=self.conf) + ctxt = mock.MagicMock() + store.set_acls(loc, public=True, context=ctxt) + container_headers = swiftclient.client.head_container('x', 'y', + 'glance') + self.assertEqual("*:*", container_headers['X-Container-Read']) + + def test_read_acl_tenants(self): + """ + Test that we can set read acl for tenants. + """ + self.config(group="swift1", swift_store_config_file=None) + self.config(group="swift1", swift_store_multi_tenant=True) + store = Store(self.conf, backend="swift1") + store.configure() + uri = "swift+http://storeurl/glance/%s" % FAKE_UUID + loc = location.get_location_from_uri_and_backend( + uri, "swift1", conf=self.conf) + read_tenants = ['matt', 'mark'] + ctxt = mock.MagicMock() + store.set_acls(loc, read_tenants=read_tenants, context=ctxt) + container_headers = swiftclient.client.head_container('x', 'y', + 'glance') + self.assertEqual('matt:*,mark:*', container_headers[ + 'X-Container-Read']) + + def test_write_acls(self): + """ + Test that we can set write acl for tenants. + """ + self.config(group="swift1", swift_store_config_file=None) + self.config(group="swift1", swift_store_multi_tenant=True) + store = Store(self.conf, backend="swift1") + store.configure() + uri = "swift+http://storeurl/glance/%s" % FAKE_UUID + loc = location.get_location_from_uri_and_backend( + uri, "swift1", conf=self.conf) + read_tenants = ['frank', 'jim'] + ctxt = mock.MagicMock() + store.set_acls(loc, write_tenants=read_tenants, context=ctxt) + container_headers = swiftclient.client.head_container('x', 'y', + 'glance') + self.assertEqual('frank:*,jim:*', container_headers[ + 'X-Container-Write']) + + @mock.patch("glance_store._drivers.swift." + "connection_manager.MultiTenantConnectionManager") + def test_get_connection_manager_multi_tenant(self, manager_class): + manager = mock.MagicMock() + manager_class.return_value = manager + self.config(group="swift1", swift_store_config_file=None) + self.config(group="swift1", swift_store_multi_tenant=True) + store = Store(self.conf, backend="swift1") + store.configure() + loc = mock.MagicMock() + self.assertEqual(store.get_manager(loc), manager) + + @mock.patch("glance_store._drivers.swift." + "connection_manager.SingleTenantConnectionManager") + def test_get_connection_manager_single_tenant(self, manager_class): + manager = mock.MagicMock() + manager_class.return_value = manager + store = Store(self.conf, backend="swift1") + store.configure() + loc = mock.MagicMock() + self.assertEqual(store.get_manager(loc), manager) + + def test_get_connection_manager_failed(self): + store = swift.BaseStore(mock.MagicMock()) + loc = mock.MagicMock() + self.assertRaises(NotImplementedError, store.get_manager, loc) + + def test_init_client_multi_tenant(self): + """Test that keystone client was initialized correctly""" + self._init_client(verify=True, swift_store_multi_tenant=True, + swift_store_config_file=None) + + def test_init_client_multi_tenant_insecure(self): + """ + Test that keystone client was initialized correctly with no + certificate verification. + """ + self._init_client(verify=False, swift_store_multi_tenant=True, + swift_store_auth_insecure=True, + swift_store_config_file=None) + + @mock.patch("glance_store._drivers.swift.store.ks_identity") + @mock.patch("glance_store._drivers.swift.store.ks_session") + @mock.patch("glance_store._drivers.swift.store.ks_client") + def _init_client(self, mock_client, mock_session, mock_identity, verify, + **kwargs): + # initialize store and connection parameters + self.config(group="swift1", **kwargs) + store = Store(self.conf, backend="swift1") + store.configure() + ref_params = sutils.SwiftParams(self.conf, backend="swift1").params + default_ref = getattr(self.conf, "swift1").default_swift_reference + default_swift_reference = ref_params.get(default_ref) + # prepare client and session + trustee_session = mock.MagicMock() + trustor_session = mock.MagicMock() + main_session = mock.MagicMock() + trustee_client = mock.MagicMock() + trustee_client.session.get_user_id.return_value = 'fake_user' + trustor_client = mock.MagicMock() + trustor_client.session.auth.get_auth_ref.return_value = { + 'roles': [{'name': 'fake_role'}] + } + trustor_client.trusts.create.return_value = mock.MagicMock( + id='fake_trust') + main_client = mock.MagicMock() + mock_session.Session.side_effect = [trustor_session, trustee_session, + main_session] + mock_client.Client.side_effect = [trustor_client, trustee_client, + main_client] + # initialize client + ctxt = mock.MagicMock() + client = store.init_client(location=mock.MagicMock(), context=ctxt) + # test trustor usage + mock_identity.V3Token.assert_called_once_with( + auth_url=default_swift_reference.get('auth_address'), + token=ctxt.auth_token, + project_id=ctxt.tenant + ) + mock_session.Session.assert_any_call(auth=mock_identity.V3Token(), + verify=verify) + mock_client.Client.assert_any_call(session=trustor_session) + # test trustee usage and trust creation + tenant_name, user = default_swift_reference.get('user').split(':') + mock_identity.V3Password.assert_any_call( + auth_url=default_swift_reference.get('auth_address'), + username=user, + password=default_swift_reference.get('key'), + project_name=tenant_name, + user_domain_id=default_swift_reference.get('user_domain_id'), + user_domain_name=default_swift_reference.get('user_domain_name'), + project_domain_id=default_swift_reference.get('project_domain_id'), + project_domain_name=default_swift_reference.get( + 'project_domain_name') + ) + mock_session.Session.assert_any_call(auth=mock_identity.V3Password(), + verify=verify) + mock_client.Client.assert_any_call(session=trustee_session) + trustor_client.trusts.create.assert_called_once_with( + trustee_user='fake_user', trustor_user=ctxt.user, + project=ctxt.tenant, impersonation=True, + role_names=['fake_role'] + ) + mock_identity.V3Password.assert_any_call( + auth_url=default_swift_reference.get('auth_address'), + username=user, + password=default_swift_reference.get('key'), + trust_id='fake_trust', + user_domain_id=default_swift_reference.get('user_domain_id'), + user_domain_name=default_swift_reference.get('user_domain_name'), + project_domain_id=default_swift_reference.get('project_domain_id'), + project_domain_name=default_swift_reference.get( + 'project_domain_name') + ) + mock_client.Client.assert_any_call(session=main_session) + self.assertEqual(main_client, client) + + +class TestStoreAuthV1(base.MultiStoreBaseTest, SwiftTests, + test_store_capabilities.TestStoreCapabilitiesChecking): + + # NOTE(flaper87): temporary until we + # can move to a fully-local lib. + # (Swift store's fault) + _CONF = cfg.ConfigOpts() + + def getConfig(self): + conf = SWIFT_CONF.copy() + conf['swift_store_auth_version'] = '1' + conf['swift_store_user'] = 'tenant:user1' + return conf + + def setUp(self): + """Establish a clean test environment.""" + super(TestStoreAuthV1, self).setUp() + enabled_backends = { + "swift1": "swift", + "swift2": "swift", + } + self.conf = self._CONF + self.conf(args=[]) + self.conf.register_opt(cfg.DictOpt('enabled_backends')) + self.config(enabled_backends=enabled_backends) + store.register_store_opts(self.conf) + self.config(default_backend='swift1', group='glance_store') + + # Ensure stores + locations cleared + location.SCHEME_TO_CLS_BACKEND_MAP = {} + + store.create_multi_stores(self.conf) + self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP', + dict()) + self.test_dir = self.useFixture(fixtures.TempDir()).path + + config = self.getConfig() + + conf_file = 'glance-swift.conf' + self.swift_config_file = self.copy_data_file(conf_file, self.test_dir) + config.update({'swift_store_config_file': self.swift_config_file}) + + moxfixture = self.useFixture(moxstubout.MoxStubout()) + self.stubs = moxfixture.stubs + stub_out_swiftclient(self.stubs, config['swift_store_auth_version']) + self.mock_keystone_client() + self.store = Store(self.conf, backend="swift1") + self.config(group="swift1", **config) + self.store.configure() + + self.register_store_backend_schemes(self.store, 'swift', 'swift1') + self.addCleanup(self.conf.reset) + + +class TestStoreAuthV2(TestStoreAuthV1): + + def getConfig(self): + config = super(TestStoreAuthV2, self).getConfig() + config['swift_store_auth_version'] = '2' + config['swift_store_user'] = 'tenant:user1' + return config + + def test_v2_with_no_tenant(self): + uri = "swift://failme:key@auth_address/glance/%s" % (FAKE_UUID) + loc = location.get_location_from_uri_and_backend( + uri, "swift1", conf=self.conf) + self.assertRaises(exceptions.BadStoreUri, + self.store.get, + loc) + + def test_v2_multi_tenant_location(self): + config = self.getConfig() + config['swift_store_multi_tenant'] = True + self.config(group="swift1", **config) + uri = "swift://auth_address/glance/%s" % (FAKE_UUID) + loc = location.get_location_from_uri_and_backend( + uri, "swift1", conf=self.conf) + self.assertEqual('swift', loc.store_name) + + +class TestStoreAuthV3(TestStoreAuthV1): + + def getConfig(self): + config = super(TestStoreAuthV3, self).getConfig() + config['swift_store_auth_version'] = '3' + config['swift_store_user'] = 'tenant:user1' + return config + + @mock.patch("glance_store._drivers.swift.store.ks_identity") + @mock.patch("glance_store._drivers.swift.store.ks_session") + @mock.patch("glance_store._drivers.swift.store.ks_client") + def test_init_client_single_tenant(self, + mock_client, + mock_session, + mock_identity): + """Test that keystone client was initialized correctly""" + # initialize client + store = Store(self.conf, backend="swift1") + store.configure() + uri = "swift://%s:key@auth_address/glance/%s" % ( + self.swift_store_user, FAKE_UUID) + loc = location.get_location_from_uri_and_backend( + uri, "swift1", conf=self.conf) + ctxt = mock.MagicMock() + store.init_client(location=loc.store_location, context=ctxt) + # check that keystone was initialized correctly + tenant = None if store.auth_version == '1' else "tenant" + username = "tenant:user1" if store.auth_version == '1' else "user1" + mock_identity.V3Password.assert_called_once_with( + auth_url=loc.store_location.swift_url + '/', + username=username, password="key", + project_name=tenant, + project_domain_id='default', project_domain_name=None, + user_domain_id='default', user_domain_name=None,) + mock_session.Session.assert_called_once_with( + auth=mock_identity.V3Password(), verify=True) + mock_client.Client.assert_called_once_with( + session=mock_session.Session()) + + +class FakeConnection(object): + def __init__(self, authurl=None, user=None, key=None, retries=5, + preauthurl=None, preauthtoken=None, starting_backoff=1, + tenant_name=None, os_options=None, auth_version="1", + insecure=False, ssl_compression=True, cacert=None): + if os_options is None: + os_options = {} + + self.authurl = authurl + self.user = user + self.key = key + self.preauthurl = preauthurl + self.preauthtoken = preauthtoken + self.tenant_name = tenant_name + self.os_options = os_options + self.auth_version = auth_version + self.insecure = insecure + self.cacert = cacert + + +class TestSingleTenantStoreConnections(base.MultiStoreBaseTest): + + # NOTE(flaper87): temporary until we + # can move to a fully-local lib. + # (Swift store's fault) + _CONF = cfg.ConfigOpts() + + def setUp(self): + super(TestSingleTenantStoreConnections, self).setUp() + enabled_backends = { + "swift1": "swift", + "swift2": "swift", + } + self.conf = self._CONF + self.conf(args=[]) + self.conf.register_opt(cfg.DictOpt('enabled_backends')) + self.config(enabled_backends=enabled_backends) + store.register_store_opts(self.conf) + self.config(default_backend='swift1', group='glance_store') + + # Ensure stores + locations cleared + location.SCHEME_TO_CLS_BACKEND_MAP = {} + + store.create_multi_stores(self.conf) + self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP', + dict()) + self.test_dir = self.useFixture(fixtures.TempDir()).path + + moxfixture = self.useFixture(moxstubout.MoxStubout()) + self.stubs = moxfixture.stubs + self.stubs.Set(swiftclient, 'Connection', FakeConnection) + self.store = swift.SingleTenantStore(self.conf, backend="swift1") + self.store.configure() + specs = {'scheme': 'swift', + 'auth_or_store_url': 'example.com/v2/', + 'user': 'tenant:user1', + 'key': 'key1', + 'container': 'cont', + 'obj': 'object'} + self.location = swift.StoreLocation(specs, self.conf, + backend_group="swift1") + + self.register_store_backend_schemes(self.store, 'swift', 'swift1') + self.addCleanup(self.conf.reset) + + def test_basic_connection(self): + connection = self.store.get_connection(self.location) + self.assertEqual('https://example.com/v2/', connection.authurl) + self.assertEqual('2', connection.auth_version) + self.assertEqual('user1', connection.user) + self.assertEqual('tenant', connection.tenant_name) + self.assertEqual('key1', connection.key) + self.assertIsNone(connection.preauthurl) + self.assertFalse(connection.insecure) + self.assertEqual({'service_type': 'object-store', + 'endpoint_type': 'publicURL'}, + connection.os_options) + + def test_connection_with_conf_endpoint(self): + ctx = mock.MagicMock(user='tenant:user1', tenant='tenant') + self.config(group="swift1", + swift_store_endpoint='https://internal.com') + self.store.configure() + connection = self.store.get_connection(self.location, context=ctx) + self.assertEqual('https://example.com/v2/', connection.authurl) + self.assertEqual('2', connection.auth_version) + self.assertEqual('user1', connection.user) + self.assertEqual('tenant', connection.tenant_name) + self.assertEqual('key1', connection.key) + self.assertEqual('https://internal.com', connection.preauthurl) + self.assertFalse(connection.insecure) + self.assertEqual({'service_type': 'object-store', + 'endpoint_type': 'publicURL'}, + connection.os_options) + + def test_connection_with_conf_endpoint_no_context(self): + self.config(group="swift1", + swift_store_endpoint='https://internal.com') + self.store.configure() + connection = self.store.get_connection(self.location) + self.assertEqual('https://example.com/v2/', connection.authurl) + self.assertEqual('2', connection.auth_version) + self.assertEqual('user1', connection.user) + self.assertEqual('tenant', connection.tenant_name) + self.assertEqual('key1', connection.key) + self.assertEqual('https://internal.com', connection.preauthurl) + self.assertFalse(connection.insecure) + self.assertEqual({'service_type': 'object-store', + 'endpoint_type': 'publicURL'}, + connection.os_options) + + def test_connection_with_no_trailing_slash(self): + self.location.auth_or_store_url = 'example.com/v2' + connection = self.store.get_connection(self.location) + self.assertEqual('https://example.com/v2/', connection.authurl) + + def test_connection_insecure(self): + self.config(group="swift1", swift_store_auth_insecure=True) + self.store.configure() + connection = self.store.get_connection(self.location) + self.assertTrue(connection.insecure) + + def test_connection_with_auth_v1(self): + self.config(group="swift1", swift_store_auth_version='1') + self.store.configure() + self.location.user = 'auth_v1_user' + connection = self.store.get_connection(self.location) + self.assertEqual('1', connection.auth_version) + self.assertEqual('auth_v1_user', connection.user) + self.assertIsNone(connection.tenant_name) + + def test_connection_invalid_user(self): + self.store.configure() + self.location.user = 'invalid:format:user' + self.assertRaises(exceptions.BadStoreUri, + self.store.get_connection, self.location) + + def test_connection_missing_user(self): + self.store.configure() + self.location.user = None + self.assertRaises(exceptions.BadStoreUri, + self.store.get_connection, self.location) + + def test_connection_with_region(self): + self.config(group="swift1", swift_store_region='Sahara') + self.store.configure() + connection = self.store.get_connection(self.location) + self.assertEqual({'region_name': 'Sahara', + 'service_type': 'object-store', + 'endpoint_type': 'publicURL'}, + connection.os_options) + + def test_connection_with_service_type(self): + self.config(group="swift1", swift_store_service_type='shoe-store') + self.store.configure() + connection = self.store.get_connection(self.location) + self.assertEqual({'service_type': 'shoe-store', + 'endpoint_type': 'publicURL'}, + connection.os_options) + + def test_connection_with_endpoint_type(self): + self.config(group="swift1", swift_store_endpoint_type='internalURL') + self.store.configure() + connection = self.store.get_connection(self.location) + self.assertEqual({'service_type': 'object-store', + 'endpoint_type': 'internalURL'}, + connection.os_options) + + def test_bad_location_uri(self): + self.store.configure() + self.location.uri = 'http://bad_uri://' + self.assertRaises(exceptions.BadStoreUri, + self.location.parse_uri, + self.location.uri) + + def test_bad_location_uri_invalid_credentials(self): + self.store.configure() + self.location.uri = 'swift://bad_creds@uri/cont/obj' + self.assertRaises(exceptions.BadStoreUri, + self.location.parse_uri, + self.location.uri) + + def test_bad_location_uri_invalid_object_path(self): + self.store.configure() + self.location.uri = 'swift://user:key@uri/cont' + self.assertRaises(exceptions.BadStoreUri, + self.location.parse_uri, + self.location.uri) + + def test_ref_overrides_defaults(self): + self.config(group="swift1", swift_store_auth_version='2', + swift_store_user='testuser', + swift_store_key='testpass', + swift_store_auth_address='testaddress', + swift_store_endpoint_type='internalURL', + swift_store_config_file='somefile') + + self.store.ref_params = {'ref1': {'auth_address': 'authurl.com', + 'auth_version': '3', + 'user': 'user:pass', + 'user_domain_id': 'default', + 'user_domain_name': 'ignored', + 'project_domain_id': 'default', + 'project_domain_name': 'ignored'}} + + self.store.configure() + + self.assertEqual('user:pass', self.store.user) + self.assertEqual('3', self.store.auth_version) + self.assertEqual('authurl.com', self.store.auth_address) + self.assertEqual('default', self.store.user_domain_id) + self.assertEqual('ignored', self.store.user_domain_name) + self.assertEqual('default', self.store.project_domain_id) + self.assertEqual('ignored', self.store.project_domain_name) + + def test_with_v3_auth(self): + self.store.ref_params = {'ref1': {'auth_address': 'authurl.com', + 'auth_version': '3', + 'user': 'user:pass', + 'key': 'password', + 'user_domain_id': 'default', + 'user_domain_name': 'ignored', + 'project_domain_id': 'default', + 'project_domain_name': 'ignored'}} + self.store.configure() + connection = self.store.get_connection(self.location) + self.assertEqual('3', connection.auth_version) + self.assertEqual({'service_type': 'object-store', + 'endpoint_type': 'publicURL', + 'user_domain_id': 'default', + 'user_domain_name': 'ignored', + 'project_domain_id': 'default', + 'project_domain_name': 'ignored'}, + connection.os_options) + + +class TestMultiTenantStoreConnections(base.MultiStoreBaseTest): + # NOTE(flaper87): temporary until we + # can move to a fully-local lib. + # (Swift store's fault) + _CONF = cfg.ConfigOpts() + + def setUp(self): + super(TestMultiTenantStoreConnections, self).setUp() + enabled_backends = { + "swift1": "swift", + "swift2": "swift", + } + self.conf = self._CONF + self.conf(args=[]) + self.conf.register_opt(cfg.DictOpt('enabled_backends')) + self.config(enabled_backends=enabled_backends) + store.register_store_opts(self.conf) + self.config(default_backend='swift1', group='glance_store') + + # Ensure stores + locations cleared + location.SCHEME_TO_CLS_BACKEND_MAP = {} + + store.create_multi_stores(self.conf) + self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP', + dict()) + self.test_dir = self.useFixture(fixtures.TempDir()).path + + moxfixture = self.useFixture(moxstubout.MoxStubout()) + self.stubs = moxfixture.stubs + self.stubs.Set(swiftclient, 'Connection', FakeConnection) + self.context = mock.MagicMock( + user='tenant:user1', tenant='tenant', auth_token='0123') + self.store = swift.MultiTenantStore(self.conf, backend="swift1") + specs = {'scheme': 'swift', + 'auth_or_store_url': 'example.com', + 'container': 'cont', + 'obj': 'object'} + self.location = swift.StoreLocation(specs, self.conf, + backend_group="swift1") + self.addCleanup(self.conf.reset) + + def test_basic_connection(self): + self.store.configure() + connection = self.store.get_connection(self.location, + context=self.context) + self.assertIsNone(connection.authurl) + self.assertEqual('1', connection.auth_version) + self.assertIsNone(connection.user) + self.assertIsNone(connection.tenant_name) + self.assertIsNone(connection.key) + self.assertEqual('https://example.com', connection.preauthurl) + self.assertEqual('0123', connection.preauthtoken) + self.assertEqual({}, connection.os_options) + + def test_connection_does_not_use_endpoint_from_catalog(self): + self.store.configure() + self.context.service_catalog = [ + { + 'endpoint_links': [], + 'endpoints': [ + { + 'region': 'RegionOne', + 'publicURL': 'https://scexample.com', + }, + ], + 'type': 'object-store', + 'name': 'Object Storage Service', + } + ] + connection = self.store.get_connection(self.location, + context=self.context) + self.assertIsNone(connection.authurl) + self.assertEqual('1', connection.auth_version) + self.assertIsNone(connection.user) + self.assertIsNone(connection.tenant_name) + self.assertIsNone(connection.key) + self.assertNotEqual('https://scexample.com', connection.preauthurl) + self.assertEqual('https://example.com', connection.preauthurl) + self.assertEqual('0123', connection.preauthtoken) + self.assertEqual({}, connection.os_options) + + def test_connection_manager_does_not_use_endpoint_from_catalog(self): + self.store.configure() + self.context.service_catalog = [ + { + 'endpoint_links': [], + 'endpoints': [ + { + 'region': 'RegionOne', + 'publicURL': 'https://scexample.com', + }, + ], + 'type': 'object-store', + 'name': 'Object Storage Service', + } + ] + connection_manager = manager.MultiTenantConnectionManager( + store=self.store, + store_location=self.location, + context=self.context + ) + conn = connection_manager._init_connection() + self.assertNotEqual('https://scexample.com', conn.preauthurl) + self.assertEqual('https://example.com', conn.preauthurl) + + +class TestMultiTenantStoreContext(base.MultiStoreBaseTest): + + # NOTE(flaper87): temporary until we + # can move to a fully-local lib. + # (Swift store's fault) + _CONF = cfg.ConfigOpts() + + def setUp(self): + """Establish a clean test environment.""" + super(TestMultiTenantStoreContext, self).setUp() + config = SWIFT_CONF.copy() + + enabled_backends = { + "swift1": "swift", + "swift2": "swift", + } + self.conf = self._CONF + self.conf(args=[]) + self.conf.register_opt(cfg.DictOpt('enabled_backends')) + self.config(enabled_backends=enabled_backends) + store.register_store_opts(self.conf) + self.config(default_backend='swift1', group='glance_store') + + # Ensure stores + locations cleared + location.SCHEME_TO_CLS_BACKEND_MAP = {} + + store.create_multi_stores(self.conf) + self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP', + dict()) + self.test_dir = self.useFixture(fixtures.TempDir()).path + + self.store = Store(self.conf, backend="swift1") + self.config(group="swift1", **config) + self.store.configure() + self.register_store_backend_schemes(self.store, 'swift', 'swift1') + + service_catalog = [ + { + 'endpoint_links': [], + 'endpoints': [ + { + 'region': 'RegionOne', + 'publicURL': 'http://127.0.0.1:0', + }, + ], + 'type': 'object-store', + 'name': 'Object Storage Service', + } + ] + self.ctx = mock.MagicMock( + service_catalog=service_catalog, user='tenant:user1', + tenant='tenant', auth_token='0123') + self.addCleanup(self.conf.reset) + + @requests_mock.mock() + def test_download_context(self, m): + """Verify context (ie token) is passed to swift on download.""" + self.config(group="swift1", swift_store_multi_tenant=True) + store = Store(self.conf, backend="swift1") + store.configure() + uri = "swift+http://127.0.0.1/glance_123/123" + loc = location.get_location_from_uri_and_backend( + uri, "swift1", conf=self.conf) + m.get("http://127.0.0.1/glance_123/123", + headers={'Content-Length': '0'}) + store.get(loc, context=self.ctx) + self.assertEqual(b'0123', m.last_request.headers['X-Auth-Token']) + + @requests_mock.mock() + def test_upload_context(self, m): + """Verify context (ie token) is passed to swift on upload.""" + head_req = m.head("http://127.0.0.1/glance_123", + text='Some data', + status_code=201) + put_req = m.put("http://127.0.0.1/glance_123/123") + + self.config(group="swift1", swift_store_multi_tenant=True) + store = Store(self.conf, backend="swift1") + store.configure() + content = b'Some data' + pseudo_file = six.BytesIO(content) + store.add('123', pseudo_file, len(content), + context=self.ctx) + self.assertEqual(b'0123', + head_req.last_request.headers['X-Auth-Token']) + self.assertEqual(b'0123', + put_req.last_request.headers['X-Auth-Token']) + + +class TestCreatingLocations(base.MultiStoreBaseTest): + + # NOTE(flaper87): temporary until we + # can move to a fully-local lib. + # (Swift store's fault) + _CONF = cfg.ConfigOpts() + + def setUp(self): + super(TestCreatingLocations, self).setUp() + enabled_backends = { + "swift1": "swift", + "swift2": "swift", + } + self.conf = self._CONF + self.conf(args=[]) + self.conf.register_opt(cfg.DictOpt('enabled_backends')) + self.config(enabled_backends=enabled_backends) + store.register_store_opts(self.conf) + self.config(default_backend='swift1', group='glance_store') + + # Ensure stores + locations cleared + location.SCHEME_TO_CLS_BACKEND_MAP = {} + + store.create_multi_stores(self.conf) + self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP', + dict()) + self.test_dir = self.useFixture(fixtures.TempDir()).path + + moxfixture = self.useFixture(moxstubout.MoxStubout()) + self.stubs = moxfixture.stubs + config = copy.deepcopy(SWIFT_CONF) + self.store = Store(self.conf, backend="swift1") + self.config(group="swift1", **config) + self.store.configure() + self.register_store_backend_schemes(self.store, 'swift', 'swift1') + + moves.reload_module(swift) + self.addCleanup(self.conf.reset) + + service_catalog = [ + { + 'endpoint_links': [], + 'endpoints': [ + { + 'adminURL': 'https://some_admin_endpoint', + 'region': 'RegionOne', + 'internalURL': 'https://some_internal_endpoint', + 'publicURL': 'https://some_endpoint', + }, + ], + 'type': 'object-store', + 'name': 'Object Storage Service', + } + ] + self.ctxt = mock.MagicMock(user='user', tenant='tenant', + auth_token='123', + service_catalog=service_catalog) + + def test_single_tenant_location(self): + conf = copy.deepcopy(SWIFT_CONF) + conf['swift_store_container'] = 'container' + conf_file = "glance-swift.conf" + self.swift_config_file = self.copy_data_file(conf_file, self.test_dir) + conf.update({'swift_store_config_file': self.swift_config_file}) + conf['default_swift_reference'] = 'ref1' + self.config(group="swift1", **conf) + moves.reload_module(swift) + + store = swift.SingleTenantStore(self.conf, backend="swift1") + store.configure() + location = store.create_location('image-id') + self.assertEqual('swift+https', location.scheme) + self.assertEqual('https://example.com', location.swift_url) + self.assertEqual('container', location.container) + self.assertEqual('image-id', location.obj) + self.assertEqual('tenant:user1', location.user) + self.assertEqual('key1', location.key) + + def test_single_tenant_location_http(self): + conf_file = "glance-swift.conf" + test_dir = self.useFixture(fixtures.TempDir()).path + self.swift_config_file = self.copy_data_file(conf_file, test_dir) + self.config(group="swift1", swift_store_container='container', + default_swift_reference='ref2', + swift_store_config_file=self.swift_config_file) + + store = swift.SingleTenantStore(self.conf, backend="swift1") + store.configure() + location = store.create_location('image-id') + self.assertEqual('swift+http', location.scheme) + self.assertEqual('http://example.com', location.swift_url) + + def test_multi_tenant_location(self): + self.config(group="swift1", swift_store_container='container') + store = swift.MultiTenantStore(self.conf, backend="swift1") + store.configure() + location = store.create_location('image-id', context=self.ctxt) + self.assertEqual('swift+https', location.scheme) + self.assertEqual('https://some_endpoint', location.swift_url) + self.assertEqual('container_image-id', location.container) + self.assertEqual('image-id', location.obj) + self.assertIsNone(location.user) + self.assertIsNone(location.key) + + def test_multi_tenant_location_http(self): + store = swift.MultiTenantStore(self.conf, backend="swift1") + store.configure() + self.ctxt.service_catalog[0]['endpoints'][0]['publicURL'] = \ + 'http://some_endpoint' + location = store.create_location('image-id', context=self.ctxt) + self.assertEqual('swift+http', location.scheme) + self.assertEqual('http://some_endpoint', location.swift_url) + + def test_multi_tenant_location_with_region(self): + self.config(group="swift1", swift_store_region='WestCarolina') + store = swift.MultiTenantStore(self.conf, backend="swift1") + store.configure() + self.ctxt.service_catalog[0]['endpoints'][0]['region'] = 'WestCarolina' + self.assertEqual('https://some_endpoint', + store._get_endpoint(self.ctxt)) + + def test_multi_tenant_location_custom_service_type(self): + self.config(group="swift1", swift_store_service_type='toy-store') + self.ctxt.service_catalog[0]['type'] = 'toy-store' + store = swift.MultiTenantStore(self.conf, backend="swift1") + store.configure() + store._get_endpoint(self.ctxt) + self.assertEqual('https://some_endpoint', + store._get_endpoint(self.ctxt)) + + def test_multi_tenant_location_custom_endpoint_type(self): + self.config(group="swift1", swift_store_endpoint_type='internalURL') + store = swift.MultiTenantStore(self.conf, backend="swift1") + store.configure() + self.assertEqual('https://some_internal_endpoint', + store._get_endpoint(self.ctxt)) + + +class TestChunkReader(base.MultiStoreBaseTest): + + # NOTE(flaper87): temporary until we + # can move to a fully-local lib. + # (Swift store's fault) + _CONF = cfg.ConfigOpts() + + def setUp(self): + super(TestChunkReader, self).setUp() + enabled_backends = { + "swift1": "swift", + "swift2": "swift", + } + self.conf = self._CONF + self.conf(args=[]) + self.conf.register_opt(cfg.DictOpt('enabled_backends')) + self.config(enabled_backends=enabled_backends) + store.register_store_opts(self.conf) + self.config(default_backend='swift1', group='glance_store') + + # Ensure stores + locations cleared + location.SCHEME_TO_CLS_BACKEND_MAP = {} + + store.create_multi_stores(self.conf) + self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP', + dict()) + self.test_dir = self.useFixture(fixtures.TempDir()).path + + config = copy.deepcopy(SWIFT_CONF) + self.store = Store(self.conf, backend="swift1") + self.config(group="swift1", **config) + self.store.configure() + self.register_store_backend_schemes(self.store, 'swift', 'swift1') + + self.addCleanup(self.conf.reset) + + def test_read_all_data(self): + """ + Replicate what goes on in the Swift driver with the + repeated creation of the ChunkReader object + """ + CHUNKSIZE = 100 + checksum = hashlib.md5() + data_file = tempfile.NamedTemporaryFile() + data_file.write(b'*' * units.Ki) + data_file.flush() + infile = open(data_file.name, 'rb') + bytes_read = 0 + while True: + cr = swift.ChunkReader(infile, checksum, CHUNKSIZE) + chunk = cr.read(CHUNKSIZE) + if len(chunk) == 0: + self.assertEqual(True, cr.is_zero_size) + break + bytes_read += len(chunk) + self.assertEqual(units.Ki, bytes_read) + self.assertEqual('fb10c6486390bec8414be90a93dfff3b', + cr.checksum.hexdigest()) + data_file.close() + infile.close() + + def test_read_zero_size_data(self): + """ + Replicate what goes on in the Swift driver with the + repeated creation of the ChunkReader object + """ + CHUNKSIZE = 100 + checksum = hashlib.md5() + data_file = tempfile.NamedTemporaryFile() + infile = open(data_file.name, 'rb') + bytes_read = 0 + while True: + cr = swift.ChunkReader(infile, checksum, CHUNKSIZE) + chunk = cr.read(CHUNKSIZE) + if len(chunk) == 0: + break + bytes_read += len(chunk) + self.assertEqual(True, cr.is_zero_size) + self.assertEqual(0, bytes_read) + self.assertEqual('d41d8cd98f00b204e9800998ecf8427e', + cr.checksum.hexdigest()) + data_file.close() + infile.close() + + +class TestMultipleContainers(base.MultiStoreBaseTest): + + # NOTE(flaper87): temporary until we + # can move to a fully-local lib. + # (Swift store's fault) + _CONF = cfg.ConfigOpts() + + def setUp(self): + super(TestMultipleContainers, self).setUp() + + enabled_backends = { + "swift1": "swift", + "swift2": "swift", + } + self.conf = self._CONF + self.conf(args=[]) + self.conf.register_opt(cfg.DictOpt('enabled_backends')) + self.config(enabled_backends=enabled_backends) + store.register_store_opts(self.conf) + self.config(default_backend='swift1', group='glance_store') + + # Ensure stores + locations cleared + location.SCHEME_TO_CLS_BACKEND_MAP = {} + + store.create_multi_stores(self.conf) + self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP', + dict()) + self.test_dir = self.useFixture(fixtures.TempDir()).path + + self.config(group="swift1", swift_store_multiple_containers_seed=3) + self.store = swift.SingleTenantStore(self.conf, backend="swift1") + self.store.configure() + self.register_store_backend_schemes(self.store, 'swift', 'swift1') + + self.addCleanup(self.conf.reset) + + def test_get_container_name_happy_path_with_seed_three(self): + + test_image_id = 'fdae39a1-bac5-4238-aba4-69bcc726e848' + actual = self.store.get_container_name(test_image_id, + 'default_container') + expected = 'default_container_fda' + self.assertEqual(expected, actual) + + def test_get_container_name_with_negative_seed(self): + self.assertRaises(ValueError, self.config, + group="swift1", + swift_store_multiple_containers_seed=-1) + + def test_get_container_name_with_seed_beyond_max(self): + self.assertRaises(ValueError, self.config, + group="swift1", + swift_store_multiple_containers_seed=33) + + def test_get_container_name_with_max_seed(self): + self.config(group="swift1", swift_store_multiple_containers_seed=32) + self.store = swift.SingleTenantStore( + self.conf, backend="swift1") + + test_image_id = 'fdae39a1-bac5-4238-aba4-69bcc726e848' + actual = self.store.get_container_name(test_image_id, + 'default_container') + expected = 'default_container_' + test_image_id + self.assertEqual(expected, actual) + + def test_get_container_name_with_dash(self): + self.config(group="swift1", swift_store_multiple_containers_seed=10) + self.store = swift.SingleTenantStore( + self.conf, backend="swift1") + + test_image_id = 'fdae39a1-bac5-4238-aba4-69bcc726e848' + actual = self.store.get_container_name(test_image_id, + 'default_container') + expected = 'default_container_' + 'fdae39a1-ba' + self.assertEqual(expected, actual) + + def test_get_container_name_with_min_seed(self): + self.config(group="swift1", swift_store_multiple_containers_seed=1) + self.store = swift.SingleTenantStore( + self.conf, backend="swift1") + + test_image_id = 'fdae39a1-bac5-4238-aba4-69bcc726e848' + actual = self.store.get_container_name(test_image_id, + 'default_container') + expected = 'default_container_' + 'f' + self.assertEqual(expected, actual) + + def test_get_container_name_with_multiple_containers_turned_off(self): + self.config(group="swift1", swift_store_multiple_containers_seed=0) + self.store.configure() + + test_image_id = 'random_id' + actual = self.store.get_container_name(test_image_id, + 'default_container') + expected = 'default_container' + self.assertEqual(expected, actual)