diff --git a/tests/fake_openstack.py b/tests/fake_openstack.py index f9032a8c25..871bfc6d94 100644 --- a/tests/fake_openstack.py +++ b/tests/fake_openstack.py @@ -109,12 +109,19 @@ class FakeOpenstackSession: return FakeOpenstackResponse({'servers': server_list}) +class FakeOpenstackConfig: + pass + + class FakeOpenstackConnection: log = logging.getLogger("zuul.FakeOpenstackConnection") def __init__(self, cloud): self.cloud = cloud self.compute = FakeOpenstackSession(cloud) + self.config = FakeOpenstackConfig() + self.config.config = {} + self.config.config['image_format'] = 'qcow2' def list_flavors(self, get_extra=False): return self.cloud.flavors diff --git a/tests/unit/test_openstack_driver.py b/tests/unit/test_openstack_driver.py index eb6288314e..a057923500 100644 --- a/tests/unit/test_openstack_driver.py +++ b/tests/unit/test_openstack_driver.py @@ -46,7 +46,7 @@ class TestOpenstackDriver(ZuulTestCase): 'metadata': { 'type': 'zuul_image', 'image_name': 'debian-local', - 'format': 'raw', + 'format': 'qcow2', 'sha256': ('59984dd82f51edb3777b969739a92780' 'a520bb314b8d64b294d5de976bd8efb9'), 'md5sum': '262278e1632567a907e4604e9edd2e83', @@ -146,7 +146,7 @@ class TestOpenstackDriver(ZuulTestCase): artifacts = self.launcher.image_build_registry.\ getArtifactsForImage(name) self.assertEqual(1, len(artifacts)) - self.assertEqual('raw', artifacts[0].format) + self.assertEqual('qcow2', artifacts[0].format) self.assertTrue(artifacts[0].validated) uploads = self.launcher.image_upload_registry.getUploadsForImage( name) diff --git a/zuul/driver/aws/awsprovider.py b/zuul/driver/aws/awsprovider.py index c5efe8ee04..a3ad36d740 100644 --- a/zuul/driver/aws/awsprovider.py +++ b/zuul/driver/aws/awsprovider.py @@ -183,13 +183,13 @@ class AwsProvider(BaseProvider, subclass_id='aws'): self._set(_endpoint=self.getEndpoint()) return self._endpoint - def parseImage(self, image_config, provider_config): + def parseImage(self, image_config, provider_config, connection): return AwsProviderImage(image_config, provider_config) - def parseFlavor(self, flavor_config, provider_config): + def parseFlavor(self, flavor_config, provider_config, connection): return AwsProviderFlavor(flavor_config, provider_config) - def parseLabel(self, label_config, provider_config): + def parseLabel(self, label_config, provider_config, connection): return AwsProviderLabel(label_config, provider_config) def getEndpoint(self): diff --git a/zuul/driver/openstack/__init__.py b/zuul/driver/openstack/__init__.py index 7a84452a65..c2b02fa4ec 100644 --- a/zuul/driver/openstack/__init__.py +++ b/zuul/driver/openstack/__init__.py @@ -48,21 +48,23 @@ class OpenstackDriver(Driver, ConnectionInterface, ProviderInterface): def getProviderNodeClass(self): return openstackmodel.OpenstackProviderNode - def getEndpoint(self, provider): - region = provider.region or '' + def _getEndpoint(self, connection, region): + region_str = region or '' endpoint_id = '/'.join([ - urllib.parse.quote_plus(provider.connection.connection_name), - urllib.parse.quote_plus(region), + urllib.parse.quote_plus(connection.connection_name), + urllib.parse.quote_plus(region_str), ]) try: return self.endpoints[endpoint_id] except KeyError: pass - endpoint = self._endpoint_class( - self, provider.connection, provider.region) + endpoint = self._endpoint_class(self, connection, region) self.endpoints[endpoint_id] = endpoint return endpoint + def getEndpoint(self, provider): + return self._getEndpoint(provider.connection, provider.region) + def stop(self): for endpoint in self.endpoints.values(): endpoint.stop() diff --git a/zuul/driver/openstack/openstackendpoint.py b/zuul/driver/openstack/openstackendpoint.py index e3bbdfb93d..7c7734e16d 100644 --- a/zuul/driver/openstack/openstackendpoint.py +++ b/zuul/driver/openstack/openstackendpoint.py @@ -576,6 +576,9 @@ class OpenstackProviderEndpoint(BaseProviderEndpoint): rate_limit=self.connection.rate, ) + def getImageFormat(self): + return self._client.config.config['image_format'] + def _submitApi(self, api, *args, **kw): return self.api_executor.submit( api, *args, **kw) diff --git a/zuul/driver/openstack/openstackprovider.py b/zuul/driver/openstack/openstackprovider.py index ab2c71677c..ee8da704a2 100644 --- a/zuul/driver/openstack/openstackprovider.py +++ b/zuul/driver/openstack/openstackprovider.py @@ -80,10 +80,11 @@ class OpenstackProviderImage(BaseProviderImage): discriminant=discriminate( lambda val, alt: val['type'] == alt['type'])) - def __init__(self, image_config, provider_config): + def __init__(self, image_config, provider_config, image_format): self.image_id = None self.image_filters = None super().__init__(image_config, provider_config) + self.format = image_format class OpenstackProviderFlavor(BaseProviderFlavor): @@ -167,13 +168,19 @@ class OpenstackProvider(BaseProvider, subclass_id='openstack'): self._set(_endpoint=self.getEndpoint()) return self._endpoint - def parseImage(self, image_config, provider_config): - return OpenstackProviderImage(image_config, provider_config) + def parseImage(self, image_config, provider_config, connection): + # We are not fully constructed yet at this point, so we need + # to peek to get the region and endpoint. + region = provider_config.get('region') + endpoint = connection.driver._getEndpoint(connection, region) + return OpenstackProviderImage( + image_config, provider_config, + image_format=endpoint.getImageFormat()) - def parseFlavor(self, flavor_config, provider_config): + def parseFlavor(self, flavor_config, provider_config, connection): return OpenstackProviderFlavor(flavor_config, provider_config) - def parseLabel(self, label_config, provider_config): + def parseLabel(self, label_config, provider_config, connection): return OpenstackProviderLabel(label_config, provider_config) def getEndpoint(self): diff --git a/zuul/model.py b/zuul/model.py index 0f512750a7..9dbad38d77 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -382,7 +382,7 @@ class ConfigurationErrorList(zkobject.ShardedZKObject): } return json.dumps(data, sort_keys=True).encode("utf8") - def deserialize(self, raw, context): + def deserialize(self, raw, context, extra=None): data = super().deserialize(raw, context) data.update({ "errors": [ConfigurationError.deserialize(d) @@ -869,7 +869,7 @@ class PipelineState(zkobject.ZKObject): self._set(**self._lateInitData()) self.internalCreate(context) - def deserialize(self, raw, context): + def deserialize(self, raw, context, extra=None): # We may have old change objects in the pipeline cache, so # make sure they are the same objects we would get from the # source change cache. @@ -1081,7 +1081,7 @@ class PipelineChangeList(zkobject.ShardedZKObject): } return json.dumps(data, sort_keys=True).encode("utf8") - def deserialize(self, raw, context): + def deserialize(self, raw, context, extra=None): data = super().deserialize(raw, context) change_keys = [] # We must have a dictionary with a 'changes' key; otherwise we @@ -1200,7 +1200,7 @@ class ChangeQueue(zkobject.ZKObject): } return json.dumps(data, sort_keys=True).encode("utf8") - def deserialize(self, raw, context): + def deserialize(self, raw, context, extra=None): data = super().deserialize(raw, context) existing_items = {} @@ -3149,7 +3149,7 @@ class FrozenJob(zkobject.ZKObject): # Use json_dumps to strip any ZuulMark entries return json_dumps(data, sort_keys=True).encode("utf8") - def deserialize(self, raw, context): + def deserialize(self, raw, context, extra=None): # Ensure that any special handling in this method is matched # in Job.freezeJob so that FrozenJobs are identical regardless # of whether they have been deserialized. @@ -4976,7 +4976,7 @@ class Build(zkobject.ZKObject): return json.dumps(data, sort_keys=True).encode("utf8") - def deserialize(self, raw, context): + def deserialize(self, raw, context, extra=None): data = super().deserialize(raw, context) # Deserialize build events @@ -5425,7 +5425,7 @@ class BuildSet(zkobject.ZKObject): } return json.dumps(data, sort_keys=True).encode("utf8") - def deserialize(self, raw, context): + def deserialize(self, raw, context, extra=None): data = super().deserialize(raw, context) # Set our UUID so that getPath() returns the correct path for # child objects. @@ -5907,7 +5907,7 @@ class QueueItem(zkobject.ZKObject): } return json.dumps(data, sort_keys=True).encode("utf8") - def deserialize(self, raw, context): + def deserialize(self, raw, context, extra=None): data = super().deserialize(raw, context) # Set our UUID so that getPath() returns the correct path for # child objects. diff --git a/zuul/provider/__init__.py b/zuul/provider/__init__.py index d90387a7c1..f007f8e013 100644 --- a/zuul/provider/__init__.py +++ b/zuul/provider/__init__.py @@ -148,7 +148,7 @@ class BaseProvider(zkobject.PolymorphicZKObjectMixin, config = config.copy() config.pop('_source_context') config.pop('_start_mark') - parsed_config = self.parseConfig(config) + parsed_config = self.parseConfig(config, connection) parsed_config.pop('connection') self._set( driver=driver, @@ -177,7 +177,8 @@ class BaseProvider(zkobject.PolymorphicZKObjectMixin, """ raw_data, zstat = cls._loadData(context, path) - obj = cls._fromRaw(raw_data, zstat) + extra = {'connections': connections} + obj = cls._fromRaw(raw_data, zstat, extra) connection = connections.connections[obj.connection_name] obj._set(connection=connection, driver=connection.driver) @@ -186,9 +187,16 @@ class BaseProvider(zkobject.PolymorphicZKObjectMixin, def getProviderSchema(self): return self.schema - def parseConfig(self, config): + def parseProviderConfig(self, config): + """Parse the provider config without any images/labels/flavors + so that the other objects can collect any information they + need from the cloud region when they are parsed""" schema = self.getProviderSchema() ret = schema(config) + return ret + + def parseFullConfig(self, config): + ret = self.parseProviderConfig(config) ret.update(dict( images=self.parseImages(config), flavors=self.parseFlavors(config), @@ -196,9 +204,23 @@ class BaseProvider(zkobject.PolymorphicZKObjectMixin, )) return ret - def deserialize(self, raw, context): + def parseConfig(self, config, connection): + schema = self.getProviderSchema() + ret = schema(config) + ret.update(dict( + images=self.parseImages(config, connection), + flavors=self.parseFlavors(config, connection), + labels=self.parseLabels(config, connection), + )) + return ret + + def deserialize(self, raw, context, extra): data = super().deserialize(raw, context) - data.update(self.parseConfig(data['config'])) + connections = extra['connections'] + connection = connections.connections[data['connection_name']] + data['connection'] = connection + data['driver'] = connection.driver + data.update(self.parseConfig(data['config'], connection)) return data def serialize(self, context): @@ -214,24 +236,24 @@ class BaseProvider(zkobject.PolymorphicZKObjectMixin, def tenant_scoped_name(self): return f'{self.tenant_name}-{self.name}' - def parseImages(self, config): + def parseImages(self, config, connection): images = {} for image_config in config.get('images', []): - i = self.parseImage(image_config, config) + i = self.parseImage(image_config, config, connection) images[i.name] = i return images - def parseFlavors(self, config): + def parseFlavors(self, config, connection): flavors = {} for flavor_config in config.get('flavors', []): - f = self.parseFlavor(flavor_config, config) + f = self.parseFlavor(flavor_config, config, connection) flavors[f.name] = f return flavors - def parseLabels(self, config): + def parseLabels(self, config, connection): labels = {} for label_config in config.get('labels', []): - l = self.parseLabel(label_config, config) + l = self.parseLabel(label_config, config, connection) labels[l.name] = l return labels diff --git a/zuul/zk/branch_cache.py b/zuul/zk/branch_cache.py index 46362a32c8..7a916c1cd1 100644 --- a/zuul/zk/branch_cache.py +++ b/zuul/zk/branch_cache.py @@ -230,8 +230,8 @@ class BranchCacheZKObject(ShardedZKObject): "default_branch": default_branch, } - def deserialize(self, raw, context): - data = super().deserialize(raw, context) + def deserialize(self, raw, context, extra=None): + data = super().deserialize(raw, context, extra) if "protected" in data: # MODEL_API < 27 self.deserialize_old(data) diff --git a/zuul/zk/cache.py b/zuul/zk/cache.py index 49ad27f244..22532f2ca0 100644 --- a/zuul/zk/cache.py +++ b/zuul/zk/cache.py @@ -289,7 +289,7 @@ class ZuulTreeCache(abc.ABC): if getattr(old_obj, 'lock', None): # Don't update a locked object return - old_obj._updateFromRaw(data, stat) + old_obj._updateFromRaw(data, stat, None) else: obj = self.objectFromDict(data, stat) self._cached_objects[key] = obj diff --git a/zuul/zk/launcher.py b/zuul/zk/launcher.py index b1d6b9caee..f1af1d90a1 100644 --- a/zuul/zk/launcher.py +++ b/zuul/zk/launcher.py @@ -97,7 +97,7 @@ class LockableZKObjectCache(ZuulTreeCache): self.updated_event() def objectFromDict(self, d, zstat): - return self.zkobject_class._fromRaw(d, zstat) + return self.zkobject_class._fromRaw(d, zstat, None) def getItem(self, item_id): self.ensureReady() diff --git a/zuul/zk/zkobject.py b/zuul/zk/zkobject.py index fb624cc45f..b4bdf68524 100644 --- a/zuul/zk/zkobject.py +++ b/zuul/zk/zkobject.py @@ -162,16 +162,19 @@ class ZKObject: raise NotImplementedError() # This should work for most classes - def deserialize(self, data, context): + def deserialize(self, data, context, extra=None): """Implement this method to convert serialized data into object attributes. :param bytes data: A byte string to deserialize :param ZKContext context: A ZKContext object with the current ZK session and lock. + :param extra dict: A dictionary of extra data for use in + deserialization. :returns: A dictionary of attributes and values to be set on the object. + """ if isinstance(data, dict): return data @@ -376,7 +379,7 @@ class ZKObject: if path is None: path = self.getPath() compressed_data, zstat = self._loadData(context, path) - self._updateFromRaw(compressed_data, zstat, context) + self._updateFromRaw(compressed_data, zstat, None, context) @classmethod def _loadData(cls, context, path): @@ -396,17 +399,16 @@ class ZKObject: return compressed_data, zstat @classmethod - def _fromRaw(cls, raw_data, zstat, **kw): + def _fromRaw(cls, raw_data, zstat, extra, **kw): obj = cls() - obj._set(**kw) - obj._updateFromRaw(raw_data, zstat) + obj._updateFromRaw(raw_data, zstat, extra) return obj - def _updateFromRaw(self, raw_data, zstat, context=None): + def _updateFromRaw(self, raw_data, zstat, extra, context=None): try: self._set(_zkobject_hash=None) data = self._decompressData(raw_data) - self._set(**self.deserialize(data, context)) + self._set(**self.deserialize(data, context, extra)) self._set(_zstat=zstat, _zkobject_hash=hash(data), _zkobject_compressed_size=len(raw_data), @@ -600,7 +602,7 @@ class PolymorphicZKObjectMixin(abc.ABC): @classmethod def fromZK(cls, context, path, **kw): raw_data, zstat = cls._loadData(context, path) - return cls._fromRaw(raw_data, zstat, **kw) + return cls._fromRaw(raw_data, zstat, None, **kw) @classmethod def _compressData(cls, data): @@ -613,11 +615,12 @@ class PolymorphicZKObjectMixin(abc.ABC): return super()._decompressData(compressed_data) @classmethod - def _fromRaw(cls, raw_data, zstat, **kw): + def _fromRaw(cls, raw_data, zstat, extra, **kw): subclass_id, _, _ = raw_data.partition(b"\0") try: klass = cls._subclasses[subclass_id] except KeyError: raise RuntimeError(f"Unknown subclass id: {subclass_id}") return super( - PolymorphicZKObjectMixin, klass)._fromRaw(raw_data, zstat, **kw) + PolymorphicZKObjectMixin, klass)._fromRaw( + raw_data, zstat, extra, **kw)