zuul/zuul
Code Issues Proposed changes

Use openstack endpoint for image format

Browse Source 
Currently zuul-launcher is hardcoded to assume every cloud wants
a raw image.  We want to determine this value from the actual
cloud endpoint.  That is simple enough with openstack since it
is part of the cloud region configuration object that we pass to
the client connection constructor.

However, we use this value while parsing or deserializing the
provider config, and we don't currently have a mechanism to obtain
an endpoint without a complete provider.

To address that, this change adds an optional "extra" dictionary to
some zkobject deserialization methods so that in the code paths
we use to construct or deseralize a provider object, we can pass
in the connection registry and obtain a connection and endpoint
for use when parsing the config and creating OpenstackProviderImage
objects.

Once that is done, the openstack test is updated to use qcow2 in
the fake cloud to exercise the change.

Change-Id: I4ce75469c60e264c2786da15206b69f3dc020add
This commit is contained in:
James E. Blair
2024-10-12 13:52:25 -07:00
parent 265c16c74d
commit 5f784ff7dc
12 changed files with 93 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tests/fake_openstack.py
View File

@@ -109,12 +109,19 @@ class FakeOpenstackSession:
return FakeOpenstackResponse({'servers': server_list})
class FakeOpenstackConfig:
pass
class FakeOpenstackConnection:
log = logging.getLogger("zuul.FakeOpenstackConnection")
def __init__(self, cloud):
self.cloud = cloud
self.compute = FakeOpenstackSession(cloud)
self.config = FakeOpenstackConfig()
self.config.config = {}
self.config.config['image_format'] = 'qcow2'
def list_flavors(self, get_extra=False):
return self.cloud.flavors

4
tests/unit/test_openstack_driver.py
View File

@@ -46,7 +46,7 @@ class TestOpenstackDriver(ZuulTestCase):
'metadata': {
'type': 'zuul_image',
'image_name': 'debian-local',
'format': 'raw',
'format': 'qcow2',
'sha256': ('59984dd82f51edb3777b969739a92780'
'a520bb314b8d64b294d5de976bd8efb9'),
'md5sum': '262278e1632567a907e4604e9edd2e83',
@@ -146,7 +146,7 @@ class TestOpenstackDriver(ZuulTestCase):
artifacts = self.launcher.image_build_registry.\
getArtifactsForImage(name)
self.assertEqual(1, len(artifacts))
self.assertEqual('raw', artifacts[0].format)
self.assertEqual('qcow2', artifacts[0].format)
self.assertTrue(artifacts[0].validated)
uploads = self.launcher.image_upload_registry.getUploadsForImage(
name)

6
zuul/driver/aws/awsprovider.py
View File

@@ -183,13 +183,13 @@ class AwsProvider(BaseProvider, subclass_id='aws'):
self._set(_endpoint=self.getEndpoint())
return self._endpoint
def parseImage(self, image_config, provider_config):
def parseImage(self, image_config, provider_config, connection):
return AwsProviderImage(image_config, provider_config)
def parseFlavor(self, flavor_config, provider_config):
def parseFlavor(self, flavor_config, provider_config, connection):
return AwsProviderFlavor(flavor_config, provider_config)
def parseLabel(self, label_config, provider_config):
def parseLabel(self, label_config, provider_config, connection):
return AwsProviderLabel(label_config, provider_config)
def getEndpoint(self):

14
zuul/driver/openstack/__init__.py
View File

@@ -48,21 +48,23 @@ class OpenstackDriver(Driver, ConnectionInterface, ProviderInterface):
def getProviderNodeClass(self):
return openstackmodel.OpenstackProviderNode
def getEndpoint(self, provider):
region = provider.region or ''
def _getEndpoint(self, connection, region):
region_str = region or ''
endpoint_id = '/'.join([
urllib.parse.quote_plus(provider.connection.connection_name),
urllib.parse.quote_plus(region),
urllib.parse.quote_plus(connection.connection_name),
urllib.parse.quote_plus(region_str),
])
try:
return self.endpoints[endpoint_id]
except KeyError:
pass
endpoint = self._endpoint_class(
self, provider.connection, provider.region)
endpoint = self._endpoint_class(self, connection, region)
self.endpoints[endpoint_id] = endpoint
return endpoint
def getEndpoint(self, provider):
return self._getEndpoint(provider.connection, provider.region)
def stop(self):
for endpoint in self.endpoints.values():
endpoint.stop()

3
zuul/driver/openstack/openstackendpoint.py
View File

@@ -576,6 +576,9 @@ class OpenstackProviderEndpoint(BaseProviderEndpoint):
rate_limit=self.connection.rate,
)
def getImageFormat(self):
return self._client.config.config['image_format']
def _submitApi(self, api, *args, **kw):
return self.api_executor.submit(
api, *args, **kw)

17
zuul/driver/openstack/openstackprovider.py
View File

@@ -80,10 +80,11 @@ class OpenstackProviderImage(BaseProviderImage):
discriminant=discriminate(
lambda val, alt: val['type'] == alt['type']))
def __init__(self, image_config, provider_config):
def __init__(self, image_config, provider_config, image_format):
self.image_id = None
self.image_filters = None
super().__init__(image_config, provider_config)
self.format = image_format
class OpenstackProviderFlavor(BaseProviderFlavor):
@@ -167,13 +168,19 @@ class OpenstackProvider(BaseProvider, subclass_id='openstack'):
self._set(_endpoint=self.getEndpoint())
return self._endpoint
def parseImage(self, image_config, provider_config):
return OpenstackProviderImage(image_config, provider_config)
def parseImage(self, image_config, provider_config, connection):
# We are not fully constructed yet at this point, so we need
# to peek to get the region and endpoint.
region = provider_config.get('region')
endpoint = connection.driver._getEndpoint(connection, region)
return OpenstackProviderImage(
image_config, provider_config,
image_format=endpoint.getImageFormat())
def parseFlavor(self, flavor_config, provider_config):
def parseFlavor(self, flavor_config, provider_config, connection):
return OpenstackProviderFlavor(flavor_config, provider_config)
def parseLabel(self, label_config, provider_config):
def parseLabel(self, label_config, provider_config, connection):
return OpenstackProviderLabel(label_config, provider_config)
def getEndpoint(self):

16
zuul/model.py
View File

@@ -382,7 +382,7 @@ class ConfigurationErrorList(zkobject.ShardedZKObject):
}
return json.dumps(data, sort_keys=True).encode("utf8")
def deserialize(self, raw, context):
def deserialize(self, raw, context, extra=None):
data = super().deserialize(raw, context)
data.update({
"errors": [ConfigurationError.deserialize(d)
@@ -869,7 +869,7 @@ class PipelineState(zkobject.ZKObject):
self._set(**self._lateInitData())
self.internalCreate(context)
def deserialize(self, raw, context):
def deserialize(self, raw, context, extra=None):
# We may have old change objects in the pipeline cache, so
# make sure they are the same objects we would get from the
# source change cache.
@@ -1081,7 +1081,7 @@ class PipelineChangeList(zkobject.ShardedZKObject):
}
return json.dumps(data, sort_keys=True).encode("utf8")
def deserialize(self, raw, context):
def deserialize(self, raw, context, extra=None):
data = super().deserialize(raw, context)
change_keys = []
# We must have a dictionary with a 'changes' key; otherwise we
@@ -1200,7 +1200,7 @@ class ChangeQueue(zkobject.ZKObject):
}
return json.dumps(data, sort_keys=True).encode("utf8")
def deserialize(self, raw, context):
def deserialize(self, raw, context, extra=None):
data = super().deserialize(raw, context)
existing_items = {}
@@ -3149,7 +3149,7 @@ class FrozenJob(zkobject.ZKObject):
# Use json_dumps to strip any ZuulMark entries
return json_dumps(data, sort_keys=True).encode("utf8")
def deserialize(self, raw, context):
def deserialize(self, raw, context, extra=None):
# Ensure that any special handling in this method is matched
# in Job.freezeJob so that FrozenJobs are identical regardless
# of whether they have been deserialized.
@@ -4976,7 +4976,7 @@ class Build(zkobject.ZKObject):
return json.dumps(data, sort_keys=True).encode("utf8")
def deserialize(self, raw, context):
def deserialize(self, raw, context, extra=None):
data = super().deserialize(raw, context)
# Deserialize build events
@@ -5425,7 +5425,7 @@ class BuildSet(zkobject.ZKObject):
}
return json.dumps(data, sort_keys=True).encode("utf8")
def deserialize(self, raw, context):
def deserialize(self, raw, context, extra=None):
data = super().deserialize(raw, context)
# Set our UUID so that getPath() returns the correct path for
# child objects.
@@ -5907,7 +5907,7 @@ class QueueItem(zkobject.ZKObject):
}
return json.dumps(data, sort_keys=True).encode("utf8")
def deserialize(self, raw, context):
def deserialize(self, raw, context, extra=None):
data = super().deserialize(raw, context)
# Set our UUID so that getPath() returns the correct path for
# child objects.

44
zuul/provider/__init__.py
View File

@@ -148,7 +148,7 @@ class BaseProvider(zkobject.PolymorphicZKObjectMixin,
config = config.copy()
config.pop('_source_context')
config.pop('_start_mark')
parsed_config = self.parseConfig(config)
parsed_config = self.parseConfig(config, connection)
parsed_config.pop('connection')
self._set(
driver=driver,
@@ -177,7 +177,8 @@ class BaseProvider(zkobject.PolymorphicZKObjectMixin,
"""
raw_data, zstat = cls._loadData(context, path)
obj = cls._fromRaw(raw_data, zstat)
extra = {'connections': connections}
obj = cls._fromRaw(raw_data, zstat, extra)
connection = connections.connections[obj.connection_name]
obj._set(connection=connection,
driver=connection.driver)
@@ -186,9 +187,16 @@ class BaseProvider(zkobject.PolymorphicZKObjectMixin,
def getProviderSchema(self):
return self.schema
def parseConfig(self, config):
def parseProviderConfig(self, config):
"""Parse the provider config without any images/labels/flavors
so that the other objects can collect any information they
need from the cloud region when they are parsed"""
schema = self.getProviderSchema()
ret = schema(config)
return ret
def parseFullConfig(self, config):
ret = self.parseProviderConfig(config)
ret.update(dict(
images=self.parseImages(config),
flavors=self.parseFlavors(config),
@@ -196,9 +204,23 @@ class BaseProvider(zkobject.PolymorphicZKObjectMixin,
))
return ret
def deserialize(self, raw, context):
def parseConfig(self, config, connection):
schema = self.getProviderSchema()
ret = schema(config)
ret.update(dict(
images=self.parseImages(config, connection),
flavors=self.parseFlavors(config, connection),
labels=self.parseLabels(config, connection),
))
return ret
def deserialize(self, raw, context, extra):
data = super().deserialize(raw, context)
data.update(self.parseConfig(data['config']))
connections = extra['connections']
connection = connections.connections[data['connection_name']]
data['connection'] = connection
data['driver'] = connection.driver
data.update(self.parseConfig(data['config'], connection))
return data
def serialize(self, context):
@@ -214,24 +236,24 @@ class BaseProvider(zkobject.PolymorphicZKObjectMixin,
def tenant_scoped_name(self):
return f'{self.tenant_name}-{self.name}'
def parseImages(self, config):
def parseImages(self, config, connection):
images = {}
for image_config in config.get('images', []):
i = self.parseImage(image_config, config)
i = self.parseImage(image_config, config, connection)
images[i.name] = i
return images
def parseFlavors(self, config):
def parseFlavors(self, config, connection):
flavors = {}
for flavor_config in config.get('flavors', []):
f = self.parseFlavor(flavor_config, config)
f = self.parseFlavor(flavor_config, config, connection)
flavors[f.name] = f
return flavors
def parseLabels(self, config):
def parseLabels(self, config, connection):
labels = {}
for label_config in config.get('labels', []):
l = self.parseLabel(label_config, config)
l = self.parseLabel(label_config, config, connection)
labels[l.name] = l
return labels

4
zuul/zk/branch_cache.py
View File

@@ -230,8 +230,8 @@ class BranchCacheZKObject(ShardedZKObject):
"default_branch": default_branch,
}
def deserialize(self, raw, context):
data = super().deserialize(raw, context)
def deserialize(self, raw, context, extra=None):
data = super().deserialize(raw, context, extra)
if "protected" in data:
# MODEL_API < 27
self.deserialize_old(data)

2
zuul/zk/cache.py
View File

@@ -289,7 +289,7 @@ class ZuulTreeCache(abc.ABC):
if getattr(old_obj, 'lock', None):
# Don't update a locked object
return
old_obj._updateFromRaw(data, stat)
old_obj._updateFromRaw(data, stat, None)
else:
obj = self.objectFromDict(data, stat)
self._cached_objects[key] = obj

2
zuul/zk/launcher.py
View File

@@ -97,7 +97,7 @@ class LockableZKObjectCache(ZuulTreeCache):
self.updated_event()
def objectFromDict(self, d, zstat):
return self.zkobject_class._fromRaw(d, zstat)
return self.zkobject_class._fromRaw(d, zstat, None)
def getItem(self, item_id):
self.ensureReady()

23
zuul/zk/zkobject.py
View File

@@ -162,16 +162,19 @@ class ZKObject:
raise NotImplementedError()
# This should work for most classes
def deserialize(self, data, context):
def deserialize(self, data, context, extra=None):
"""Implement this method to convert serialized data into object
attributes.
:param bytes data: A byte string to deserialize
:param ZKContext context: A ZKContext object with the current
ZK session and lock.
:param extra dict: A dictionary of extra data for use in
deserialization.
:returns: A dictionary of attributes and values to be set on
the object.
"""
if isinstance(data, dict):
return data
@@ -376,7 +379,7 @@ class ZKObject:
if path is None:
path = self.getPath()
compressed_data, zstat = self._loadData(context, path)
self._updateFromRaw(compressed_data, zstat, context)
self._updateFromRaw(compressed_data, zstat, None, context)
@classmethod
def _loadData(cls, context, path):
@@ -396,17 +399,16 @@ class ZKObject:
return compressed_data, zstat
@classmethod
def _fromRaw(cls, raw_data, zstat, **kw):
def _fromRaw(cls, raw_data, zstat, extra, **kw):
obj = cls()
obj._set(**kw)
obj._updateFromRaw(raw_data, zstat)
obj._updateFromRaw(raw_data, zstat, extra)
return obj
def _updateFromRaw(self, raw_data, zstat, context=None):
def _updateFromRaw(self, raw_data, zstat, extra, context=None):
try:
self._set(_zkobject_hash=None)
data = self._decompressData(raw_data)
self._set(**self.deserialize(data, context))
self._set(**self.deserialize(data, context, extra))
self._set(_zstat=zstat,
_zkobject_hash=hash(data),
_zkobject_compressed_size=len(raw_data),
@@ -600,7 +602,7 @@ class PolymorphicZKObjectMixin(abc.ABC):
@classmethod
def fromZK(cls, context, path, **kw):
raw_data, zstat = cls._loadData(context, path)
return cls._fromRaw(raw_data, zstat, **kw)
return cls._fromRaw(raw_data, zstat, None, **kw)
@classmethod
def _compressData(cls, data):
@@ -613,11 +615,12 @@ class PolymorphicZKObjectMixin(abc.ABC):
return super()._decompressData(compressed_data)
@classmethod
def _fromRaw(cls, raw_data, zstat, **kw):
def _fromRaw(cls, raw_data, zstat, extra, **kw):
subclass_id, _, _ = raw_data.partition(b"\0")
try:
klass = cls._subclasses[subclass_id]
except KeyError:
raise RuntimeError(f"Unknown subclass id: {subclass_id}")
return super(
PolymorphicZKObjectMixin, klass)._fromRaw(raw_data, zstat, **kw)
PolymorphicZKObjectMixin, klass)._fromRaw(
raw_data, zstat, extra, **kw)