From 5c40e366b323d03d0767977f9f7d66ee3a26ac3f Mon Sep 17 00:00:00 2001 From: Alex Schultz Date: Tue, 5 Oct 2021 08:12:50 -0600 Subject: [PATCH] Revert "Revert "Allow for OCI manifests"" Add support for OCI manifests and includes a fix to handle the schemaVersion 1 metadata which is used by the prometheus containers upstream. This reverts commit b2f76f8514baefb57e10c9a5562457b651ec29af. Change-Id: I352cd33578eeb447d3a36cbca7767b1bfd5de7aa Depends-On: https://review.opendev.org/c/openstack/tripleo-ci/+/812506 Related-Bug: #1860585 (cherry picked from commit 3f2853b9a2e6f7de16bf928721aa2b3b4fa5c801) --- tripleo_common/image/image_uploader.py | 54 ++++++++++++++++--- .../tests/image/test_image_uploader.py | 9 ++-- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/tripleo_common/image/image_uploader.py b/tripleo_common/image/image_uploader.py index 62ca9abf8..4f22cceef 100644 --- a/tripleo_common/image/image_uploader.py +++ b/tripleo_common/image/image_uploader.py @@ -90,7 +90,10 @@ MEDIA_TYPES = ( MEDIA_MANIFEST_V2, MEDIA_MANIFEST_V2_LIST, MEDIA_OCI_MANIFEST_V1, + MEDIA_OCI_CONFIG_V1, MEDIA_OCI_INDEX_V1, + MEDIA_OCI_LAYER, + MEDIA_OCI_LAYER_COMPRESSED, MEDIA_CONFIG, MEDIA_BLOB, MEDIA_BLOB_COMPRESSED @@ -100,7 +103,10 @@ MEDIA_TYPES = ( 'application/vnd.docker.distribution.manifest.v2+json', 'application/vnd.docker.distribution.manifest.list.v2+json', 'application/vnd.oci.image.manifest.v1+json', + 'application/vnd.oci.image.config.v1+json', 'application/vnd.oci.image.index.v1+json', + 'application/vnd.oci.image.layer.v1.tar', + 'application/vnd.oci.image.layer.v1.tar+gzip', 'application/vnd.docker.container.image.v1+json', 'application/vnd.docker.image.rootfs.diff.tar', 'application/vnd.docker.image.rootfs.diff.tar.gzip' @@ -883,7 +889,9 @@ class BaseImageUploader(object): manifest_url = cls._build_url( image_url, CALL_MANIFEST % parts ) - manifest_headers = {'Accept': MEDIA_MANIFEST_V2} + # prefer docker manifest over oci + manifest_headers = {'Accept': ", ".join([ + MEDIA_MANIFEST_V2 + ";q=1", MEDIA_OCI_MANIFEST_V1 + ";q=0.5"])} try: manifest_r = RegistrySessionHelper.get( @@ -1786,7 +1794,9 @@ class PythonImageUploader(BaseImageUploader): if multi_arch: manifest_headers = {'Accept': MEDIA_MANIFEST_V2_LIST} else: - manifest_headers = {'Accept': MEDIA_MANIFEST_V2} + # prefer docker manifest over oci + manifest_headers = {'Accept': ", ".join([ + MEDIA_MANIFEST_V2 + ";q=1", MEDIA_OCI_MANIFEST_V1 + ";q=0.5"])} try: r = RegistrySessionHelper.get( session, @@ -1811,12 +1821,16 @@ class PythonImageUploader(BaseImageUploader): ) manifests_str.append(manifest_str) manifest = json.loads(manifest_str) + media_type = manifest.get('mediaType', + manifest.get('config', {}).get('mediaType')) if manifest.get('schemaVersion', 2) == 1: layers.extend(reversed([x['blobSum'] for x in manifest['fsLayers']])) - elif manifest.get('mediaType') == MEDIA_MANIFEST_V2: + elif not media_type or media_type in [MEDIA_MANIFEST_V2, + MEDIA_OCI_MANIFEST_V1, + MEDIA_OCI_CONFIG_V1]: layers.extend(x['digest'] for x in manifest['layers']) - elif manifest.get('mediaType') == MEDIA_MANIFEST_V2_LIST: + elif media_type == MEDIA_MANIFEST_V2_LIST: image, _, tag = image_url.geturl().rpartition(':') for man in manifest.get('manifests', []): # replace image tag with the manifest hash in the list @@ -2026,7 +2040,14 @@ class PythonImageUploader(BaseImageUploader): for source_manifest in source_manifests: manifest = json.loads(source_manifest) config_str = None - if manifest.get('mediaType') == MEDIA_MANIFEST_V2: + # NOTE(mwhahaha): mediaType will not be set when it's + # schemaVersion 1 + media_type = manifest.get('mediaType', + manifest.get('config', + {}).get('mediaType')) + if media_type in [MEDIA_MANIFEST_V2, + MEDIA_OCI_MANIFEST_V1, + MEDIA_OCI_CONFIG_V1]: config_digest = manifest['config']['digest'] LOG.debug('[%s] Uploading config with digest: %s' % (image, config_digest)) @@ -2078,8 +2099,27 @@ class PythonImageUploader(BaseImageUploader): # is explicitly OCI because buildah uses OCI by default but we # convert the metadata to Docker format in the uploader. # See LP#1860585 - manifest_type = manifest.get('mediaType', False) - if not manifest_type or manifest_type == MEDIA_OCI_MANIFEST_V1: + manifest_type = manifest.get('mediaType', + manifest.get('config', + {}).get('mediaType')) + if manifest_type in [MEDIA_OCI_MANIFEST_V1, + MEDIA_OCI_CONFIG_V1]: + manifest_type = MEDIA_MANIFEST_V2 + # convert config mediaType to docker.container.image + manifest['config']['mediaType'] = MEDIA_CONFIG + layers = manifest.get('layers') + # convert layer type to docker layer type + if layers: + new_layers = [] + for layer in layers: + layer_type = layer.get('mediaType') + if layer_type == MEDIA_OCI_LAYER_COMPRESSED: + layer['mediaType'] = MEDIA_BLOB_COMPRESSED + elif layer_type == MEDIA_OCI_LAYER: + layer['mediaType'] = MEDIA_BLOB + new_layers.append(layer) + manifest['layers'] = new_layers + elif manifest_type == MEDIA_CONFIG: manifest_type = MEDIA_MANIFEST_V2 elif manifest_type == MEDIA_OCI_INDEX_V1: manifest_type = MEDIA_MANIFEST_V2_LIST diff --git a/tripleo_common/tests/image/test_image_uploader.py b/tripleo_common/tests/image/test_image_uploader.py index 8518834e2..574a3516f 100644 --- a/tripleo_common/tests/image/test_image_uploader.py +++ b/tripleo_common/tests/image/test_image_uploader.py @@ -2087,8 +2087,10 @@ class TestPythonImageUploader(base.TestCase): 'nova-api/manifests/tripleo-current', timeout=30, headers={ - 'Accept': 'application/vnd.docker.distribution' - '.manifest.v2+json' + 'Accept': 'application/' + 'vnd.docker.distribution.manifest.v2+json;q=1, ' + 'application/' + 'vnd.oci.image.manifest.v1+json;q=0.5' } ) @@ -2446,13 +2448,12 @@ class TestPythonImageUploader(base.TestCase): 'config': { 'digest': 'sha256:1234', 'size': 2, - 'mediaType': image_uploader.MEDIA_CONFIG + 'mediaType': image_uploader.MEDIA_OCI_CONFIG_V1 }, 'layers': [ {'digest': 'sha256:aaaa'}, {'digest': 'sha256:bbbb'}, ], - 'mediaType': image_uploader.MEDIA_OCI_MANIFEST_V1 }) expected_manifest = { 'config': {