diff --git a/tripleo_common/image/image_export.py b/tripleo_common/image/image_export.py index 4f17e0bac..c172ffb39 100644 --- a/tripleo_common/image/image_export.py +++ b/tripleo_common/image/image_export.py @@ -26,6 +26,26 @@ LOG = logging.getLogger(__name__) IMAGE_EXPORT_DIR = '/var/lib/image-serve' +MEDIA_TYPES = ( + MEDIA_MANIFEST_V1, + MEDIA_MANIFEST_V1_SIGNED, + MEDIA_MANIFEST_V2, +) = ( + 'application/vnd.docker.distribution.manifest.v1+json', + 'application/vnd.docker.distribution.manifest.v1+prettyjws', + 'application/vnd.docker.distribution.manifest.v2+json', +) + +TYPE_KEYS = ( + TYPE_KEY_URI, + TYPE_KEY_TYPE +) = ( + 'URI', + 'Content-Type' +) + +TYPE_MAP_EXTENSION = '.type-map' + def make_dir(path): if os.path.exists(path): @@ -148,14 +168,18 @@ def export_manifest_config(target_url, manifests_path = os.path.join( IMAGE_EXPORT_DIR, 'v2', image, 'manifests') + manifests_htaccess_path = os.path.join(manifests_path, '.htaccess') manifest_dir_path = os.path.join(manifests_path, manifest_digest) - manifest_symlink_path = os.path.join(manifests_path, tag) manifest_path = os.path.join(manifest_dir_path, 'index.json') htaccess_path = os.path.join(manifest_dir_path, '.htaccess') make_dir(manifest_dir_path) build_catalog() + with open(manifests_htaccess_path, 'w+') as f: + f.write('AddHandler type-map %s\n' % TYPE_MAP_EXTENSION) + f.write('MultiviewsMatch Handlers\n') + headers = collections.OrderedDict() headers['Content-Type'] = manifest_type headers['Docker-Content-Digest'] = manifest_digest @@ -165,14 +189,53 @@ def export_manifest_config(target_url, f.write('Header set %s "%s"\n' % header) with open(manifest_path, 'w+b') as f: - f.write(manifest_str.encode('utf-8')) + manifest_data = manifest_str.encode('utf-8') + f.write(manifest_data) - if os.path.exists(manifest_symlink_path): - os.remove(manifest_symlink_path) - os.symlink(manifest_dir_path, manifest_symlink_path) + write_type_map_file(image, tag, manifest_digest) build_tags_list(image) +def write_type_map_file(image, tag, manifest_digest): + manifests_path = os.path.join( + IMAGE_EXPORT_DIR, 'v2', image, 'manifests') + type_map_path = os.path.join(manifests_path, '%s%s' % + (tag, TYPE_MAP_EXTENSION)) + with open(type_map_path, 'w+') as f: + f.write('URI: %s\n\n' % tag) + f.write('Content-Type: %s\n' % MEDIA_MANIFEST_V2) + f.write('URI: %s/index.json\n\n' % manifest_digest) + + +def parse_type_map_file(type_map_path): + uri = None + content_type = None + type_map = {} + with open(type_map_path, 'r') as f: + for l in f: + line = l[:-1] + if not line: + if uri and content_type: + type_map[content_type] = uri + uri = None + content_type = None + else: + key, value = line.split(': ') + if key == TYPE_KEY_URI: + uri = value + elif key == TYPE_KEY_TYPE: + content_type = value + return type_map + + +def migrate_to_type_map_file(image, manifest_symlink_path): + tag = os.path.split(manifest_symlink_path)[-1] + manifest_dir = os.readlink(manifest_symlink_path) + manifest_digest = os.path.split(manifest_dir)[-1] + write_type_map_file(image, tag, manifest_digest) + os.remove(manifest_symlink_path) + + def build_tags_list(image): manifests_path = os.path.join( IMAGE_EXPORT_DIR, 'v2', image, 'manifests') @@ -185,6 +248,9 @@ def build_tags_list(image): f_path = os.path.join(manifests_path, f) if os.path.islink(f_path): tags.append(f) + migrate_to_type_map_file(image, f_path) + if f.endswith(TYPE_MAP_EXTENSION): + tags.append(f[:-len(TYPE_MAP_EXTENSION)]) tags_data = { "name": image, @@ -216,11 +282,18 @@ def delete_image(image_url): image, tag = image_tag_from_url(image_url) manifests_path = os.path.join( IMAGE_EXPORT_DIR, 'v2', image, 'manifests') - manifest_symlink_path = os.path.join(manifests_path, tag) - # delete manifest_symlink_path - LOG.debug('Deleting tag symlink %s' % manifest_symlink_path) - os.remove(manifest_symlink_path) + manifest_symlink_path = os.path.join(manifests_path, tag) + if os.path.exists(manifest_symlink_path): + LOG.debug('Deleting legacy tag symlink %s' % manifest_symlink_path) + os.remove(manifest_symlink_path) + + type_map_path = os.path.join(manifests_path, '%s%s' % + (tag, TYPE_MAP_EXTENSION)) + if os.path.exists(type_map_path): + LOG.debug('Deleting typemap file %s' % type_map_path) + os.remove(type_map_path) + build_tags_list(image) # build list of manifest_dir_path without symlinks @@ -228,8 +301,11 @@ def delete_image(image_url): manifest_dirs = set() for f in os.listdir(manifests_path): f_path = os.path.join(manifests_path, f) - if os.path.islink(f_path): - linked_manifest_dirs.add(os.readlink(f_path)) + if f_path.endswith(TYPE_MAP_EXTENSION): + for uri in parse_type_map_file(f_path).values(): + linked_manifest_dir = os.path.dirname( + os.path.join(manifests_path, uri)) + linked_manifest_dirs.add(linked_manifest_dir) elif os.path.isdir(f_path): manifest_dirs.add(f_path) @@ -275,7 +351,8 @@ def delete_image(image_url): os.remove(blob) # if no files left in manifests_path, delete the whole image - if not os.listdir(manifests_path): + remaining = os.listdir(manifests_path) + if not remaining or remaining == ['.htaccess']: image_path = os.path.join(IMAGE_EXPORT_DIR, 'v2', image) LOG.debug('Deleting image directory %s' % image_path) shutil.rmtree(image_path) diff --git a/tripleo_common/tests/image/test_image_export.py b/tripleo_common/tests/image/test_image_export.py index a5579460e..b35cf40c9 100644 --- a/tripleo_common/tests/image/test_image_export.py +++ b/tripleo_common/tests/image/test_image_export.py @@ -225,6 +225,95 @@ Header set ETag "%s" with open(manifest_htaccess_path, 'r') as f: self.assertEqual(expected_htaccess, f.read()) + def test_write_parse_type_map_file(self): + manifest_dir_path = os.path.join( + image_export.IMAGE_EXPORT_DIR, + 'v2/foo/bar/manifests' + ) + map_file_path = os.path.join( + image_export.IMAGE_EXPORT_DIR, + manifest_dir_path, 'latest.type-map' + ) + + image_export.make_dir(manifest_dir_path) + image_export.write_type_map_file( + 'foo/bar', 'latest', 'sha256:1234abcd') + + expected_map_file = '''URI: latest + +Content-Type: application/vnd.docker.distribution.manifest.v2+json +URI: sha256:1234abcd/index.json + +''' + # assert the file contains the expected content + with open(map_file_path, 'r') as f: + self.assertEqual(expected_map_file, f.read()) + + # assert parse_type_map_file correctly reads that file + self.assertEqual( + { + 'application/vnd.docker.distribution.manifest.v2+json': + 'sha256:1234abcd/index.json' + }, + image_export.parse_type_map_file(map_file_path) + ) + + # assert a multi-entry file is correctly parsed + multi_map_file = '''URI: latest + +Content-Type: application/vnd.docker.distribution.manifest.v2+json +URI: sha256:1234abcd/index.json + +Content-Type: application/vnd.docker.distribution.manifest.list.v2+json +URI: sha256:eeeeeeee/index.json + +''' + with open(map_file_path, 'w+') as f: + f.write(multi_map_file) + self.assertEqual( + { + 'application/vnd.docker.distribution.manifest.v2+json': + 'sha256:1234abcd/index.json', + 'application/vnd.docker.distribution.manifest.list.v2+json': + 'sha256:eeeeeeee/index.json' + }, + image_export.parse_type_map_file(map_file_path) + ) + + def test_migrate_to_type_map_file(self): + manifest_dir_path = os.path.join( + image_export.IMAGE_EXPORT_DIR, + 'v2/foo/bar/manifests' + ) + map_file_path = os.path.join( + image_export.IMAGE_EXPORT_DIR, + manifest_dir_path, 'latest.type-map' + ) + symlink_path = os.path.join( + image_export.IMAGE_EXPORT_DIR, + manifest_dir_path, 'latest' + ) + manifest_path = os.path.join( + image_export.IMAGE_EXPORT_DIR, + manifest_dir_path, 'sha256:1234abcd' + ) + image_export.make_dir(manifest_dir_path) + # create legacy symlink + os.symlink(manifest_path, symlink_path) + + # run the migration + image_export.migrate_to_type_map_file('foo/bar', symlink_path) + + expected_map_file = '''URI: latest + +Content-Type: application/vnd.docker.distribution.manifest.v2+json +URI: sha256:1234abcd/index.json + +''' + # assert the migrated file contains the expected content + with open(map_file_path, 'r') as f: + self.assertEqual(expected_map_file, f.read()) + def _write_test_image(self, url, manifest): image, tag = image_uploader.BaseImageUploader._image_tag_from_url( url) @@ -256,18 +345,13 @@ Header set ETag "%s" f.write('The Blob') return manifest_digest - def assertFiles(self, dirs, files, symlinks, deleted): + def assertFiles(self, dirs, files, deleted): for d in dirs: self.assertTrue(os.path.isdir(d), 'is dir: %s' % d) for f in files: self.assertTrue(os.path.isfile(f), 'is file: %s' % f) for d in deleted: - self.assertFalse(os.path.exists(d), 'not exists: %s' % d) - for l, f in symlinks.items(): - self.assertTrue(os.path.islink(l), 'is link: %s' % l) - self.assertEqual(f, os.readlink(l)) - self.assertTrue(os.path.exists(f), - 'link target exists: %s' % f) + self.assertFalse(os.path.exists(d), 'deleted still exists: %s' % d) def test_delete_image(self): url1 = urlparse('docker://localhost:8787/t/nova-api:latest') @@ -309,7 +393,7 @@ Header set ETag "%s" blob_dir = os.path.join(image_dir, 'blobs') m_dir = os.path.join(image_dir, 'manifests') - # assert every directory, file and symlink for the 2 images + # assert every directory and file for the 2 images self.assertFiles( dirs=[ v2_dir, @@ -326,11 +410,9 @@ Header set ETag "%s" os.path.join(blob_dir, 'sha256:4dc536.gz'), os.path.join(blob_dir, 'sha256:5678'), os.path.join(blob_dir, 'sha256:eeeeee.gz'), + os.path.join(m_dir, 'latest.type-map'), + os.path.join(m_dir, 'abc.type-map'), ], - symlinks={ - os.path.join(m_dir, 'latest'): os.path.join(m_dir, m1_digest), - os.path.join(m_dir, 'abc'): os.path.join(m_dir, m2_digest), - }, deleted=[] ) @@ -349,10 +431,8 @@ Header set ETag "%s" os.path.join(m_dir, m1_digest, 'index.json'), os.path.join(blob_dir, 'sha256:aeb786.gz'), os.path.join(blob_dir, 'sha256:4dc536.gz'), + os.path.join(m_dir, 'latest.type-map'), ], - symlinks={ - os.path.join(m_dir, 'latest'): os.path.join(m_dir, m1_digest), - }, deleted=[ os.path.join(m_dir, 'abc'), os.path.join(m_dir, m2_digest), @@ -370,7 +450,6 @@ Header set ETag "%s" v2_dir, ], files=[], - symlinks={}, deleted=[ image_dir, blob_dir,