Use a typemap file instead of symlinks for tags

To support manifest lists, the response for requesting a tagged
manifest will need to return different contents based on the
content-type requested. Representing a tag as a symlink to the
manifest directory is not flexible enough for this need.

This change switches over to a type-map[1] based implementation so
that future changes can add entries to different files based on the
content-type. Whenever any write operation is performed on an image,
any existing symlink based tags will be migrated to a typemap.

[1] https://httpd.apache.org/docs/2.4/mod/mod_negotiation.html#typemaps
Change-Id: I0e56e1d74ef8468f1b89da5329a8f1ced711c0f3

tripleo_common/image/image_export.py

@@ -26,6 +26,26 @@ LOG = logging.getLogger(__name__)
 
 IMAGE_EXPORT_DIR = '/var/lib/image-serve'
 
+MEDIA_TYPES = (
+    MEDIA_MANIFEST_V1,
+    MEDIA_MANIFEST_V1_SIGNED,
+    MEDIA_MANIFEST_V2,
+) = (
+    'application/vnd.docker.distribution.manifest.v1+json',
+    'application/vnd.docker.distribution.manifest.v1+prettyjws',
+    'application/vnd.docker.distribution.manifest.v2+json',
+)
+
+TYPE_KEYS = (
+    TYPE_KEY_URI,
+    TYPE_KEY_TYPE
+) = (
+    'URI',
+    'Content-Type'
+)
+
+TYPE_MAP_EXTENSION = '.type-map'
+
 
 def make_dir(path):
     if os.path.exists(path):
@@ -148,14 +168,18 @@ def export_manifest_config(target_url,
 
     manifests_path = os.path.join(
         IMAGE_EXPORT_DIR, 'v2', image, 'manifests')
+    manifests_htaccess_path = os.path.join(manifests_path, '.htaccess')
     manifest_dir_path = os.path.join(manifests_path, manifest_digest)
-    manifest_symlink_path = os.path.join(manifests_path, tag)
     manifest_path = os.path.join(manifest_dir_path, 'index.json')
     htaccess_path = os.path.join(manifest_dir_path, '.htaccess')
 
     make_dir(manifest_dir_path)
     build_catalog()
 
+    with open(manifests_htaccess_path, 'w+') as f:
+        f.write('AddHandler type-map %s\n' % TYPE_MAP_EXTENSION)
+        f.write('MultiviewsMatch Handlers\n')
+
     headers = collections.OrderedDict()
     headers['Content-Type'] = manifest_type
     headers['Docker-Content-Digest'] = manifest_digest
@@ -165,14 +189,53 @@ def export_manifest_config(target_url,
             f.write('Header set %s "%s"\n' % header)
 
     with open(manifest_path, 'w+b') as f:
-        f.write(manifest_str.encode('utf-8'))
+        manifest_data = manifest_str.encode('utf-8')
+        f.write(manifest_data)
 
-    if os.path.exists(manifest_symlink_path):
+    write_type_map_file(image, tag, manifest_digest)
-        os.remove(manifest_symlink_path)
-    os.symlink(manifest_dir_path, manifest_symlink_path)
     build_tags_list(image)
 
 
+def write_type_map_file(image, tag, manifest_digest):
+    manifests_path = os.path.join(
+        IMAGE_EXPORT_DIR, 'v2', image, 'manifests')
+    type_map_path = os.path.join(manifests_path, '%s%s' %
+                                 (tag, TYPE_MAP_EXTENSION))
+    with open(type_map_path, 'w+') as f:
+        f.write('URI: %s\n\n' % tag)
+        f.write('Content-Type: %s\n' % MEDIA_MANIFEST_V2)
+        f.write('URI: %s/index.json\n\n' % manifest_digest)
+
+
+def parse_type_map_file(type_map_path):
+    uri = None
+    content_type = None
+    type_map = {}
+    with open(type_map_path, 'r') as f:
+        for l in f:
+            line = l[:-1]
+            if not line:
+                if uri and content_type:
+                    type_map[content_type] = uri
+                uri = None
+                content_type = None
+            else:
+                key, value = line.split(': ')
+                if key == TYPE_KEY_URI:
+                    uri = value
+                elif key == TYPE_KEY_TYPE:
+                    content_type = value
+    return type_map
+
+
+def migrate_to_type_map_file(image, manifest_symlink_path):
+    tag = os.path.split(manifest_symlink_path)[-1]
+    manifest_dir = os.readlink(manifest_symlink_path)
+    manifest_digest = os.path.split(manifest_dir)[-1]
+    write_type_map_file(image, tag, manifest_digest)
+    os.remove(manifest_symlink_path)
+
+
 def build_tags_list(image):
     manifests_path = os.path.join(
         IMAGE_EXPORT_DIR, 'v2', image, 'manifests')
@@ -185,6 +248,9 @@ def build_tags_list(image):
         f_path = os.path.join(manifests_path, f)
         if os.path.islink(f_path):
             tags.append(f)
+            migrate_to_type_map_file(image, f_path)
+        if f.endswith(TYPE_MAP_EXTENSION):
+            tags.append(f[:-len(TYPE_MAP_EXTENSION)])
 
     tags_data = {
         "name": image,
@@ -216,11 +282,18 @@ def delete_image(image_url):
     image, tag = image_tag_from_url(image_url)
     manifests_path = os.path.join(
         IMAGE_EXPORT_DIR, 'v2', image, 'manifests')
-    manifest_symlink_path = os.path.join(manifests_path, tag)
 
-    # delete manifest_symlink_path
+    manifest_symlink_path = os.path.join(manifests_path, tag)
-    LOG.debug('Deleting tag symlink %s' % manifest_symlink_path)
+    if os.path.exists(manifest_symlink_path):
-    os.remove(manifest_symlink_path)
+        LOG.debug('Deleting legacy tag symlink %s' % manifest_symlink_path)
+        os.remove(manifest_symlink_path)
+
+    type_map_path = os.path.join(manifests_path, '%s%s' %
+                                 (tag, TYPE_MAP_EXTENSION))
+    if os.path.exists(type_map_path):
+        LOG.debug('Deleting typemap file %s' % type_map_path)
+        os.remove(type_map_path)
+
     build_tags_list(image)
 
     # build list of manifest_dir_path without symlinks
@@ -228,8 +301,11 @@ def delete_image(image_url):
     manifest_dirs = set()
     for f in os.listdir(manifests_path):
         f_path = os.path.join(manifests_path, f)
-        if os.path.islink(f_path):
+        if f_path.endswith(TYPE_MAP_EXTENSION):
-            linked_manifest_dirs.add(os.readlink(f_path))
+            for uri in parse_type_map_file(f_path).values():
+                linked_manifest_dir = os.path.dirname(
+                    os.path.join(manifests_path, uri))
+                linked_manifest_dirs.add(linked_manifest_dir)
         elif os.path.isdir(f_path):
             manifest_dirs.add(f_path)
 
@@ -275,7 +351,8 @@ def delete_image(image_url):
         os.remove(blob)
 
     # if no files left in manifests_path, delete the whole image
-    if not os.listdir(manifests_path):
+    remaining = os.listdir(manifests_path)
+    if not remaining or remaining == ['.htaccess']:
         image_path = os.path.join(IMAGE_EXPORT_DIR, 'v2', image)
         LOG.debug('Deleting image directory %s' % image_path)
         shutil.rmtree(image_path)

tripleo_common/tests/image/test_image_export.py

@@ -225,6 +225,95 @@ Header set ETag "%s"
         with open(manifest_htaccess_path, 'r') as f:
             self.assertEqual(expected_htaccess, f.read())
 
+    def test_write_parse_type_map_file(self):
+        manifest_dir_path = os.path.join(
+            image_export.IMAGE_EXPORT_DIR,
+            'v2/foo/bar/manifests'
+        )
+        map_file_path = os.path.join(
+            image_export.IMAGE_EXPORT_DIR,
+            manifest_dir_path, 'latest.type-map'
+        )
+
+        image_export.make_dir(manifest_dir_path)
+        image_export.write_type_map_file(
+            'foo/bar', 'latest', 'sha256:1234abcd')
+
+        expected_map_file = '''URI: latest
+
+Content-Type: application/vnd.docker.distribution.manifest.v2+json
+URI: sha256:1234abcd/index.json
+
+'''
+        # assert the file contains the expected content
+        with open(map_file_path, 'r') as f:
+            self.assertEqual(expected_map_file, f.read())
+
+        # assert parse_type_map_file correctly reads that file
+        self.assertEqual(
+            {
+                'application/vnd.docker.distribution.manifest.v2+json':
+                'sha256:1234abcd/index.json'
+            },
+            image_export.parse_type_map_file(map_file_path)
+        )
+
+        # assert a multi-entry file is correctly parsed
+        multi_map_file = '''URI: latest
+
+Content-Type: application/vnd.docker.distribution.manifest.v2+json
+URI: sha256:1234abcd/index.json
+
+Content-Type: application/vnd.docker.distribution.manifest.list.v2+json
+URI: sha256:eeeeeeee/index.json
+
+'''
+        with open(map_file_path, 'w+') as f:
+            f.write(multi_map_file)
+        self.assertEqual(
+            {
+                'application/vnd.docker.distribution.manifest.v2+json':
+                'sha256:1234abcd/index.json',
+                'application/vnd.docker.distribution.manifest.list.v2+json':
+                'sha256:eeeeeeee/index.json'
+            },
+            image_export.parse_type_map_file(map_file_path)
+        )
+
+    def test_migrate_to_type_map_file(self):
+        manifest_dir_path = os.path.join(
+            image_export.IMAGE_EXPORT_DIR,
+            'v2/foo/bar/manifests'
+        )
+        map_file_path = os.path.join(
+            image_export.IMAGE_EXPORT_DIR,
+            manifest_dir_path, 'latest.type-map'
+        )
+        symlink_path = os.path.join(
+            image_export.IMAGE_EXPORT_DIR,
+            manifest_dir_path, 'latest'
+        )
+        manifest_path = os.path.join(
+            image_export.IMAGE_EXPORT_DIR,
+            manifest_dir_path, 'sha256:1234abcd'
+        )
+        image_export.make_dir(manifest_dir_path)
+        # create legacy symlink
+        os.symlink(manifest_path, symlink_path)
+
+        # run the migration
+        image_export.migrate_to_type_map_file('foo/bar', symlink_path)
+
+        expected_map_file = '''URI: latest
+
+Content-Type: application/vnd.docker.distribution.manifest.v2+json
+URI: sha256:1234abcd/index.json
+
+'''
+        # assert the migrated file contains the expected content
+        with open(map_file_path, 'r') as f:
+            self.assertEqual(expected_map_file, f.read())
+
     def _write_test_image(self, url, manifest):
         image, tag = image_uploader.BaseImageUploader._image_tag_from_url(
             url)
@@ -256,18 +345,13 @@ Header set ETag "%s"
                 f.write('The Blob')
         return manifest_digest
 
-    def assertFiles(self, dirs, files, symlinks, deleted):
+    def assertFiles(self, dirs, files, deleted):
         for d in dirs:
             self.assertTrue(os.path.isdir(d), 'is dir: %s' % d)
         for f in files:
             self.assertTrue(os.path.isfile(f), 'is file: %s' % f)
         for d in deleted:
-            self.assertFalse(os.path.exists(d), 'not exists: %s' % d)
+            self.assertFalse(os.path.exists(d), 'deleted still exists: %s' % d)
-        for l, f in symlinks.items():
-            self.assertTrue(os.path.islink(l), 'is link: %s' % l)
-            self.assertEqual(f, os.readlink(l))
-            self.assertTrue(os.path.exists(f),
-                            'link target exists: %s' % f)
 
     def test_delete_image(self):
         url1 = urlparse('docker://localhost:8787/t/nova-api:latest')
@@ -309,7 +393,7 @@ Header set ETag "%s"
         blob_dir = os.path.join(image_dir, 'blobs')
         m_dir = os.path.join(image_dir, 'manifests')
 
-        # assert every directory, file and symlink for the 2 images
+        # assert every directory and file for the 2 images
         self.assertFiles(
             dirs=[
                 v2_dir,
@@ -326,11 +410,9 @@ Header set ETag "%s"
                 os.path.join(blob_dir, 'sha256:4dc536.gz'),
                 os.path.join(blob_dir, 'sha256:5678'),
                 os.path.join(blob_dir, 'sha256:eeeeee.gz'),
+                os.path.join(m_dir, 'latest.type-map'),
+                os.path.join(m_dir, 'abc.type-map'),
             ],
-            symlinks={
-                os.path.join(m_dir, 'latest'): os.path.join(m_dir, m1_digest),
-                os.path.join(m_dir, 'abc'): os.path.join(m_dir, m2_digest),
-            },
             deleted=[]
         )
 
@@ -349,10 +431,8 @@ Header set ETag "%s"
                 os.path.join(m_dir, m1_digest, 'index.json'),
                 os.path.join(blob_dir, 'sha256:aeb786.gz'),
                 os.path.join(blob_dir, 'sha256:4dc536.gz'),
+                os.path.join(m_dir, 'latest.type-map'),
             ],
-            symlinks={
-                os.path.join(m_dir, 'latest'): os.path.join(m_dir, m1_digest),
-            },
             deleted=[
                 os.path.join(m_dir, 'abc'),
                 os.path.join(m_dir, m2_digest),
@@ -370,7 +450,6 @@ Header set ETag "%s"
                 v2_dir,
             ],
             files=[],
-            symlinks={},
             deleted=[
                 image_dir,
                 blob_dir,