Implement image delete for exported images

This will allow an image delete command to be implemented so that
users can manually manage which images are available for deployment.

Blueprint: podman-support
Change-Id: Ia534c3344c52ae8e031a8e71392ef116a3cea237

tripleo_common/image/image_export.py

@@ -17,6 +17,7 @@ import collections
 import hashlib
 import json
 import os
+import shutil
 
 from oslo_log import log as logging
 
@@ -151,11 +152,8 @@ def export_manifest_config(target_url,
     manifest_symlink_path = os.path.join(manifests_path, tag)
     manifest_path = os.path.join(manifest_dir_path, 'index.json')
     htaccess_path = os.path.join(manifest_dir_path, '.htaccess')
-    tags_dir_path = os.path.join(IMAGE_EXPORT_DIR, 'v2', image, 'tags')
-    tags_list_path = os.path.join(tags_dir_path, 'list')
 
     make_dir(manifest_dir_path)
-    make_dir(tags_dir_path)
     build_catalog()
 
     headers = collections.OrderedDict()
@@ -172,7 +170,16 @@ def export_manifest_config(target_url,
     if os.path.exists(manifest_symlink_path):
         os.remove(manifest_symlink_path)
     os.symlink(manifest_dir_path, manifest_symlink_path)
+    build_tags_list(image)
 
+
+def build_tags_list(image):
+    manifests_path = os.path.join(
+        IMAGE_EXPORT_DIR, 'v2', image, 'manifests')
+    tags_dir_path = os.path.join(IMAGE_EXPORT_DIR, 'v2', image, 'tags')
+    tags_list_path = os.path.join(tags_dir_path, 'list')
+    LOG.debug('Rebuilding %s' % tags_dir_path)
+    make_dir(tags_dir_path)
     tags = []
     for f in os.listdir(manifests_path):
         f_path = os.path.join(manifests_path, f)
@@ -203,3 +210,75 @@ def build_catalog():
     catalog = {'repositories': catalog_entries}
     with open(catalog_path, 'w+b') as f:
         f.write(json.dumps(catalog, ensure_ascii=False).encode('utf-8'))
+
+
+def delete_image(image_url):
+    image, tag = image_tag_from_url(image_url)
+    manifests_path = os.path.join(
+        IMAGE_EXPORT_DIR, 'v2', image, 'manifests')
+    manifest_symlink_path = os.path.join(manifests_path, tag)
+
+    # delete manifest_symlink_path
+    LOG.debug('Deleting tag symlink %s' % manifest_symlink_path)
+    os.remove(manifest_symlink_path)
+    build_tags_list(image)
+
+    # build list of manifest_dir_path without symlinks
+    linked_manifest_dirs = set()
+    manifest_dirs = set()
+    for f in os.listdir(manifests_path):
+        f_path = os.path.join(manifests_path, f)
+        if os.path.islink(f_path):
+            linked_manifest_dirs.add(os.readlink(f_path))
+        elif os.path.isdir(f_path):
+            manifest_dirs.add(f_path)
+
+    delete_manifest_dirs = manifest_dirs.difference(linked_manifest_dirs)
+
+    # delete list of manifest_dir_path without symlinks
+    for manifest_dir in delete_manifest_dirs:
+        LOG.debug('Deleting manifest %s' % manifest_dir)
+        shutil.rmtree(manifest_dir)
+
+    # load all remaining manifests and build the set of of in-use blobs,
+    # delete any layer blob not in-use
+    reffed_blobs = set()
+    blobs_path = os.path.join(IMAGE_EXPORT_DIR, 'v2', image, 'blobs')
+
+    def add_reffed_blob(digest):
+        blob_path = os.path.join(blobs_path, digest)
+        gz_blob_path = os.path.join(blobs_path, '%s.gz' % digest)
+        if os.path.isfile(gz_blob_path):
+            reffed_blobs.add(gz_blob_path)
+        elif os.path.isfile(blob_path):
+            reffed_blobs.add(blob_path)
+
+    for manifest_dir in linked_manifest_dirs:
+        manifest_path = os.path.join(manifest_dir, 'index.json')
+        with open(manifest_path) as f:
+            manifest = json.load(f)
+        v1manifest = manifest.get('schemaVersion', 2) == 1
+
+        if v1manifest:
+            for digest in manifest.get('fsLayers', []):
+                add_reffed_blob(digest)
+        else:
+            for layer in manifest.get('layers', []):
+                add_reffed_blob(layer.get('digest'))
+            add_reffed_blob(manifest.get('config', {}).get('digest'))
+
+    all_blobs = set([os.path.join(blobs_path, b)
+                     for b in os.listdir(blobs_path)])
+    delete_blobs = all_blobs.difference(reffed_blobs)
+    for blob in delete_blobs:
+        LOG.debug('Deleting layer blob %s' % blob)
+        os.remove(blob)
+
+    # if no files left in manifests_path, delete the whole image
+    if not os.listdir(manifests_path):
+        image_path = os.path.join(IMAGE_EXPORT_DIR, 'v2', image)
+        LOG.debug('Deleting image directory %s' % image_path)
+        shutil.rmtree(image_path)
+
+    # rebuild the catalog for the current image list
+    build_catalog()

tripleo_common/image/image_uploader.py

@@ -477,6 +477,14 @@ class BaseImageUploader(object):
         image_url = self._image_to_url(image)
         return self._inspect(image_url, session)
 
+    def delete(self, image, session=None):
+        image_url = self._image_to_url(image)
+        return self._delete(image_url, session)
+
+    @classmethod
+    def _delete(cls, image, session=None):
+        raise NotImplementedError()
+
     @classmethod
     @tenacity.retry(  # Retry up to 5 times with jittered exponential backoff
         reraise=True,
@@ -782,7 +790,7 @@ class SkopeoImageUploader(BaseImageUploader):
         return out
 
     @classmethod
-    def _delete(cls, image_url):
+    def _delete(cls, image_url, session=None):
         insecure = cls.is_insecure_registry(image_url.netloc)
         image = image_url.geturl()
         LOG.info('Deleting %s' % image)
@@ -1513,9 +1521,18 @@ class PythonImageUploader(BaseImageUploader):
         }
 
     @classmethod
-    def _delete(cls, image_url):
+    def _delete_from_registry(cls, image_url, session=None):
+        if not cls._detect_target_export(image_url, session):
+            raise NotImplementedError(
+                'Deleting not supported via the registry API')
+        return image_export.delete_image(image_url)
+
+    @classmethod
+    def _delete(cls, image_url, session=None):
         image = image_url.geturl()
         LOG.info('Deleting %s' % image)
+        if image_url.scheme == 'docker':
+            return cls._delete_from_registry(image_url, session)
         if image_url.scheme != 'containers-storage':
             raise ImageUploaderException('Delete not implemented for %s' %
                                          image_url.geturl())

tripleo_common/tests/image/test_image_export.py

@@ -224,3 +224,166 @@ Header set ETag "%s"
             self.assertEqual(manifest_str, f.read())
         with open(manifest_htaccess_path, 'r') as f:
             self.assertEqual(expected_htaccess, f.read())
+
+    def _write_test_image(self, url, manifest):
+        image, tag = image_uploader.BaseImageUploader._image_tag_from_url(
+            url)
+        blob_dir = os.path.join(
+            image_export.IMAGE_EXPORT_DIR, 'v2', image[1:], 'blobs')
+        image_export.make_dir(blob_dir)
+
+        config_str = '{"config": {}}'
+        manifest_str = json.dumps(manifest)
+        calc_digest = hashlib.sha256()
+        calc_digest.update(manifest_str.encode('utf-8'))
+        manifest_digest = 'sha256:%s' % calc_digest.hexdigest()
+
+        image_export.export_manifest_config(
+            url, manifest_str,
+            image_uploader.MEDIA_MANIFEST_V2, config_str
+        )
+        for layer in manifest['layers']:
+            blob_path = os.path.join(blob_dir, '%s.gz' % layer['digest'])
+
+            with open(blob_path, 'w+') as f:
+                f.write('The Blob')
+        return manifest_digest
+
+    def assertFiles(self, dirs, files, symlinks, deleted):
+        for d in dirs:
+            self.assertTrue(os.path.isdir(d), 'is dir: %s' % d)
+        for f in files:
+            self.assertTrue(os.path.isfile(f), 'is file: %s' % f)
+        for d in deleted:
+            self.assertFalse(os.path.exists(d), 'not exists: %s' % d)
+        for l, f in symlinks.items():
+            self.assertTrue(os.path.islink(l), 'is link: %s' % l)
+            self.assertEqual(f, os.readlink(l))
+            self.assertTrue(os.path.exists(f),
+                            'link target exists: %s' % f)
+
+    def test_delete_image(self):
+        url1 = urlparse('docker://localhost:8787/t/nova-api:latest')
+        url2 = urlparse('docker://localhost:8787/t/nova-api:abc')
+        manifest_1 = {
+            'config': {
+                'digest': 'sha256:1234',
+                'size': 2,
+                'mediaType': 'application/vnd.docker.container.image.v1+json'
+            },
+            'layers': [
+                {'digest': 'sha256:aeb786'},
+                {'digest': 'sha256:4dc536'},
+            ],
+            'mediaType': 'application/vnd.docker.'
+                         'distribution.manifest.v2+json',
+        }
+        manifest_2 = {
+            'config': {
+                'digest': 'sha256:5678',
+                'size': 2,
+                'mediaType': 'application/vnd.docker.container.image.v1+json'
+            },
+            'layers': [
+                {'digest': 'sha256:aeb786'},  # shared with manifest_1
+                {'digest': 'sha256:eeeeee'},  # different to manifest_1
+            ],
+            'mediaType': 'application/vnd.docker.'
+                         'distribution.manifest.v2+json',
+        }
+
+        m1_digest = self._write_test_image(
+            url=url1,
+            manifest=manifest_1
+        )
+        m2_digest = self._write_test_image(
+            url=url2,
+            manifest=manifest_2
+        )
+
+        v2_dir = os.path.join(image_export.IMAGE_EXPORT_DIR, 'v2')
+        image_dir = os.path.join(v2_dir, 't/nova-api')
+        blob_dir = os.path.join(image_dir, 'blobs')
+        m_dir = os.path.join(image_dir, 'manifests')
+
+        # assert every directory, file and symlink for the 2 images
+        self.assertFiles(
+            dirs=[
+                v2_dir,
+                image_dir,
+                blob_dir,
+                m_dir,
+                os.path.join(m_dir, m1_digest),
+                os.path.join(m_dir, m2_digest),
+            ],
+            files=[
+                os.path.join(m_dir, m1_digest, 'index.json'),
+                os.path.join(m_dir, m2_digest, 'index.json'),
+                os.path.join(blob_dir, 'sha256:1234'),
+                os.path.join(blob_dir, 'sha256:aeb786.gz'),
+                os.path.join(blob_dir, 'sha256:4dc536.gz'),
+                os.path.join(blob_dir, 'sha256:5678'),
+                os.path.join(blob_dir, 'sha256:eeeeee.gz'),
+            ],
+            symlinks={
+                os.path.join(m_dir, 'latest'): os.path.join(m_dir, m1_digest),
+                os.path.join(m_dir, 'abc'): os.path.join(m_dir, m2_digest),
+            },
+            deleted=[]
+        )
+
+        image_export.delete_image(url2)
+
+        # assert files deleted for nova-api:abc
+        self.assertFiles(
+            dirs=[
+                v2_dir,
+                image_dir,
+                blob_dir,
+                m_dir,
+                os.path.join(m_dir, m1_digest),
+            ],
+            files=[
+                os.path.join(m_dir, m1_digest, 'index.json'),
+                os.path.join(blob_dir, 'sha256:1234'),
+                os.path.join(blob_dir, 'sha256:aeb786.gz'),
+                os.path.join(blob_dir, 'sha256:4dc536.gz'),
+            ],
+            symlinks={
+                os.path.join(m_dir, 'latest'): os.path.join(m_dir, m1_digest),
+            },
+            deleted=[
+                os.path.join(m_dir, 'abc'),
+                os.path.join(m_dir, m2_digest),
+                os.path.join(m_dir, m2_digest, 'index.json'),
+                os.path.join(blob_dir, 'sha256:5678'),
+                os.path.join(blob_dir, 'sha256:eeeeee.gz'),
+            ]
+        )
+
+        image_export.delete_image(url1)
+
+        # assert all nova-api files deleted after deleting the last image
+        self.assertFiles(
+            dirs=[
+                v2_dir,
+            ],
+            files=[],
+            symlinks={},
+            deleted=[
+                image_dir,
+                blob_dir,
+                m_dir,
+                os.path.join(m_dir, 'abc'),
+                os.path.join(m_dir, 'latest'),
+                os.path.join(m_dir, m1_digest),
+                os.path.join(m_dir, m1_digest, 'index.json'),
+                os.path.join(m_dir, m2_digest),
+                os.path.join(m_dir, m2_digest, 'index.json'),
+                os.path.join(blob_dir, 'sha256:5678'),
+                os.path.join(blob_dir, 'sha256:eeeeee.gz'),
+                os.path.join(blob_dir, 'sha256:1234'),
+                os.path.join(blob_dir, 'sha256:aeb786.gz'),
+                os.path.join(blob_dir, 'sha256:4dc536.gz'),
+            ]
+        )