From c2b969b72bb2e66fe43e75474a84dcc0bb9cd7b0 Mon Sep 17 00:00:00 2001 From: Steve Baker Date: Wed, 13 Mar 2019 13:01:39 +1300 Subject: [PATCH] Implement image delete for exported images This will allow an image delete command to be implemented so that users can manually manage which images are available for deployment. Blueprint: podman-support Change-Id: Ia534c3344c52ae8e031a8e71392ef116a3cea237 --- tripleo_common/image/image_export.py | 85 ++++++++- tripleo_common/image/image_uploader.py | 21 ++- .../tests/image/test_image_export.py | 163 ++++++++++++++++++ 3 files changed, 264 insertions(+), 5 deletions(-) diff --git a/tripleo_common/image/image_export.py b/tripleo_common/image/image_export.py index bc57a1bda..679294545 100644 --- a/tripleo_common/image/image_export.py +++ b/tripleo_common/image/image_export.py @@ -17,6 +17,7 @@ import collections import hashlib import json import os +import shutil from oslo_log import log as logging @@ -151,11 +152,8 @@ def export_manifest_config(target_url, manifest_symlink_path = os.path.join(manifests_path, tag) manifest_path = os.path.join(manifest_dir_path, 'index.json') htaccess_path = os.path.join(manifest_dir_path, '.htaccess') - tags_dir_path = os.path.join(IMAGE_EXPORT_DIR, 'v2', image, 'tags') - tags_list_path = os.path.join(tags_dir_path, 'list') make_dir(manifest_dir_path) - make_dir(tags_dir_path) build_catalog() headers = collections.OrderedDict() @@ -172,7 +170,16 @@ def export_manifest_config(target_url, if os.path.exists(manifest_symlink_path): os.remove(manifest_symlink_path) os.symlink(manifest_dir_path, manifest_symlink_path) + build_tags_list(image) + +def build_tags_list(image): + manifests_path = os.path.join( + IMAGE_EXPORT_DIR, 'v2', image, 'manifests') + tags_dir_path = os.path.join(IMAGE_EXPORT_DIR, 'v2', image, 'tags') + tags_list_path = os.path.join(tags_dir_path, 'list') + LOG.debug('Rebuilding %s' % tags_dir_path) + make_dir(tags_dir_path) tags = [] for f in os.listdir(manifests_path): f_path = os.path.join(manifests_path, f) @@ -203,3 +210,75 @@ def build_catalog(): catalog = {'repositories': catalog_entries} with open(catalog_path, 'w+b') as f: f.write(json.dumps(catalog, ensure_ascii=False).encode('utf-8')) + + +def delete_image(image_url): + image, tag = image_tag_from_url(image_url) + manifests_path = os.path.join( + IMAGE_EXPORT_DIR, 'v2', image, 'manifests') + manifest_symlink_path = os.path.join(manifests_path, tag) + + # delete manifest_symlink_path + LOG.debug('Deleting tag symlink %s' % manifest_symlink_path) + os.remove(manifest_symlink_path) + build_tags_list(image) + + # build list of manifest_dir_path without symlinks + linked_manifest_dirs = set() + manifest_dirs = set() + for f in os.listdir(manifests_path): + f_path = os.path.join(manifests_path, f) + if os.path.islink(f_path): + linked_manifest_dirs.add(os.readlink(f_path)) + elif os.path.isdir(f_path): + manifest_dirs.add(f_path) + + delete_manifest_dirs = manifest_dirs.difference(linked_manifest_dirs) + + # delete list of manifest_dir_path without symlinks + for manifest_dir in delete_manifest_dirs: + LOG.debug('Deleting manifest %s' % manifest_dir) + shutil.rmtree(manifest_dir) + + # load all remaining manifests and build the set of of in-use blobs, + # delete any layer blob not in-use + reffed_blobs = set() + blobs_path = os.path.join(IMAGE_EXPORT_DIR, 'v2', image, 'blobs') + + def add_reffed_blob(digest): + blob_path = os.path.join(blobs_path, digest) + gz_blob_path = os.path.join(blobs_path, '%s.gz' % digest) + if os.path.isfile(gz_blob_path): + reffed_blobs.add(gz_blob_path) + elif os.path.isfile(blob_path): + reffed_blobs.add(blob_path) + + for manifest_dir in linked_manifest_dirs: + manifest_path = os.path.join(manifest_dir, 'index.json') + with open(manifest_path) as f: + manifest = json.load(f) + v1manifest = manifest.get('schemaVersion', 2) == 1 + + if v1manifest: + for digest in manifest.get('fsLayers', []): + add_reffed_blob(digest) + else: + for layer in manifest.get('layers', []): + add_reffed_blob(layer.get('digest')) + add_reffed_blob(manifest.get('config', {}).get('digest')) + + all_blobs = set([os.path.join(blobs_path, b) + for b in os.listdir(blobs_path)]) + delete_blobs = all_blobs.difference(reffed_blobs) + for blob in delete_blobs: + LOG.debug('Deleting layer blob %s' % blob) + os.remove(blob) + + # if no files left in manifests_path, delete the whole image + if not os.listdir(manifests_path): + image_path = os.path.join(IMAGE_EXPORT_DIR, 'v2', image) + LOG.debug('Deleting image directory %s' % image_path) + shutil.rmtree(image_path) + + # rebuild the catalog for the current image list + build_catalog() diff --git a/tripleo_common/image/image_uploader.py b/tripleo_common/image/image_uploader.py index 966850a1d..10bc5391d 100644 --- a/tripleo_common/image/image_uploader.py +++ b/tripleo_common/image/image_uploader.py @@ -477,6 +477,14 @@ class BaseImageUploader(object): image_url = self._image_to_url(image) return self._inspect(image_url, session) + def delete(self, image, session=None): + image_url = self._image_to_url(image) + return self._delete(image_url, session) + + @classmethod + def _delete(cls, image, session=None): + raise NotImplementedError() + @classmethod @tenacity.retry( # Retry up to 5 times with jittered exponential backoff reraise=True, @@ -782,7 +790,7 @@ class SkopeoImageUploader(BaseImageUploader): return out @classmethod - def _delete(cls, image_url): + def _delete(cls, image_url, session=None): insecure = cls.is_insecure_registry(image_url.netloc) image = image_url.geturl() LOG.info('Deleting %s' % image) @@ -1513,9 +1521,18 @@ class PythonImageUploader(BaseImageUploader): } @classmethod - def _delete(cls, image_url): + def _delete_from_registry(cls, image_url, session=None): + if not cls._detect_target_export(image_url, session): + raise NotImplementedError( + 'Deleting not supported via the registry API') + return image_export.delete_image(image_url) + + @classmethod + def _delete(cls, image_url, session=None): image = image_url.geturl() LOG.info('Deleting %s' % image) + if image_url.scheme == 'docker': + return cls._delete_from_registry(image_url, session) if image_url.scheme != 'containers-storage': raise ImageUploaderException('Delete not implemented for %s' % image_url.geturl()) diff --git a/tripleo_common/tests/image/test_image_export.py b/tripleo_common/tests/image/test_image_export.py index 0ae72b14c..964e0ac48 100644 --- a/tripleo_common/tests/image/test_image_export.py +++ b/tripleo_common/tests/image/test_image_export.py @@ -224,3 +224,166 @@ Header set ETag "%s" self.assertEqual(manifest_str, f.read()) with open(manifest_htaccess_path, 'r') as f: self.assertEqual(expected_htaccess, f.read()) + + def _write_test_image(self, url, manifest): + image, tag = image_uploader.BaseImageUploader._image_tag_from_url( + url) + blob_dir = os.path.join( + image_export.IMAGE_EXPORT_DIR, 'v2', image[1:], 'blobs') + image_export.make_dir(blob_dir) + + config_str = '{"config": {}}' + manifest_str = json.dumps(manifest) + calc_digest = hashlib.sha256() + calc_digest.update(manifest_str.encode('utf-8')) + manifest_digest = 'sha256:%s' % calc_digest.hexdigest() + + image_export.export_manifest_config( + url, manifest_str, + image_uploader.MEDIA_MANIFEST_V2, config_str + ) + for layer in manifest['layers']: + blob_path = os.path.join(blob_dir, '%s.gz' % layer['digest']) + + with open(blob_path, 'w+') as f: + f.write('The Blob') + return manifest_digest + + def assertFiles(self, dirs, files, symlinks, deleted): + for d in dirs: + self.assertTrue(os.path.isdir(d), 'is dir: %s' % d) + for f in files: + self.assertTrue(os.path.isfile(f), 'is file: %s' % f) + for d in deleted: + self.assertFalse(os.path.exists(d), 'not exists: %s' % d) + for l, f in symlinks.items(): + self.assertTrue(os.path.islink(l), 'is link: %s' % l) + self.assertEqual(f, os.readlink(l)) + self.assertTrue(os.path.exists(f), + 'link target exists: %s' % f) + + def test_delete_image(self): + url1 = urlparse('docker://localhost:8787/t/nova-api:latest') + url2 = urlparse('docker://localhost:8787/t/nova-api:abc') + manifest_1 = { + 'config': { + 'digest': 'sha256:1234', + 'size': 2, + 'mediaType': 'application/vnd.docker.container.image.v1+json' + }, + 'layers': [ + {'digest': 'sha256:aeb786'}, + {'digest': 'sha256:4dc536'}, + ], + 'mediaType': 'application/vnd.docker.' + 'distribution.manifest.v2+json', + } + manifest_2 = { + 'config': { + 'digest': 'sha256:5678', + 'size': 2, + 'mediaType': 'application/vnd.docker.container.image.v1+json' + }, + 'layers': [ + {'digest': 'sha256:aeb786'}, # shared with manifest_1 + {'digest': 'sha256:eeeeee'}, # different to manifest_1 + ], + 'mediaType': 'application/vnd.docker.' + 'distribution.manifest.v2+json', + } + + m1_digest = self._write_test_image( + url=url1, + manifest=manifest_1 + ) + m2_digest = self._write_test_image( + url=url2, + manifest=manifest_2 + ) + + v2_dir = os.path.join(image_export.IMAGE_EXPORT_DIR, 'v2') + image_dir = os.path.join(v2_dir, 't/nova-api') + blob_dir = os.path.join(image_dir, 'blobs') + m_dir = os.path.join(image_dir, 'manifests') + + # assert every directory, file and symlink for the 2 images + self.assertFiles( + dirs=[ + v2_dir, + image_dir, + blob_dir, + m_dir, + os.path.join(m_dir, m1_digest), + os.path.join(m_dir, m2_digest), + ], + files=[ + os.path.join(m_dir, m1_digest, 'index.json'), + os.path.join(m_dir, m2_digest, 'index.json'), + os.path.join(blob_dir, 'sha256:1234'), + os.path.join(blob_dir, 'sha256:aeb786.gz'), + os.path.join(blob_dir, 'sha256:4dc536.gz'), + os.path.join(blob_dir, 'sha256:5678'), + os.path.join(blob_dir, 'sha256:eeeeee.gz'), + ], + symlinks={ + os.path.join(m_dir, 'latest'): os.path.join(m_dir, m1_digest), + os.path.join(m_dir, 'abc'): os.path.join(m_dir, m2_digest), + }, + deleted=[] + ) + + image_export.delete_image(url2) + + # assert files deleted for nova-api:abc + self.assertFiles( + dirs=[ + v2_dir, + image_dir, + blob_dir, + m_dir, + os.path.join(m_dir, m1_digest), + ], + files=[ + os.path.join(m_dir, m1_digest, 'index.json'), + os.path.join(blob_dir, 'sha256:1234'), + os.path.join(blob_dir, 'sha256:aeb786.gz'), + os.path.join(blob_dir, 'sha256:4dc536.gz'), + ], + symlinks={ + os.path.join(m_dir, 'latest'): os.path.join(m_dir, m1_digest), + }, + deleted=[ + os.path.join(m_dir, 'abc'), + os.path.join(m_dir, m2_digest), + os.path.join(m_dir, m2_digest, 'index.json'), + os.path.join(blob_dir, 'sha256:5678'), + os.path.join(blob_dir, 'sha256:eeeeee.gz'), + ] + ) + + image_export.delete_image(url1) + + # assert all nova-api files deleted after deleting the last image + self.assertFiles( + dirs=[ + v2_dir, + ], + files=[], + symlinks={}, + deleted=[ + image_dir, + blob_dir, + m_dir, + os.path.join(m_dir, 'abc'), + os.path.join(m_dir, 'latest'), + os.path.join(m_dir, m1_digest), + os.path.join(m_dir, m1_digest, 'index.json'), + os.path.join(m_dir, m2_digest), + os.path.join(m_dir, m2_digest, 'index.json'), + os.path.join(blob_dir, 'sha256:5678'), + os.path.join(blob_dir, 'sha256:eeeeee.gz'), + os.path.join(blob_dir, 'sha256:1234'), + os.path.join(blob_dir, 'sha256:aeb786.gz'), + os.path.join(blob_dir, 'sha256:4dc536.gz'), + ] + )