openstack/tripleo-common
Code Issues Proposed changes

Implement image delete for exported images

Browse Source 
This will allow an image delete command to be implemented so that
users can manually manage which images are available for deployment.

Blueprint: podman-support
Change-Id: Ia534c3344c52ae8e031a8e71392ef116a3cea237
This commit is contained in:
Steve Baker 2019-03-13 13:01:39 +13:00
parent 577188682a
commit c2b969b72b
3 changed files with 264 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
tripleo_common/image/image_export.py
View File

@ -17,6 +17,7 @@ import collections
import hashlib
import json
import os
import shutil
from oslo_log import log as logging
@ -151,11 +152,8 @@ def export_manifest_config(target_url,
manifest_symlink_path = os.path.join(manifests_path, tag)
manifest_path = os.path.join(manifest_dir_path, 'index.json')
htaccess_path = os.path.join(manifest_dir_path, '.htaccess')
tags_dir_path = os.path.join(IMAGE_EXPORT_DIR, 'v2', image, 'tags')
tags_list_path = os.path.join(tags_dir_path, 'list')
make_dir(manifest_dir_path)
make_dir(tags_dir_path)
build_catalog()
headers = collections.OrderedDict()
@ -172,7 +170,16 @@ def export_manifest_config(target_url,
if os.path.exists(manifest_symlink_path):
os.remove(manifest_symlink_path)
os.symlink(manifest_dir_path, manifest_symlink_path)
build_tags_list(image)
def build_tags_list(image):
manifests_path = os.path.join(
IMAGE_EXPORT_DIR, 'v2', image, 'manifests')
tags_dir_path = os.path.join(IMAGE_EXPORT_DIR, 'v2', image, 'tags')
tags_list_path = os.path.join(tags_dir_path, 'list')
LOG.debug('Rebuilding %s' % tags_dir_path)
make_dir(tags_dir_path)
tags = []
for f in os.listdir(manifests_path):
f_path = os.path.join(manifests_path, f)
@ -203,3 +210,75 @@ def build_catalog():
catalog = {'repositories': catalog_entries}
with open(catalog_path, 'w+b') as f:
f.write(json.dumps(catalog, ensure_ascii=False).encode('utf-8'))
def delete_image(image_url):
image, tag = image_tag_from_url(image_url)
manifests_path = os.path.join(
IMAGE_EXPORT_DIR, 'v2', image, 'manifests')
manifest_symlink_path = os.path.join(manifests_path, tag)
# delete manifest_symlink_path
LOG.debug('Deleting tag symlink %s' % manifest_symlink_path)
os.remove(manifest_symlink_path)
build_tags_list(image)
# build list of manifest_dir_path without symlinks
linked_manifest_dirs = set()
manifest_dirs = set()
for f in os.listdir(manifests_path):
f_path = os.path.join(manifests_path, f)
if os.path.islink(f_path):
linked_manifest_dirs.add(os.readlink(f_path))
elif os.path.isdir(f_path):
manifest_dirs.add(f_path)
delete_manifest_dirs = manifest_dirs.difference(linked_manifest_dirs)
# delete list of manifest_dir_path without symlinks
for manifest_dir in delete_manifest_dirs:
LOG.debug('Deleting manifest %s' % manifest_dir)
shutil.rmtree(manifest_dir)
# load all remaining manifests and build the set of of in-use blobs,
# delete any layer blob not in-use
reffed_blobs = set()
blobs_path = os.path.join(IMAGE_EXPORT_DIR, 'v2', image, 'blobs')
def add_reffed_blob(digest):
blob_path = os.path.join(blobs_path, digest)
gz_blob_path = os.path.join(blobs_path, '%s.gz' % digest)
if os.path.isfile(gz_blob_path):
reffed_blobs.add(gz_blob_path)
elif os.path.isfile(blob_path):
reffed_blobs.add(blob_path)
for manifest_dir in linked_manifest_dirs:
manifest_path = os.path.join(manifest_dir, 'index.json')
with open(manifest_path) as f:
manifest = json.load(f)
v1manifest = manifest.get('schemaVersion', 2) == 1
if v1manifest:
for digest in manifest.get('fsLayers', []):
add_reffed_blob(digest)
else:
for layer in manifest.get('layers', []):
add_reffed_blob(layer.get('digest'))
add_reffed_blob(manifest.get('config', {}).get('digest'))
all_blobs = set([os.path.join(blobs_path, b)
for b in os.listdir(blobs_path)])
delete_blobs = all_blobs.difference(reffed_blobs)
for blob in delete_blobs:
LOG.debug('Deleting layer blob %s' % blob)
os.remove(blob)
# if no files left in manifests_path, delete the whole image
if not os.listdir(manifests_path):
image_path = os.path.join(IMAGE_EXPORT_DIR, 'v2', image)
LOG.debug('Deleting image directory %s' % image_path)
shutil.rmtree(image_path)
# rebuild the catalog for the current image list
build_catalog()

21
tripleo_common/image/image_uploader.py
View File

@ -477,6 +477,14 @@ class BaseImageUploader(object):
image_url = self._image_to_url(image)
return self._inspect(image_url, session)
def delete(self, image, session=None):
image_url = self._image_to_url(image)
return self._delete(image_url, session)
@classmethod
def _delete(cls, image, session=None):
raise NotImplementedError()
@classmethod
@tenacity.retry( # Retry up to 5 times with jittered exponential backoff
reraise=True,
@ -782,7 +790,7 @@ class SkopeoImageUploader(BaseImageUploader):
return out
@classmethod
def _delete(cls, image_url):
def _delete(cls, image_url, session=None):
insecure = cls.is_insecure_registry(image_url.netloc)
image = image_url.geturl()
LOG.info('Deleting %s' % image)
@ -1513,9 +1521,18 @@ class PythonImageUploader(BaseImageUploader):
}
@classmethod
def _delete(cls, image_url):
def _delete_from_registry(cls, image_url, session=None):
if not cls._detect_target_export(image_url, session):
raise NotImplementedError(
'Deleting not supported via the registry API')
return image_export.delete_image(image_url)
@classmethod
def _delete(cls, image_url, session=None):
image = image_url.geturl()
LOG.info('Deleting %s' % image)
if image_url.scheme == 'docker':
return cls._delete_from_registry(image_url, session)
if image_url.scheme != 'containers-storage':
raise ImageUploaderException('Delete not implemented for %s' %
image_url.geturl())

163
tripleo_common/tests/image/test_image_export.py
View File

@ -224,3 +224,166 @@ Header set ETag "%s"
self.assertEqual(manifest_str, f.read())
with open(manifest_htaccess_path, 'r') as f:
self.assertEqual(expected_htaccess, f.read())
def _write_test_image(self, url, manifest):
image, tag = image_uploader.BaseImageUploader._image_tag_from_url(
url)
blob_dir = os.path.join(
image_export.IMAGE_EXPORT_DIR, 'v2', image[1:], 'blobs')
image_export.make_dir(blob_dir)
config_str = '{"config": {}}'
manifest_str = json.dumps(manifest)
calc_digest = hashlib.sha256()
calc_digest.update(manifest_str.encode('utf-8'))
manifest_digest = 'sha256:%s' % calc_digest.hexdigest()
image_export.export_manifest_config(
url, manifest_str,
image_uploader.MEDIA_MANIFEST_V2, config_str
)
for layer in manifest['layers']:
blob_path = os.path.join(blob_dir, '%s.gz' % layer['digest'])
with open(blob_path, 'w+') as f:
f.write('The Blob')
return manifest_digest
def assertFiles(self, dirs, files, symlinks, deleted):
for d in dirs:
self.assertTrue(os.path.isdir(d), 'is dir: %s' % d)
for f in files:
self.assertTrue(os.path.isfile(f), 'is file: %s' % f)
for d in deleted:
self.assertFalse(os.path.exists(d), 'not exists: %s' % d)
for l, f in symlinks.items():
self.assertTrue(os.path.islink(l), 'is link: %s' % l)
self.assertEqual(f, os.readlink(l))
self.assertTrue(os.path.exists(f),
'link target exists: %s' % f)
def test_delete_image(self):
url1 = urlparse('docker://localhost:8787/t/nova-api:latest')
url2 = urlparse('docker://localhost:8787/t/nova-api:abc')
manifest_1 = {
'config': {
'digest': 'sha256:1234',
'size': 2,
'mediaType': 'application/vnd.docker.container.image.v1+json'
},
'layers': [
{'digest': 'sha256:aeb786'},
{'digest': 'sha256:4dc536'},
],
'mediaType': 'application/vnd.docker.'
'distribution.manifest.v2+json',
}
manifest_2 = {
'config': {
'digest': 'sha256:5678',
'size': 2,
'mediaType': 'application/vnd.docker.container.image.v1+json'
},
'layers': [
{'digest': 'sha256:aeb786'}, # shared with manifest_1
{'digest': 'sha256:eeeeee'}, # different to manifest_1
],
'mediaType': 'application/vnd.docker.'
'distribution.manifest.v2+json',
}
m1_digest = self._write_test_image(
url=url1,
manifest=manifest_1
)
m2_digest = self._write_test_image(
url=url2,
manifest=manifest_2
)
v2_dir = os.path.join(image_export.IMAGE_EXPORT_DIR, 'v2')
image_dir = os.path.join(v2_dir, 't/nova-api')
blob_dir = os.path.join(image_dir, 'blobs')
m_dir = os.path.join(image_dir, 'manifests')
# assert every directory, file and symlink for the 2 images
self.assertFiles(
dirs=[
v2_dir,
image_dir,
blob_dir,
m_dir,
os.path.join(m_dir, m1_digest),
os.path.join(m_dir, m2_digest),
],
files=[
os.path.join(m_dir, m1_digest, 'index.json'),
os.path.join(m_dir, m2_digest, 'index.json'),
os.path.join(blob_dir, 'sha256:1234'),
os.path.join(blob_dir, 'sha256:aeb786.gz'),
os.path.join(blob_dir, 'sha256:4dc536.gz'),
os.path.join(blob_dir, 'sha256:5678'),
os.path.join(blob_dir, 'sha256:eeeeee.gz'),
],
symlinks={
os.path.join(m_dir, 'latest'): os.path.join(m_dir, m1_digest),
os.path.join(m_dir, 'abc'): os.path.join(m_dir, m2_digest),
},
deleted=[]
)
image_export.delete_image(url2)
# assert files deleted for nova-api:abc
self.assertFiles(
dirs=[
v2_dir,
image_dir,
blob_dir,
m_dir,
os.path.join(m_dir, m1_digest),
],
files=[
os.path.join(m_dir, m1_digest, 'index.json'),
os.path.join(blob_dir, 'sha256:1234'),
os.path.join(blob_dir, 'sha256:aeb786.gz'),
os.path.join(blob_dir, 'sha256:4dc536.gz'),
],
symlinks={
os.path.join(m_dir, 'latest'): os.path.join(m_dir, m1_digest),
},
deleted=[
os.path.join(m_dir, 'abc'),
os.path.join(m_dir, m2_digest),
os.path.join(m_dir, m2_digest, 'index.json'),
os.path.join(blob_dir, 'sha256:5678'),
os.path.join(blob_dir, 'sha256:eeeeee.gz'),
]
)
image_export.delete_image(url1)
# assert all nova-api files deleted after deleting the last image
self.assertFiles(
dirs=[
v2_dir,
],
files=[],
symlinks={},
deleted=[
image_dir,
blob_dir,
m_dir,
os.path.join(m_dir, 'abc'),
os.path.join(m_dir, 'latest'),
os.path.join(m_dir, m1_digest),
os.path.join(m_dir, m1_digest, 'index.json'),
os.path.join(m_dir, m2_digest),
os.path.join(m_dir, m2_digest, 'index.json'),
os.path.join(blob_dir, 'sha256:5678'),
os.path.join(blob_dir, 'sha256:eeeeee.gz'),
os.path.join(blob_dir, 'sha256:1234'),
os.path.join(blob_dir, 'sha256:aeb786.gz'),
os.path.join(blob_dir, 'sha256:4dc536.gz'),
]
)