openstack/tripleo-common
Code Issues Proposed changes

Merge "Use a typemap file instead of symlinks for tags"

Browse Source
This commit is contained in:
Zuul 2019-05-29 05:48:35 +00:00 committed by Gerrit Code Review
commit 6c4832a7ca
2 changed files with 184 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
tripleo_common/image/image_export.py
View File

@ -26,6 +26,26 @@ LOG = logging.getLogger(__name__)
IMAGE_EXPORT_DIR = '/var/lib/image-serve'
MEDIA_TYPES = (
MEDIA_MANIFEST_V1,
MEDIA_MANIFEST_V1_SIGNED,
MEDIA_MANIFEST_V2,
) = (
'application/vnd.docker.distribution.manifest.v1+json',
'application/vnd.docker.distribution.manifest.v1+prettyjws',
'application/vnd.docker.distribution.manifest.v2+json',
)
TYPE_KEYS = (
TYPE_KEY_URI,
TYPE_KEY_TYPE
) = (
'URI',
'Content-Type'
)
TYPE_MAP_EXTENSION = '.type-map'
def make_dir(path):
if os.path.exists(path):
@ -148,14 +168,18 @@ def export_manifest_config(target_url,
manifests_path = os.path.join(
IMAGE_EXPORT_DIR, 'v2', image, 'manifests')
manifests_htaccess_path = os.path.join(manifests_path, '.htaccess')
manifest_dir_path = os.path.join(manifests_path, manifest_digest)
manifest_symlink_path = os.path.join(manifests_path, tag)
manifest_path = os.path.join(manifest_dir_path, 'index.json')
htaccess_path = os.path.join(manifest_dir_path, '.htaccess')
make_dir(manifest_dir_path)
build_catalog()
with open(manifests_htaccess_path, 'w+') as f:
f.write('AddHandler type-map %s\n' % TYPE_MAP_EXTENSION)
f.write('MultiviewsMatch Handlers\n')
headers = collections.OrderedDict()
headers['Content-Type'] = manifest_type
headers['Docker-Content-Digest'] = manifest_digest
@ -165,14 +189,53 @@ def export_manifest_config(target_url,
f.write('Header set %s "%s"\n' % header)
with open(manifest_path, 'w+b') as f:
f.write(manifest_str.encode('utf-8'))
manifest_data = manifest_str.encode('utf-8')
f.write(manifest_data)
if os.path.exists(manifest_symlink_path):
os.remove(manifest_symlink_path)
os.symlink(manifest_dir_path, manifest_symlink_path)
write_type_map_file(image, tag, manifest_digest)
build_tags_list(image)
def write_type_map_file(image, tag, manifest_digest):
manifests_path = os.path.join(
IMAGE_EXPORT_DIR, 'v2', image, 'manifests')
type_map_path = os.path.join(manifests_path, '%s%s' %
(tag, TYPE_MAP_EXTENSION))
with open(type_map_path, 'w+') as f:
f.write('URI: %s\n\n' % tag)
f.write('Content-Type: %s\n' % MEDIA_MANIFEST_V2)
f.write('URI: %s/index.json\n\n' % manifest_digest)
def parse_type_map_file(type_map_path):
uri = None
content_type = None
type_map = {}
with open(type_map_path, 'r') as f:
for l in f:
line = l[:-1]
if not line:
if uri and content_type:
type_map[content_type] = uri
uri = None
content_type = None
else:
key, value = line.split(': ')
if key == TYPE_KEY_URI:
uri = value
elif key == TYPE_KEY_TYPE:
content_type = value
return type_map
def migrate_to_type_map_file(image, manifest_symlink_path):
tag = os.path.split(manifest_symlink_path)[-1]
manifest_dir = os.readlink(manifest_symlink_path)
manifest_digest = os.path.split(manifest_dir)[-1]
write_type_map_file(image, tag, manifest_digest)
os.remove(manifest_symlink_path)
def build_tags_list(image):
manifests_path = os.path.join(
IMAGE_EXPORT_DIR, 'v2', image, 'manifests')
@ -185,6 +248,9 @@ def build_tags_list(image):
f_path = os.path.join(manifests_path, f)
if os.path.islink(f_path):
tags.append(f)
migrate_to_type_map_file(image, f_path)
if f.endswith(TYPE_MAP_EXTENSION):
tags.append(f[:-len(TYPE_MAP_EXTENSION)])
tags_data = {
"name": image,
@ -216,11 +282,18 @@ def delete_image(image_url):
image, tag = image_tag_from_url(image_url)
manifests_path = os.path.join(
IMAGE_EXPORT_DIR, 'v2', image, 'manifests')
manifest_symlink_path = os.path.join(manifests_path, tag)
# delete manifest_symlink_path
LOG.debug('Deleting tag symlink %s' % manifest_symlink_path)
os.remove(manifest_symlink_path)
manifest_symlink_path = os.path.join(manifests_path, tag)
if os.path.exists(manifest_symlink_path):
LOG.debug('Deleting legacy tag symlink %s' % manifest_symlink_path)
os.remove(manifest_symlink_path)
type_map_path = os.path.join(manifests_path, '%s%s' %
(tag, TYPE_MAP_EXTENSION))
if os.path.exists(type_map_path):
LOG.debug('Deleting typemap file %s' % type_map_path)
os.remove(type_map_path)
build_tags_list(image)
# build list of manifest_dir_path without symlinks
@ -228,8 +301,11 @@ def delete_image(image_url):
manifest_dirs = set()
for f in os.listdir(manifests_path):
f_path = os.path.join(manifests_path, f)
if os.path.islink(f_path):
linked_manifest_dirs.add(os.readlink(f_path))
if f_path.endswith(TYPE_MAP_EXTENSION):
for uri in parse_type_map_file(f_path).values():
linked_manifest_dir = os.path.dirname(
os.path.join(manifests_path, uri))
linked_manifest_dirs.add(linked_manifest_dir)
elif os.path.isdir(f_path):
manifest_dirs.add(f_path)
@ -275,7 +351,8 @@ def delete_image(image_url):
os.remove(blob)
# if no files left in manifests_path, delete the whole image
if not os.listdir(manifests_path):
remaining = os.listdir(manifests_path)
if not remaining or remaining == ['.htaccess']:
image_path = os.path.join(IMAGE_EXPORT_DIR, 'v2', image)
LOG.debug('Deleting image directory %s' % image_path)
shutil.rmtree(image_path)

111
tripleo_common/tests/image/test_image_export.py
View File

@ -225,6 +225,95 @@ Header set ETag "%s"
with open(manifest_htaccess_path, 'r') as f:
self.assertEqual(expected_htaccess, f.read())
def test_write_parse_type_map_file(self):
manifest_dir_path = os.path.join(
image_export.IMAGE_EXPORT_DIR,
'v2/foo/bar/manifests'
)
map_file_path = os.path.join(
image_export.IMAGE_EXPORT_DIR,
manifest_dir_path, 'latest.type-map'
)
image_export.make_dir(manifest_dir_path)
image_export.write_type_map_file(
'foo/bar', 'latest', 'sha256:1234abcd')
expected_map_file = '''URI: latest
Content-Type: application/vnd.docker.distribution.manifest.v2+json
URI: sha256:1234abcd/index.json
'''
# assert the file contains the expected content
with open(map_file_path, 'r') as f:
self.assertEqual(expected_map_file, f.read())
# assert parse_type_map_file correctly reads that file
self.assertEqual(
{
'application/vnd.docker.distribution.manifest.v2+json':
'sha256:1234abcd/index.json'
},
image_export.parse_type_map_file(map_file_path)
)
# assert a multi-entry file is correctly parsed
multi_map_file = '''URI: latest
Content-Type: application/vnd.docker.distribution.manifest.v2+json
URI: sha256:1234abcd/index.json
Content-Type: application/vnd.docker.distribution.manifest.list.v2+json
URI: sha256:eeeeeeee/index.json
'''
with open(map_file_path, 'w+') as f:
f.write(multi_map_file)
self.assertEqual(
{
'application/vnd.docker.distribution.manifest.v2+json':
'sha256:1234abcd/index.json',
'application/vnd.docker.distribution.manifest.list.v2+json':
'sha256:eeeeeeee/index.json'
},
image_export.parse_type_map_file(map_file_path)
)
def test_migrate_to_type_map_file(self):
manifest_dir_path = os.path.join(
image_export.IMAGE_EXPORT_DIR,
'v2/foo/bar/manifests'
)
map_file_path = os.path.join(
image_export.IMAGE_EXPORT_DIR,
manifest_dir_path, 'latest.type-map'
)
symlink_path = os.path.join(
image_export.IMAGE_EXPORT_DIR,
manifest_dir_path, 'latest'
)
manifest_path = os.path.join(
image_export.IMAGE_EXPORT_DIR,
manifest_dir_path, 'sha256:1234abcd'
)
image_export.make_dir(manifest_dir_path)
# create legacy symlink
os.symlink(manifest_path, symlink_path)
# run the migration
image_export.migrate_to_type_map_file('foo/bar', symlink_path)
expected_map_file = '''URI: latest
Content-Type: application/vnd.docker.distribution.manifest.v2+json
URI: sha256:1234abcd/index.json
'''
# assert the migrated file contains the expected content
with open(map_file_path, 'r') as f:
self.assertEqual(expected_map_file, f.read())
def _write_test_image(self, url, manifest):
image, tag = image_uploader.BaseImageUploader._image_tag_from_url(
url)
@ -256,18 +345,13 @@ Header set ETag "%s"
f.write('The Blob')
return manifest_digest
def assertFiles(self, dirs, files, symlinks, deleted):
def assertFiles(self, dirs, files, deleted):
for d in dirs:
self.assertTrue(os.path.isdir(d), 'is dir: %s' % d)
for f in files:
self.assertTrue(os.path.isfile(f), 'is file: %s' % f)
for d in deleted:
self.assertFalse(os.path.exists(d), 'not exists: %s' % d)
for l, f in symlinks.items():
self.assertTrue(os.path.islink(l), 'is link: %s' % l)
self.assertEqual(f, os.readlink(l))
self.assertTrue(os.path.exists(f),
'link target exists: %s' % f)
self.assertFalse(os.path.exists(d), 'deleted still exists: %s' % d)
def test_delete_image(self):
url1 = urlparse('docker://localhost:8787/t/nova-api:latest')
@ -309,7 +393,7 @@ Header set ETag "%s"
blob_dir = os.path.join(image_dir, 'blobs')
m_dir = os.path.join(image_dir, 'manifests')
# assert every directory, file and symlink for the 2 images
# assert every directory and file for the 2 images
self.assertFiles(
dirs=[
v2_dir,
@ -326,11 +410,9 @@ Header set ETag "%s"
os.path.join(blob_dir, 'sha256:4dc536.gz'),
os.path.join(blob_dir, 'sha256:5678'),
os.path.join(blob_dir, 'sha256:eeeeee.gz'),
os.path.join(m_dir, 'latest.type-map'),
os.path.join(m_dir, 'abc.type-map'),
],
symlinks={
os.path.join(m_dir, 'latest'): os.path.join(m_dir, m1_digest),
os.path.join(m_dir, 'abc'): os.path.join(m_dir, m2_digest),
},
deleted=[]
)
@ -349,10 +431,8 @@ Header set ETag "%s"
os.path.join(m_dir, m1_digest, 'index.json'),
os.path.join(blob_dir, 'sha256:aeb786.gz'),
os.path.join(blob_dir, 'sha256:4dc536.gz'),
os.path.join(m_dir, 'latest.type-map'),
],
symlinks={
os.path.join(m_dir, 'latest'): os.path.join(m_dir, m1_digest),
},
deleted=[
os.path.join(m_dir, 'abc'),
os.path.join(m_dir, m2_digest),
@ -370,7 +450,6 @@ Header set ETag "%s"
v2_dir,
],
files=[],
symlinks={},
deleted=[
image_dir,
blob_dir,