Use a typemap file instead of symlinks for tags
To support manifest lists, the response for requesting a tagged manifest will need to return different contents based on the content-type requested. Representing a tag as a symlink to the manifest directory is not flexible enough for this need. This change switches over to a type-map[1] based implementation so that future changes can add entries to different files based on the content-type. Whenever any write operation is performed on an image, any existing symlink based tags will be migrated to a typemap. [1] https://httpd.apache.org/docs/2.4/mod/mod_negotiation.html#typemaps Change-Id: I0e56e1d74ef8468f1b89da5329a8f1ced711c0f3
This commit is contained in:
Steve Baker
parent
08ae3286c7
commit
35cfa6d363
@ -26,6 +26,26 @@ LOG = logging.getLogger(__name__)
|
||||
|
||||
IMAGE_EXPORT_DIR = '/var/lib/image-serve'
|
||||
|
||||
MEDIA_TYPES = (
|
||||
MEDIA_MANIFEST_V1,
|
||||
MEDIA_MANIFEST_V1_SIGNED,
|
||||
MEDIA_MANIFEST_V2,
|
||||
) = (
|
||||
'application/vnd.docker.distribution.manifest.v1+json',
|
||||
'application/vnd.docker.distribution.manifest.v1+prettyjws',
|
||||
'application/vnd.docker.distribution.manifest.v2+json',
|
||||
)
|
||||
|
||||
TYPE_KEYS = (
|
||||
TYPE_KEY_URI,
|
||||
TYPE_KEY_TYPE
|
||||
) = (
|
||||
'URI',
|
||||
'Content-Type'
|
||||
)
|
||||
|
||||
TYPE_MAP_EXTENSION = '.type-map'
|
||||
|
||||
|
||||
def make_dir(path):
|
||||
if os.path.exists(path):
|
||||
@ -148,14 +168,18 @@ def export_manifest_config(target_url,
|
||||
|
||||
manifests_path = os.path.join(
|
||||
IMAGE_EXPORT_DIR, 'v2', image, 'manifests')
|
||||
manifests_htaccess_path = os.path.join(manifests_path, '.htaccess')
|
||||
manifest_dir_path = os.path.join(manifests_path, manifest_digest)
|
||||
manifest_symlink_path = os.path.join(manifests_path, tag)
|
||||
manifest_path = os.path.join(manifest_dir_path, 'index.json')
|
||||
htaccess_path = os.path.join(manifest_dir_path, '.htaccess')
|
||||
|
||||
make_dir(manifest_dir_path)
|
||||
build_catalog()
|
||||
|
||||
with open(manifests_htaccess_path, 'w+') as f:
|
||||
f.write('AddHandler type-map %s\n' % TYPE_MAP_EXTENSION)
|
||||
f.write('MultiviewsMatch Handlers\n')
|
||||
|
||||
headers = collections.OrderedDict()
|
||||
headers['Content-Type'] = manifest_type
|
||||
headers['Docker-Content-Digest'] = manifest_digest
|
||||
@ -165,14 +189,53 @@ def export_manifest_config(target_url,
|
||||
f.write('Header set %s "%s"\n' % header)
|
||||
|
||||
with open(manifest_path, 'w+b') as f:
|
||||
f.write(manifest_str.encode('utf-8'))
|
||||
manifest_data = manifest_str.encode('utf-8')
|
||||
f.write(manifest_data)
|
||||
|
||||
if os.path.exists(manifest_symlink_path):
|
||||
os.remove(manifest_symlink_path)
|
||||
os.symlink(manifest_dir_path, manifest_symlink_path)
|
||||
write_type_map_file(image, tag, manifest_digest)
|
||||
build_tags_list(image)
|
||||
|
||||
|
||||
def write_type_map_file(image, tag, manifest_digest):
|
||||
manifests_path = os.path.join(
|
||||
IMAGE_EXPORT_DIR, 'v2', image, 'manifests')
|
||||
type_map_path = os.path.join(manifests_path, '%s%s' %
|
||||
(tag, TYPE_MAP_EXTENSION))
|
||||
with open(type_map_path, 'w+') as f:
|
||||
f.write('URI: %s\n\n' % tag)
|
||||
f.write('Content-Type: %s\n' % MEDIA_MANIFEST_V2)
|
||||
f.write('URI: %s/index.json\n\n' % manifest_digest)
|
||||
|
||||
|
||||
def parse_type_map_file(type_map_path):
|
||||
uri = None
|
||||
content_type = None
|
||||
type_map = {}
|
||||
with open(type_map_path, 'r') as f:
|
||||
for l in f:
|
||||
line = l[:-1]
|
||||
if not line:
|
||||
if uri and content_type:
|
||||
type_map[content_type] = uri
|
||||
uri = None
|
||||
content_type = None
|
||||
else:
|
||||
key, value = line.split(': ')
|
||||
if key == TYPE_KEY_URI:
|
||||
uri = value
|
||||
elif key == TYPE_KEY_TYPE:
|
||||
content_type = value
|
||||
return type_map
|
||||
|
||||
|
||||
def migrate_to_type_map_file(image, manifest_symlink_path):
|
||||
tag = os.path.split(manifest_symlink_path)[-1]
|
||||
manifest_dir = os.readlink(manifest_symlink_path)
|
||||
manifest_digest = os.path.split(manifest_dir)[-1]
|
||||
write_type_map_file(image, tag, manifest_digest)
|
||||
os.remove(manifest_symlink_path)
|
||||
|
||||
|
||||
def build_tags_list(image):
|
||||
manifests_path = os.path.join(
|
||||
IMAGE_EXPORT_DIR, 'v2', image, 'manifests')
|
||||
@ -185,6 +248,9 @@ def build_tags_list(image):
|
||||
f_path = os.path.join(manifests_path, f)
|
||||
if os.path.islink(f_path):
|
||||
tags.append(f)
|
||||
migrate_to_type_map_file(image, f_path)
|
||||
if f.endswith(TYPE_MAP_EXTENSION):
|
||||
tags.append(f[:-len(TYPE_MAP_EXTENSION)])
|
||||
|
||||
tags_data = {
|
||||
"name": image,
|
||||
@ -216,11 +282,18 @@ def delete_image(image_url):
|
||||
image, tag = image_tag_from_url(image_url)
|
||||
manifests_path = os.path.join(
|
||||
IMAGE_EXPORT_DIR, 'v2', image, 'manifests')
|
||||
manifest_symlink_path = os.path.join(manifests_path, tag)
|
||||
|
||||
# delete manifest_symlink_path
|
||||
LOG.debug('Deleting tag symlink %s' % manifest_symlink_path)
|
||||
os.remove(manifest_symlink_path)
|
||||
manifest_symlink_path = os.path.join(manifests_path, tag)
|
||||
if os.path.exists(manifest_symlink_path):
|
||||
LOG.debug('Deleting legacy tag symlink %s' % manifest_symlink_path)
|
||||
os.remove(manifest_symlink_path)
|
||||
|
||||
type_map_path = os.path.join(manifests_path, '%s%s' %
|
||||
(tag, TYPE_MAP_EXTENSION))
|
||||
if os.path.exists(type_map_path):
|
||||
LOG.debug('Deleting typemap file %s' % type_map_path)
|
||||
os.remove(type_map_path)
|
||||
|
||||
build_tags_list(image)
|
||||
|
||||
# build list of manifest_dir_path without symlinks
|
||||
@ -228,8 +301,11 @@ def delete_image(image_url):
|
||||
manifest_dirs = set()
|
||||
for f in os.listdir(manifests_path):
|
||||
f_path = os.path.join(manifests_path, f)
|
||||
if os.path.islink(f_path):
|
||||
linked_manifest_dirs.add(os.readlink(f_path))
|
||||
if f_path.endswith(TYPE_MAP_EXTENSION):
|
||||
for uri in parse_type_map_file(f_path).values():
|
||||
linked_manifest_dir = os.path.dirname(
|
||||
os.path.join(manifests_path, uri))
|
||||
linked_manifest_dirs.add(linked_manifest_dir)
|
||||
elif os.path.isdir(f_path):
|
||||
manifest_dirs.add(f_path)
|
||||
|
||||
@ -275,7 +351,8 @@ def delete_image(image_url):
|
||||
os.remove(blob)
|
||||
|
||||
# if no files left in manifests_path, delete the whole image
|
||||
if not os.listdir(manifests_path):
|
||||
remaining = os.listdir(manifests_path)
|
||||
if not remaining or remaining == ['.htaccess']:
|
||||
image_path = os.path.join(IMAGE_EXPORT_DIR, 'v2', image)
|
||||
LOG.debug('Deleting image directory %s' % image_path)
|
||||
shutil.rmtree(image_path)
|
||||
|
||||
@ -225,6 +225,95 @@ Header set ETag "%s"
|
||||
with open(manifest_htaccess_path, 'r') as f:
|
||||
self.assertEqual(expected_htaccess, f.read())
|
||||
|
||||
def test_write_parse_type_map_file(self):
|
||||
manifest_dir_path = os.path.join(
|
||||
image_export.IMAGE_EXPORT_DIR,
|
||||
'v2/foo/bar/manifests'
|
||||
)
|
||||
map_file_path = os.path.join(
|
||||
image_export.IMAGE_EXPORT_DIR,
|
||||
manifest_dir_path, 'latest.type-map'
|
||||
)
|
||||
|
||||
image_export.make_dir(manifest_dir_path)
|
||||
image_export.write_type_map_file(
|
||||
'foo/bar', 'latest', 'sha256:1234abcd')
|
||||
|
||||
expected_map_file = '''URI: latest
|
||||
|
||||
Content-Type: application/vnd.docker.distribution.manifest.v2+json
|
||||
URI: sha256:1234abcd/index.json
|
||||
|
||||
'''
|
||||
# assert the file contains the expected content
|
||||
with open(map_file_path, 'r') as f:
|
||||
self.assertEqual(expected_map_file, f.read())
|
||||
|
||||
# assert parse_type_map_file correctly reads that file
|
||||
self.assertEqual(
|
||||
{
|
||||
'application/vnd.docker.distribution.manifest.v2+json':
|
||||
'sha256:1234abcd/index.json'
|
||||
},
|
||||
image_export.parse_type_map_file(map_file_path)
|
||||
)
|
||||
|
||||
# assert a multi-entry file is correctly parsed
|
||||
multi_map_file = '''URI: latest
|
||||
|
||||
Content-Type: application/vnd.docker.distribution.manifest.v2+json
|
||||
URI: sha256:1234abcd/index.json
|
||||
|
||||
Content-Type: application/vnd.docker.distribution.manifest.list.v2+json
|
||||
URI: sha256:eeeeeeee/index.json
|
||||
|
||||
'''
|
||||
with open(map_file_path, 'w+') as f:
|
||||
f.write(multi_map_file)
|
||||
self.assertEqual(
|
||||
{
|
||||
'application/vnd.docker.distribution.manifest.v2+json':
|
||||
'sha256:1234abcd/index.json',
|
||||
'application/vnd.docker.distribution.manifest.list.v2+json':
|
||||
'sha256:eeeeeeee/index.json'
|
||||
},
|
||||
image_export.parse_type_map_file(map_file_path)
|
||||
)
|
||||
|
||||
def test_migrate_to_type_map_file(self):
|
||||
manifest_dir_path = os.path.join(
|
||||
image_export.IMAGE_EXPORT_DIR,
|
||||
'v2/foo/bar/manifests'
|
||||
)
|
||||
map_file_path = os.path.join(
|
||||
image_export.IMAGE_EXPORT_DIR,
|
||||
manifest_dir_path, 'latest.type-map'
|
||||
)
|
||||
symlink_path = os.path.join(
|
||||
image_export.IMAGE_EXPORT_DIR,
|
||||
manifest_dir_path, 'latest'
|
||||
)
|
||||
manifest_path = os.path.join(
|
||||
image_export.IMAGE_EXPORT_DIR,
|
||||
manifest_dir_path, 'sha256:1234abcd'
|
||||
)
|
||||
image_export.make_dir(manifest_dir_path)
|
||||
# create legacy symlink
|
||||
os.symlink(manifest_path, symlink_path)
|
||||
|
||||
# run the migration
|
||||
image_export.migrate_to_type_map_file('foo/bar', symlink_path)
|
||||
|
||||
expected_map_file = '''URI: latest
|
||||
|
||||
Content-Type: application/vnd.docker.distribution.manifest.v2+json
|
||||
URI: sha256:1234abcd/index.json
|
||||
|
||||
'''
|
||||
# assert the migrated file contains the expected content
|
||||
with open(map_file_path, 'r') as f:
|
||||
self.assertEqual(expected_map_file, f.read())
|
||||
|
||||
def _write_test_image(self, url, manifest):
|
||||
image, tag = image_uploader.BaseImageUploader._image_tag_from_url(
|
||||
url)
|
||||
@ -256,18 +345,13 @@ Header set ETag "%s"
|
||||
f.write('The Blob')
|
||||
return manifest_digest
|
||||
|
||||
def assertFiles(self, dirs, files, symlinks, deleted):
|
||||
def assertFiles(self, dirs, files, deleted):
|
||||
for d in dirs:
|
||||
self.assertTrue(os.path.isdir(d), 'is dir: %s' % d)
|
||||
for f in files:
|
||||
self.assertTrue(os.path.isfile(f), 'is file: %s' % f)
|
||||
for d in deleted:
|
||||
self.assertFalse(os.path.exists(d), 'not exists: %s' % d)
|
||||
for l, f in symlinks.items():
|
||||
self.assertTrue(os.path.islink(l), 'is link: %s' % l)
|
||||
self.assertEqual(f, os.readlink(l))
|
||||
self.assertTrue(os.path.exists(f),
|
||||
'link target exists: %s' % f)
|
||||
self.assertFalse(os.path.exists(d), 'deleted still exists: %s' % d)
|
||||
|
||||
def test_delete_image(self):
|
||||
url1 = urlparse('docker://localhost:8787/t/nova-api:latest')
|
||||
@ -309,7 +393,7 @@ Header set ETag "%s"
|
||||
blob_dir = os.path.join(image_dir, 'blobs')
|
||||
m_dir = os.path.join(image_dir, 'manifests')
|
||||
|
||||
# assert every directory, file and symlink for the 2 images
|
||||
# assert every directory and file for the 2 images
|
||||
self.assertFiles(
|
||||
dirs=[
|
||||
v2_dir,
|
||||
@ -326,11 +410,9 @@ Header set ETag "%s"
|
||||
os.path.join(blob_dir, 'sha256:4dc536.gz'),
|
||||
os.path.join(blob_dir, 'sha256:5678'),
|
||||
os.path.join(blob_dir, 'sha256:eeeeee.gz'),
|
||||
os.path.join(m_dir, 'latest.type-map'),
|
||||
os.path.join(m_dir, 'abc.type-map'),
|
||||
],
|
||||
symlinks={
|
||||
os.path.join(m_dir, 'latest'): os.path.join(m_dir, m1_digest),
|
||||
os.path.join(m_dir, 'abc'): os.path.join(m_dir, m2_digest),
|
||||
},
|
||||
deleted=[]
|
||||
)
|
||||
|
||||
@ -349,10 +431,8 @@ Header set ETag "%s"
|
||||
os.path.join(m_dir, m1_digest, 'index.json'),
|
||||
os.path.join(blob_dir, 'sha256:aeb786.gz'),
|
||||
os.path.join(blob_dir, 'sha256:4dc536.gz'),
|
||||
os.path.join(m_dir, 'latest.type-map'),
|
||||
],
|
||||
symlinks={
|
||||
os.path.join(m_dir, 'latest'): os.path.join(m_dir, m1_digest),
|
||||
},
|
||||
deleted=[
|
||||
os.path.join(m_dir, 'abc'),
|
||||
os.path.join(m_dir, m2_digest),
|
||||
@ -370,7 +450,6 @@ Header set ETag "%s"
|
||||
v2_dir,
|
||||
],
|
||||
files=[],
|
||||
symlinks={},
|
||||
deleted=[
|
||||
image_dir,
|
||||
blob_dir,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user