openstack/tripleo-common
Code Issues Proposed changes

Replace skopeo inspect with python

Browse Source 
This replaces the skopeo inspect calls with python equivalent. It is
faster than the skopeo inspect for two reasons:
- the auth token is shared for all requests
- the tags list request is made concurrently

This should also help with running the dry-run prepare in the mistral
podman container since /run is not involved at all in this
implementation.

Change-Id: Ia898d0acfdeac1699e7e08e2935a2a4eaf578531
Closes-Bug: #1797114
This commit is contained in:
Steve Baker 2018-10-11 17:38:50 +13:00 committed by Emilien Macchi
parent b4053ad111
commit 28a5ba0a01
3 changed files with 383 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
test-requirements.txt
View File

@ -18,3 +18,4 @@ PyYAML>=3.12 # MIT
reno>=2.5.0 # Apache-2.0
urllib3>=1.21.1 # MIT
bashate>=0.2 # Apache-2.0
requests-mock>=1.2.0 # Apache-2.0

237
tripleo_common/image/image_uploader.py
View File

@ -19,10 +19,12 @@ from concurrent import futures
import json
import netifaces
import os
import re
import requests
from requests import auth as requests_auth
import shutil
import six
from six.moves.urllib.parse import urlparse
from six.moves.urllib import parse
import subprocess
import tempfile
import tenacity
@ -47,6 +49,7 @@ LOG = logging.getLogger(__name__)
SECURE_REGISTRIES = (
'trunk.registry.rdoproject.org',
'docker.io',
'registry-1.docker.io',
)
CLEANUP = (
@ -84,10 +87,12 @@ class ImageUploadManager(BaseImageManager):
self.dry_run = dry_run
self.cleanup = cleanup
def discover_image_tag(self, image, tag_from_label=None):
def discover_image_tag(self, image, tag_from_label=None,
username=None, password=None):
uploader = self.uploader(DEFAULT_UPLOADER)
return uploader.discover_image_tag(
image, tag_from_label=tag_from_label)
image, tag_from_label=tag_from_label,
username=username, password=password)
def uploader(self, uploader):
if uploader not in self.uploaders:
@ -162,7 +167,8 @@ class ImageUploader(object):
pass
@abc.abstractmethod
def discover_image_tag(self, image, tag_from_label=None):
def discover_image_tag(self, image, tag_from_label=None,
username=None, password=None):
"""Discover a versioned tag for an image"""
pass
@ -267,10 +273,10 @@ class BaseImageUploader(ImageUploader):
'Modifying image %s failed' % target_image)
@staticmethod
def _images_match(image1, image2, insecure_registries):
def _images_match(image1, image2, insecure_registries, session1=None):
try:
image1_digest = BaseImageUploader._image_digest(
image1, insecure_registries)
image1, insecure_registries, session=session1)
except Exception:
return False
try:
@ -285,27 +291,74 @@ class BaseImageUploader(ImageUploader):
return image1_digest == image2_digest
@staticmethod
def _image_digest(image, insecure_registries):
def _image_digest(image, insecure_registries, session=None):
image_url = BaseImageUploader._image_to_url(image)
insecure = image_url.netloc in insecure_registries
i = BaseImageUploader._inspect(image_url, insecure)
i = BaseImageUploader._inspect(image_url, insecure, session)
return i.get('Digest')
@staticmethod
def _image_labels(image, insecure):
image_url = BaseImageUploader._image_to_url(image)
i = BaseImageUploader._inspect(image_url, insecure)
def _image_labels(image_url, insecure, session=None):
i = BaseImageUploader._inspect(image_url, insecure, session)
return i.get('Labels', {}) or {}
@staticmethod
def _image_exists(image, insecure_registries):
def _image_exists(image, insecure_registries, session=None):
try:
BaseImageUploader._image_digest(image, insecure_registries)
BaseImageUploader._image_digest(
image, insecure_registries, session=session)
except ImageNotFoundException:
return False
else:
return True
@staticmethod
def authenticate(image_url, username=None, password=None, insecure=False):
image_url = BaseImageUploader._fix_dockerio_url(image_url)
netloc = image_url.netloc
if insecure:
scheme = 'http'
else:
scheme = 'https'
image, tag = image_url.path.split(':')
url = '%s://%s/v2/' % (scheme, netloc)
session = requests.Session()
r = session.get(url, timeout=30)
LOG.debug('%s status code %s' % (url, r.status_code))
if r.status_code != 401:
return session
if 'www-authenticate' not in r.headers:
raise ImageUploaderException(
'Unknown authentication method for headers: %s' % r.headers)
www_auth = r.headers['www-authenticate']
if not www_auth.startswith('Bearer '):
raise ImageUploaderException(
'Unknown www-authenticate value: %s' % www_auth)
token_param = {}
realm = re.search('realm="(.*?)"', www_auth).group(1)
token_param['service'] = re.search(
'service="(.*?)"', www_auth).group(1)
token_param['scope'] = 'repository:%s:pull' % image[1:]
auth = None
if username:
auth = requests_auth.HTTPBasicAuth(username, password)
rauth = session.get(realm, params=token_param, auth=auth, timeout=30)
rauth.raise_for_status()
session.headers['Authorization'] = 'Bearer %s' % rauth.json()['token']
return session
@staticmethod
def _fix_dockerio_url(url):
one = 'docker.io'
two = 'registry-1.docker.io'
if url.netloc != one:
return url
return parse.ParseResult(url.scheme, two,
url.path, url.params,
url.query, url.fragment)
@staticmethod
@tenacity.retry( # Retry up to 5 times with jittered exponential backoff
reraise=True,
@ -313,39 +366,79 @@ class BaseImageUploader(ImageUploader):
wait=tenacity.wait_random_exponential(multiplier=1, max=10),
stop=tenacity.stop_after_attempt(5)
)
def _inspect(image_url, insecure=False):
image = image_url.geturl()
cmd = ['skopeo', 'inspect']
def _inspect(image_url, insecure=False, session=None):
original_image_url = image_url
image_url = BaseImageUploader._fix_dockerio_url(image_url)
parts = {
'netloc': image_url.netloc
}
if insecure:
cmd.append('--tls-verify=false')
cmd.append(image)
parts['scheme'] = 'http'
else:
parts['scheme'] = 'https'
image, tag = image_url.path.split(':')
parts['image'] = image
parts['tag'] = tag
LOG.info('Running %s' % ' '.join(cmd))
env = os.environ.copy()
process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
manifest_url = ('%(scheme)s://%(netloc)s/v2'
'%(image)s/manifests/%(tag)s' % parts)
tags_url = ('%(scheme)s://%(netloc)s/v2'
'%(image)s/tags/list' % parts)
manifest_headers = {
'Accept': 'application/vnd.docker.distribution.manifest.v2+json'
}
out, err = process.communicate()
if process.returncode != 0:
not_found_msgs = (
u'manifest unknown',
# returned by docker.io
u'requested access to the resource is denied'
)
if any(n in err for n in not_found_msgs):
raise ImageNotFoundException('Not found image: %s\n%s' %
(image, err))
raise ImageUploaderException('Error inspecting image: %s\n%s' %
(image, err))
return json.loads(out)
p = futures.ThreadPoolExecutor(max_workers=2)
manifest_f = p.submit(
session.get, manifest_url, headers=manifest_headers, timeout=30)
tags_f = p.submit(session.get, tags_url, timeout=30)
manifest_r = manifest_f.result()
tags_r = tags_f.result()
if manifest_r.status_code == 404:
raise ImageNotFoundException('Not found image: %s' %
image_url.geturl())
manifest_r.raise_for_status()
tags_r.raise_for_status()
manifest = manifest_r.json()
layers = [l['digest'] for l in manifest['layers']]
parts['config_digest'] = manifest['config']['digest']
config_headers = {
'Accept': manifest['config']['mediaType']
}
config_url = ('%(scheme)s://%(netloc)s/v2'
'%(image)s/blobs/%(config_digest)s' % parts)
config_f = p.submit(
session.get, config_url, headers=config_headers, timeout=30)
config_r = config_f.result()
config_r.raise_for_status()
tags = tags_r.json()['tags']
digest = manifest_r.headers['Docker-Content-Digest']
config = config_r.json()
name = '%s%s' % (original_image_url.netloc, image)
return {
'Name': name,
'Digest': digest,
'RepoTags': tags,
'Created': config['created'],
'DockerVersion': config['docker_version'],
'Labels': config['config']['Labels'],
'Architecture': config['architecture'],
'Os': config['os'],
'Layers': layers,
}
@staticmethod
def _image_to_url(image):
if '://' not in image:
image = 'docker://' + image
return urlparse(image)
url = parse.urlparse(image)
return url
@staticmethod
def _discover_tag_from_inspect(i, image, tag_from_label=None,
@ -412,20 +505,25 @@ class BaseImageUploader(ImageUploader):
return versioned_images
def discover_image_tag(self, image, tag_from_label=None,
fallback_tag=None):
fallback_tag=None, username=None, password=None):
image_url = self._image_to_url(image)
insecure = self.is_insecure_registry(image_url.netloc)
i = self._inspect(image_url, insecure)
session = self.authenticate(
image_url, insecure=insecure, username=username, password=password)
i = self._inspect(image_url, insecure, session)
return self._discover_tag_from_inspect(i, image, tag_from_label,
fallback_tag)
def filter_images_with_labels(self, images, labels):
def filter_images_with_labels(self, images, labels,
username=None, password=None):
images_with_labels = []
for image in images:
url = self._image_to_url(image)
image_labels = self._image_labels(url.geturl(),
self.is_insecure_registry(
url.netloc))
insecure = self.is_insecure_registry(url.netloc)
session = self.authenticate(
url, insecure=insecure, username=username, password=password)
image_labels = self._image_labels(
url, insecure=insecure, session=session)
if set(labels).issubset(set(image_labels)):
images_with_labels.append(image)
@ -465,7 +563,7 @@ class BaseImageUploader(ImageUploader):
@staticmethod
def _cross_repo_mount(target_image_url, image_layers,
source_layers, insecure_registries):
source_layers, insecure_registries, session):
netloc = target_image_url.netloc
name = target_image_url.path.split(':')[0][1:]
if netloc in insecure_registries:
@ -483,7 +581,7 @@ class BaseImageUploader(ImageUploader):
'mount': layer,
'from': existing_name
}
r = requests.post(url, data=data)
r = session.post(url, data=data)
LOG.debug('%s %s' % (r.status_code, r.reason))
@ -500,24 +598,34 @@ class DockerImageUploader(BaseImageUploader):
source_tag = names['source_tag']
repo = names['repo']
source_image = names['source_image']
source_image_url = BaseImageUploader._image_to_url(source_image)
source_insecure = source_image_url.netloc in insecure_registries
target_image_no_tag = names['target_image_no_tag']
append_tag = names['append_tag']
target_tag = names['target_tag']
target_image_source_tag = names['target_image_source_tag']
target_image = names['target_image']
target_image_url = BaseImageUploader._image_to_url(target_image)
target_insecure = target_image_url.netloc in insecure_registries
if dry_run:
return []
if modify_role:
target_session = BaseImageUploader.authenticate(
target_image_url, insecure=target_insecure)
if BaseImageUploader._image_exists(target_image,
insecure_registries):
insecure_registries,
session=target_session):
LOG.warning('Skipping upload for modified image %s' %
target_image)
return []
else:
source_session = BaseImageUploader.authenticate(
source_image_url, insecure=source_insecure)
if BaseImageUploader._images_match(source_image, target_image,
insecure_registries):
insecure_registries,
session1=source_session):
LOG.warning('Skipping upload for image %s' % image_name)
return []
@ -631,29 +739,40 @@ class SkopeoImageUploader(BaseImageUploader):
source_image = names['source_image']
source_image_url = BaseImageUploader._image_to_url(source_image)
source_image_local_url = urlparse('containers-storage:%s'
% source_image)
source_image_local_url = parse.urlparse('containers-storage:%s'
% source_image)
source_insecure = source_image_url.netloc in insecure_registries
append_tag = names['append_tag']
target_image_source_tag = names['target_image_source_tag']
target_image = names['target_image']
target_image_url = BaseImageUploader._image_to_url(target_image)
target_image_local_url = urlparse('containers-storage:%s' %
target_image)
target_image_local_url = parse.urlparse('containers-storage:%s' %
target_image)
target_insecure = target_image_local_url.netloc in insecure_registries
if dry_run:
return []
target_session = BaseImageUploader.authenticate(
target_image_url, insecure=target_insecure)
if modify_role and BaseImageUploader._image_exists(
target_image, insecure_registries):
target_image, insecure_registries, target_session):
LOG.warning('Skipping upload for modified image %s' %
target_image)
return []
source_inspect = BaseImageUploader._inspect(source_image_url)
source_session = BaseImageUploader.authenticate(
source_image_url, insecure=source_insecure)
source_inspect = BaseImageUploader._inspect(
source_image_url, insecure=source_insecure, session=source_session)
source_layers = source_inspect.get('Layers', [])
BaseImageUploader._cross_repo_mount(
target_image_url, image_layers, source_layers, insecure_registries)
target_image_url, image_layers, source_layers, insecure_registries,
session=source_session)
to_cleanup = []
if modify_role:
@ -671,8 +790,6 @@ class SkopeoImageUploader(BaseImageUploader):
modify_role, modify_vars, source_image,
target_image_source_tag, append_tag,
container_build_tool='buildah')
# Inspect to confirm the playbook created the target image
BaseImageUploader._inspect(target_image_local_url)
if cleanup == CLEANUP_FULL:
to_cleanup.append(target_image)
@ -756,7 +873,7 @@ class SkopeoImageUploader(BaseImageUploader):
if not image:
continue
LOG.warning('Removing local copy of %s' % image)
image_url = urlparse('containers-storage:%s' % image)
image_url = parse.urlparse('containers-storage:%s' % image)
SkopeoImageUploader._delete(image_url)
def run_tasks(self):
@ -795,7 +912,9 @@ def discover_tag_from_inspect(args):
image, tag_from_label, insecure_registries = args
image_url = BaseImageUploader._image_to_url(image)
insecure = image_url.netloc in insecure_registries
i = BaseImageUploader._inspect(image_url, insecure)
session = BaseImageUploader.authenticate(image_url, insecure=insecure)
i = BaseImageUploader._inspect(image_url, insecure=insecure,
session=session)
if ':' in image_url.path:
# break out the tag from the url to be the fallback tag
path = image.rpartition(':')

313
tripleo_common/tests/image/test_image_uploader.py
View File

@ -13,11 +13,11 @@
# under the License.
#
import json
import mock
import operator
import os
import requests
from requests_mock.contrib import fixture as rm_fixture
import six
from six.moves.urllib.parse import urlparse
import tempfile
@ -50,6 +50,8 @@ class TestImageUploadManager(base.TestCase):
files.append('testfile')
self.filelist = files
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
@mock.patch('tripleo_common.image.base.open',
@ -66,7 +68,8 @@ class TestImageUploadManager(base.TestCase):
@mock.patch('tripleo_common.image.image_uploader.'
'get_undercloud_registry', return_value='192.0.2.0:8787')
def test_file_parsing(self, mock_gur, mockdocker, mockioctl, mockpath,
mock_images_match, mock_is_insecure, mock_inspect):
mock_images_match, mock_is_insecure, mock_inspect,
mock_auth):
mock_inspect.return_value = {}
manager = image_uploader.ImageUploadManager(self.filelist, debug=True)
@ -171,89 +174,101 @@ class TestBaseImageUploader(base.TestCase):
super(TestBaseImageUploader, self).setUp()
self.uploader = image_uploader.BaseImageUploader()
self.uploader._inspect.retry.sleep = mock.Mock()
self.requests = self.useFixture(rm_fixture.Fixture())
@mock.patch('requests.get')
def test_is_insecure_registry_known(self, mock_get):
def test_is_insecure_registry_known(self):
self.assertFalse(
self.uploader.is_insecure_registry('docker.io'))
@mock.patch('requests.get')
def test_is_insecure_registry_secure(self, mock_get):
def test_is_insecure_registry_secure(self):
self.assertFalse(
self.uploader.is_insecure_registry('192.0.2.0:8787'))
self.assertFalse(
self.uploader.is_insecure_registry('192.0.2.0:8787'))
mock_get.assert_called_once_with('https://192.0.2.0:8787/')
self.assertEqual(
'https://192.0.2.0:8787/',
self.requests.request_history[0].url
)
@mock.patch('requests.get')
def test_is_insecure_registry_timeout(self, mock_get):
mock_get.side_effect = requests.exceptions.ReadTimeout('ouch')
def test_is_insecure_registry_timeout(self):
self.requests.get(
'https://192.0.2.0:8787/',
exc=requests.exceptions.ReadTimeout('ouch'))
self.assertFalse(
self.uploader.is_insecure_registry('192.0.2.0:8787'))
self.assertFalse(
self.uploader.is_insecure_registry('192.0.2.0:8787'))
mock_get.assert_called_once_with('https://192.0.2.0:8787/')
self.assertEqual(
'https://192.0.2.0:8787/',
self.requests.request_history[0].url
)
@mock.patch('requests.get')
def test_is_insecure_registry_insecure(self, mock_get):
mock_get.side_effect = requests.exceptions.SSLError('ouch')
def test_is_insecure_registry_insecure(self):
self.requests.get(
'https://192.0.2.0:8787/',
exc=requests.exceptions.SSLError('ouch'))
self.assertTrue(
self.uploader.is_insecure_registry('192.0.2.0:8787'))
self.assertTrue(
self.uploader.is_insecure_registry('192.0.2.0:8787'))
mock_get.assert_called_once_with('https://192.0.2.0:8787/')
self.assertEqual(
'https://192.0.2.0:8787/',
self.requests.request_history[0].url
)
@mock.patch('subprocess.Popen')
def test_discover_image_tag(self, mock_popen):
result = {
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
def test_discover_image_tag(self, mock_inspect, mock_auth):
mock_inspect.return_value = {
'Labels': {
'rdo_version': 'a',
'build_version': '4.0.0'
},
'RepoTags': ['a']
}
mock_process = mock.Mock()
mock_process.communicate.return_value = (json.dumps(result), '')
mock_process.returncode = 0
mock_popen.return_value = mock_process
self.assertEqual(
'a',
self.uploader.discover_image_tag('docker.io/t/foo', 'rdo_version')
self.uploader.discover_image_tag('docker.io/t/foo:b',
'rdo_version')
)
# no tag_from_label specified
self.assertRaises(
ImageUploaderException,
self.uploader.discover_image_tag,
'docker.io/t/foo')
'docker.io/t/foo:b')
# missing RepoTags entry
self.assertRaises(
ImageUploaderException,
self.uploader.discover_image_tag,
'docker.io/t/foo',
'docker.io/t/foo:b',
'build_version')
# missing Labels entry
self.assertRaises(
ImageUploaderException,
self.uploader.discover_image_tag,
'docker.io/t/foo',
'docker.io/t/foo:b',
'version')
# inspect call failed
mock_process.returncode = 1
mock_process.communicate.return_value = ('', 'manifest unknown')
mock_inspect.side_effect = ImageNotFoundException()
self.assertRaises(
ImageNotFoundException,
self.uploader.discover_image_tag,
'docker.io/t/foo',
'docker.io/t/foo:b',
'rdo_version')
@mock.patch('subprocess.Popen')
def test_discover_tag_from_inspect(self, mock_popen):
result = {
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
def test_discover_tag_from_inspect(self, mock_inspect, mock_auth):
mock_inspect.return_value = {
'Labels': {
'rdo_version': 'a',
'build_version': '4.0.0',
@ -262,10 +277,6 @@ class TestBaseImageUploader(base.TestCase):
},
'RepoTags': ['a', '1.0.0-20180125']
}
mock_process = mock.Mock()
mock_process.communicate.return_value = (json.dumps(result), '')
mock_process.returncode = 0
mock_popen.return_value = mock_process
sr = image_uploader.SECURE_REGISTRIES
# simple label -> tag
@ -332,7 +343,7 @@ class TestBaseImageUploader(base.TestCase):
)
# inspect call failed
mock_process.returncode = 1
mock_inspect.side_effect = ImageUploaderException()
self.assertRaises(
ImageUploaderException,
image_uploader.discover_tag_from_inspect,
@ -388,6 +399,119 @@ class TestBaseImageUploader(base.TestCase):
mock_inspect.side_effect = ImageUploaderException()
self.assertFalse(self.uploader._images_match('foo', 'bar', set()))
def test_authenticate(self):
req = self.requests
auth = image_uploader.BaseImageUploader.authenticate
url1 = urlparse('docker://docker.io/t/nova-api:latest')
# no auth required
req.get('https://registry-1.docker.io/v2/', status_code=200)
self.assertNotIn('Authorization', auth(url1).headers)
# missing 'www-authenticate' header
req.get('https://registry-1.docker.io/v2/', status_code=401)
self.assertRaises(ImageUploaderException, auth, url1)
# unknown 'www-authenticate' header
req.get('https://registry-1.docker.io/v2/', status_code=401,
headers={'www-authenticate': 'Foo'})
self.assertRaises(ImageUploaderException, auth, url1)
# successful auth requests
headers = {
'www-authenticate': 'Bearer '
'realm="https://auth.docker.io/token",'
'service="registry.docker.io"'
}
req.get('https://registry-1.docker.io/v2/', status_code=401,
headers=headers)
req.get('https://auth.docker.io/token', json={"token": "asdf1234"})
self.assertEqual(
'Bearer asdf1234',
auth(url1).headers['Authorization']
)
def test_fix_dockerio_url(self):
url1 = urlparse('docker://docker.io/t/nova-api:latest')
url2 = urlparse('docker://registry-1.docker.io/t/nova-api:latest')
url3 = urlparse('docker://192.0.2.1:8787/t/nova-api:latest')
fix = image_uploader.BaseImageUploader._fix_dockerio_url
# fix urls
self.assertEqual(url2, fix(url1))
# no change urls
self.assertEqual(url2, fix(url2))
self.assertEqual(url3, fix(url3))
def test_inspect(self):
req = self.requests
session = requests.Session()
session.headers['Authorization'] = 'Bearer asdf1234'
inspect = image_uploader.BaseImageUploader._inspect
url1 = urlparse('docker://docker.io/t/nova-api:latest')
manifest_resp = {
'config': {
'mediaType': 'text/html',
'digest': 'abcdef'
},
'layers': [
{'digest': 'aaa'},
{'digest': 'bbb'},
{'digest': 'ccc'},
]
}
manifest_headers = {'Docker-Content-Digest': 'eeeeee'}
tags_resp = {'tags': ['one', 'two', 'latest']}
config_resp = {
'created': '2018-10-02T11:13:45.567533229Z',
'docker_version': '1.13.1',
'config': {
'Labels': {
'build-date': '20181002',
'build_id': '1538477701',
'kolla_version': '7.0.0'
}
},
'architecture': 'amd64',
'os': 'linux',
}
req.get('https://registry-1.docker.io/v2/t/nova-api/tags/list',
json=tags_resp)
req.get('https://registry-1.docker.io/v2/t/nova-api/blobs/abcdef',
json=config_resp)
# test 404 response
req.get('https://registry-1.docker.io/v2/t/nova-api/manifests/latest',
status_code=404)
self.assertRaises(ImageNotFoundException, inspect, url1,
session=session)
# test full response
req.get('https://registry-1.docker.io/v2/t/nova-api/manifests/latest',
json=manifest_resp, headers=manifest_headers)
self.assertEqual(
{
'Architecture': 'amd64',
'Created': '2018-10-02T11:13:45.567533229Z',
'Digest': 'eeeeee',
'DockerVersion': '1.13.1',
'Labels': {
'build-date': '20181002',
'build_id': '1538477701',
'kolla_version': '7.0.0'
},
'Layers': ['aaa', 'bbb', 'ccc'],
'Name': 'docker.io/t/nova-api',
'Os': 'linux',
'RepoTags': ['one', 'two', 'latest']
},
inspect(url1, session=session)
)
class TestDockerImageUploader(base.TestCase):
@ -405,22 +529,18 @@ class TestDockerImageUploader(base.TestCase):
super(TestDockerImageUploader, self).tearDown()
self.patcher.stop()
@mock.patch('subprocess.Popen')
def test_upload_image(self, mock_popen):
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
def test_upload_image(self, mock_inspect, mock_auth):
result1 = {
'Digest': 'a'
}
result2 = {
'Digest': 'b'
}
mock_process = mock.Mock()
mock_process.communicate.side_effect = [
(json.dumps(result1), ''),
(json.dumps(result2), ''),
]
mock_process.returncode = 0
mock_popen.return_value = mock_process
mock_inspect.side_effect = [result1, result2]
image = 'docker.io/tripleomaster/heat-docker-agents-centos'
tag = 'latest'
@ -457,46 +577,14 @@ class TestDockerImageUploader(base.TestCase):
push_image,
tag=tag, stream=True)
@mock.patch('subprocess.Popen')
def test_upload_image_missing_tag(self, mock_popen):
image = 'docker.io/tripleomaster/heat-docker-agents-centos'
expected_tag = 'latest'
push_destination = 'localhost:8787'
push_image = 'localhost:8787/tripleomaster/heat-docker-agents-centos'
self.uploader.upload_image(image,
None,
push_destination,
set(),
None,
None,
None,
False,
'full',
{})
self.dockermock.assert_called_once_with(
base_url='unix://var/run/docker.sock', version='auto')
self.dockermock.return_value.pull.assert_called_once_with(
image, tag=expected_tag, stream=True)
self.dockermock.return_value.tag.assert_called_once_with(
image=image + ':' + expected_tag,
repository=push_image,
tag=expected_tag, force=True)
self.dockermock.return_value.push.assert_called_once_with(
push_image,
tag=expected_tag, stream=True)
@mock.patch('subprocess.Popen')
def test_upload_image_existing(self, mock_popen):
result = {
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
def test_upload_image_existing(self, mock_inspect, mock_auth):
mock_inspect.return_value = {
'Digest': 'a'
}
mock_process = mock.Mock()
mock_process.communicate.return_value = (json.dumps(result), '')
mock_process.returncode = 0
mock_popen.return_value = mock_process
image = 'docker.io/tripleomaster/heat-docker-agents-centos'
tag = 'latest'
push_destination = 'localhost:8787'
@ -523,16 +611,14 @@ class TestDockerImageUploader(base.TestCase):
self.dockermock.return_value.tag.assert_not_called()
self.dockermock.return_value.push.assert_not_called()
@mock.patch('subprocess.Popen')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
@mock.patch('tripleo_common.actions.'
'ansible.AnsiblePlaybookAction', autospec=True)
def test_modify_upload_image(self, mock_ansible, mock_popen):
mock_process = mock.Mock()
mock_process.communicate.return_value = (
'', 'FATA[0000] Error reading manifest: manifest unknown')
mock_process.returncode = 1
mock_popen.return_value = mock_process
def test_modify_upload_image(self, mock_ansible, mock_inspect, mock_auth):
mock_inspect.side_effect = ImageNotFoundException()
mock_ansible.return_value.run.return_value = {}
image = 'docker.io/tripleomaster/heat-docker-agents-centos'
@ -591,15 +677,14 @@ class TestDockerImageUploader(base.TestCase):
tag=tag + append_tag,
stream=True)
@mock.patch('subprocess.Popen')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
@mock.patch('tripleo_common.actions.'
'ansible.AnsiblePlaybookAction', autospec=True)
def test_modify_image_failed(self, mock_ansible, mock_popen):
mock_process = mock.Mock()
mock_process.communicate.return_value = ('', 'manifest unknown')
mock_process.returncode = 1
mock_popen.return_value = mock_process
def test_modify_image_failed(self, mock_ansible, mock_inspect, mock_auth):
mock_inspect.side_effect = ImageNotFoundException()
image = 'docker.io/tripleomaster/heat-docker-agents-centos'
tag = 'latest'
@ -655,11 +740,14 @@ class TestDockerImageUploader(base.TestCase):
mock_process.communicate.assert_not_called()
self.assertEqual([], result)
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
@mock.patch('tripleo_common.actions.'
'ansible.AnsiblePlaybookAction', autospec=True)
def test_modify_image_existing(self, mock_ansible, mock_inspect):
def test_modify_image_existing(self, mock_ansible, mock_inspect,
mock_auth):
mock_inspect.return_value = {'Digest': 'a'}
image = 'docker.io/tripleomaster/heat-docker-agents-centos'
@ -772,7 +860,10 @@ class TestSkopeoImageUploader(base.TestCase):
@mock.patch('subprocess.Popen')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
def test_upload_image(self, mock_inspect, mock_popen, mock_environ):
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
def test_upload_image(self, mock_auth, mock_inspect,
mock_popen, mock_environ):
mock_process = mock.Mock()
mock_process.communicate.return_value = ('copy complete', '')
mock_process.returncode = 0
@ -807,6 +898,8 @@ class TestSkopeoImageUploader(base.TestCase):
env={}, stdout=-1
)
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
@mock.patch('tripleo_common.image.image_uploader.'
@ -816,7 +909,7 @@ class TestSkopeoImageUploader(base.TestCase):
@mock.patch('tripleo_common.actions.'
'ansible.AnsiblePlaybookAction', autospec=True)
def test_modify_upload_image(self, mock_ansible, mock_exists, mock_copy,
mock_inspect):
mock_inspect, mock_auth):
mock_exists.return_value = False
mock_inspect.return_value = {}
with tempfile.NamedTemporaryFile(delete=False) as logfile:
@ -868,10 +961,7 @@ class TestSkopeoImageUploader(base.TestCase):
mock_inspect.assert_has_calls([
mock.call(urlparse(
'docker://docker.io/t/nova-api:latest'
)),
mock.call(urlparse(
'containers-storage:localhost:8787/t/nova-api:latestmodify-123'
))
), insecure=False, session=mock.ANY)
])
mock_copy.assert_has_calls([
mock.call(
@ -894,6 +984,8 @@ class TestSkopeoImageUploader(base.TestCase):
extra_env_variables=mock.ANY
)
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
@mock.patch('tripleo_common.image.image_uploader.'
@ -903,7 +995,7 @@ class TestSkopeoImageUploader(base.TestCase):
@mock.patch('tripleo_common.actions.'
'ansible.AnsiblePlaybookAction', autospec=True)
def test_modify_image_failed(self, mock_ansible, mock_exists, mock_copy,
mock_inspect):
mock_inspect, mock_auth):
mock_exists.return_value = False
mock_inspect.return_value = {}
@ -959,11 +1051,14 @@ class TestSkopeoImageUploader(base.TestCase):
mock_process.communicate.assert_not_called()
self.assertEqual([], result)
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader.authenticate')
@mock.patch('tripleo_common.image.image_uploader.'
'BaseImageUploader._inspect')
@mock.patch('tripleo_common.actions.'
'ansible.AnsiblePlaybookAction', autospec=True)
def test_modify_image_existing(self, mock_ansible, mock_inspect):
def test_modify_image_existing(self, mock_ansible, mock_inspect,
mock_auth):
mock_inspect.return_value = {'Digest': 'a'}
image = 'docker.io/t/nova-api'