openstack/tripleo-common
Code Issues Proposed changes

Merge "Implement image customization during upload"

Browse Source
This commit is contained in:
Zuul 2018-05-15 23:29:32 +00:00 committed by Gerrit Code Review
commit e4becc4545
4 changed files with 235 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
tripleo_common/image/image_uploader.py
View File

@ -21,10 +21,13 @@ import logging
import netifaces
import os
import requests
import shutil
import six
from six.moves import urllib
import subprocess
import tempfile
import tenacity
import yaml
import docker
try:
@ -32,6 +35,7 @@ try:
except ImportError:
from docker import Client
from oslo_concurrency import processutils
from tripleo_common.actions import ansible
from tripleo_common.image.base import BaseImageManager
from tripleo_common.image.exception import ImageUploaderException
@ -96,9 +100,13 @@ class ImageUploadManager(BaseImageManager):
# This updates the parsed upload_images dict with real values
item['push_destination'] = push_destination
append_tag = item.get('append_tag')
modify_role = item.get('modify_role')
modify_vars = item.get('modify_vars')
self.uploader(uploader).add_upload_task(
image_name, pull_source, push_destination)
image_name, pull_source, push_destination,
append_tag, modify_role, modify_vars)
for uploader in self.uploaders.values():
uploader.run_tasks()
@ -122,7 +130,8 @@ class ImageUploader(object):
pass
@abc.abstractmethod
def add_upload_task(self, image_name, pull_source, push_destination):
def add_upload_task(self, image_name, pull_source, push_destination,
append_tag, modify_role, modify_vars):
"""Add an upload task to be executed later"""
pass
@ -150,41 +159,90 @@ class DockerImageUploader(ImageUploader):
self.secure_registries = set(SECURE_REGISTRIES)
self.insecure_registries = set()
@staticmethod
def run_modify_playbook(modify_role, modify_vars,
source_image, target_image, append_tag):
vars = {}
if modify_vars:
vars.update(modify_vars)
vars['source_image'] = source_image
vars['target_image'] = target_image
vars['modified_append_tag'] = append_tag
LOG.debug('Playbook variables: \n%s' % yaml.safe_dump(
vars, default_flow_style=False))
playbook = [{
'hosts': 'localhost',
'tasks': [{
'name': 'Import role %s' % modify_role,
'import_role': {
'name': modify_role
},
'vars': vars
}]
}]
LOG.debug('Playbook: \n%s' % yaml.safe_dump(
playbook, default_flow_style=False))
work_dir = tempfile.mkdtemp(prefix='tripleo-modify-image-playbook-')
try:
action = ansible.AnsiblePlaybookAction(
playbook=playbook,
work_dir=work_dir
)
result = action.run(None)
LOG.debug(result.get('stdout', ''))
finally:
shutil.rmtree(work_dir)
@staticmethod
def upload_image(image_name, pull_source, push_destination,
insecure_registries):
insecure_registries, append_tag, modify_role,
modify_vars):
LOG.info('imagename: %s' % image_name)
dockerc = Client(base_url='unix://var/run/docker.sock', version='auto')
if ':' in image_name:
image = image_name.rpartition(':')[0]
tag = image_name.rpartition(':')[2]
source_tag = image_name.rpartition(':')[2]
else:
image = image_name
tag = 'latest'
source_tag = 'latest'
if pull_source:
repo = pull_source + '/' + image
else:
repo = image
full_image = repo + ':' + tag
new_repo = push_destination + '/' + repo.partition('/')[2]
full_new_repo = new_repo + ':' + tag
source_image = repo + ':' + source_tag
target_image_no_tag = push_destination + '/' + repo.partition('/')[2]
append_tag = append_tag or ''
target_tag = source_tag + append_tag
target_image_source_tag = target_image_no_tag + ':' + source_tag
target_image = target_image_no_tag + ':' + target_tag
if DockerImageUploader._images_match(full_image, full_new_repo,
if DockerImageUploader._images_match(source_image, target_image,
insecure_registries):
LOG.info('Skipping upload for image %s' % image_name)
return []
DockerImageUploader._pull(dockerc, repo, tag=tag)
DockerImageUploader._pull(dockerc, repo, tag=source_tag)
response = dockerc.tag(image=full_image, repository=new_repo,
tag=tag, force=True)
if modify_role:
DockerImageUploader.run_modify_playbook(
modify_role, modify_vars, source_image,
target_image_source_tag, append_tag)
# raise an exception if the playbook didn't tag
# the expected target image
dockerc.inspect_image(target_image)
else:
response = dockerc.tag(
image=source_image, repository=target_image_no_tag,
tag=target_tag, force=True
)
LOG.debug(response)
DockerImageUploader._push(dockerc, new_repo, tag=tag)
DockerImageUploader._push(dockerc, target_image_no_tag, tag=target_tag)
LOG.info('Completed upload for image %s' % image_name)
return full_image, full_new_repo
return source_image, target_image
@staticmethod
@tenacity.retry( # Retry up to 5 times with jittered exponential backoff
@ -353,7 +411,8 @@ class DockerImageUploader(ImageUploader):
else:
LOG.warning(e)
def add_upload_task(self, image_name, pull_source, push_destination):
def add_upload_task(self, image_name, pull_source, push_destination,
append_tag, modify_role, modify_vars):
# prime self.insecure_registries
if pull_source:
self.is_insecure_registry(self._image_to_url(pull_source).netloc)
@ -361,7 +420,8 @@ class DockerImageUploader(ImageUploader):
self.is_insecure_registry(self._image_to_url(image_name).netloc)
self.is_insecure_registry(self._image_to_url(push_destination).netloc)
self.upload_tasks.append((image_name, pull_source, push_destination,
self.insecure_registries))
self.insecure_registries, append_tag,
modify_role, modify_vars))
def run_tasks(self):
if not self.upload_tasks:

35
tripleo_common/image/kolla_builder.py
View File

@ -21,6 +21,7 @@ import re
import subprocess
import sys
import tempfile
import time
import yaml
from tripleo_common.image import base
@ -114,11 +115,18 @@ def container_images_prepare_multi(environment, roles_data):
env_params = {}
service_filter = build_service_filter(environment, roles_data)
modified_timestamp = time.strftime('-modified-%Y%m%d%H%M%S')
for cip_entry in cip:
mapping_args = cip_entry.get('set')
push_destination = cip_entry.get('push_destination')
pull_source = cip_entry.get('pull_source')
modify_role = cip_entry.get('modify_role')
modify_vars = cip_entry.get('modify_vars')
if modify_role:
append_tag = modified_timestamp
else:
append_tag = None
prepare_data = container_images_prepare(
excludes=cip_entry.get('excludes'),
@ -129,16 +137,21 @@ def container_images_prepare_multi(environment, roles_data):
output_env_file='image_params',
output_images_file='upload_data',
tag_from_label=cip_entry.get('tag_from_label'),
append_tag=append_tag,
modify_role=modify_role,
modify_vars=modify_vars
)
env_params.update(prepare_data['image_params'])
if push_destination or pull_source:
if push_destination or pull_source or modify_role:
with tempfile.NamedTemporaryFile(mode='w') as f:
yaml.safe_dump({
'container_images': prepare_data['upload_data']
}, f)
uploader = image_uploader.ImageUploadManager(
[f.name], verbose=True)
[f.name],
verbose=True,
)
uploader.upload()
return env_params
@ -157,7 +170,9 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
excludes=None, service_filter=None,
pull_source=None, push_destination=None,
mapping_args=None, output_env_file=None,
output_images_file=None, tag_from_label=None):
output_images_file=None, tag_from_label=None,
append_tag=None, modify_role=None,
modify_vars=None):
"""Perform container image preparation
:param template_file: path to Jinja2 file containing all image entries
@ -174,12 +189,18 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
:param output_images_file: key to use for image upload data
:param tag_from_label: string when set will trigger tag discovery on every
image
:param append_tag: string to append to the tag for the destination image
:param modify_role: string of ansible role name to run during upload before
the push to destination
:param modify_vars: dict of variables to pass to modify_role
:returns: dict with entries for the supplied output_env_file or
output_images_file
"""
if mapping_args is None:
mapping_args = {}
if not append_tag:
append_tag = ''
if service_filter:
if 'OS::TripleO::Services::OpenDaylightApi' in service_filter:
@ -227,9 +248,15 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
# push_destination, since that is where they will be uploaded to
image = imagename.partition('/')[2]
imagename = '/'.join((push_destination, image))
if append_tag:
entry['append_tag'] = append_tag
if modify_role:
entry['modify_role'] = modify_role
if modify_vars:
entry['modify_vars'] = modify_vars
if 'params' in entry:
for p in entry.pop('params'):
params[p] = imagename
params[p] = imagename + append_tag
if 'services' in entry:
del(entry['services'])

119
tripleo_common/tests/image/test_image_uploader.py
View File

@ -20,6 +20,7 @@ import requests
import six
import urllib3
from oslo_concurrency import processutils
from tripleo_common.image.exception import ImageUploaderException
from tripleo_common.image import image_uploader
from tripleo_common.tests import base
@ -164,7 +165,10 @@ class TestDockerImageUploader(base.TestCase):
self.uploader.upload_image(image + ':' + tag,
None,
push_destination,
set())
set(),
None,
None,
None)
self.dockermock.assert_called_once_with(
base_url='unix://var/run/docker.sock', version='auto')
@ -189,7 +193,10 @@ class TestDockerImageUploader(base.TestCase):
self.uploader.upload_image(image,
None,
push_destination,
set())
set(),
None,
None,
None)
self.dockermock.assert_called_once_with(
base_url='unix://var/run/docker.sock', version='auto')
@ -220,7 +227,10 @@ class TestDockerImageUploader(base.TestCase):
self.uploader.upload_image(image + ':' + tag,
None,
push_destination,
set())
set(),
None,
None,
None)
self.dockermock.assert_called_once_with(
base_url='unix://var/run/docker.sock', version='auto')
@ -230,6 +240,109 @@ class TestDockerImageUploader(base.TestCase):
self.dockermock.return_value.tag.assert_not_called()
self.dockermock.return_value.push.assert_not_called()
@mock.patch('subprocess.Popen')
@mock.patch('tripleo_common.actions.'
'ansible.AnsiblePlaybookAction', autospec=True)
def test_modify_upload_image(self, mock_ansible, mock_popen):
result1 = {
'Digest': 'a'
}
result2 = {
'Digest': 'b'
}
mock_process = mock.Mock()
mock_process.communicate.side_effect = [
(json.dumps(result1), ''),
(json.dumps(result2), ''),
]
mock_process.returncode = 0
mock_popen.return_value = mock_process
image = 'docker.io/tripleomaster/heat-docker-agents-centos'
tag = 'latest'
append_tag = 'modify-123'
push_destination = 'localhost:8787'
push_image = 'localhost:8787/tripleomaster/heat-docker-agents-centos'
playbook = [{
'tasks': [{
'import_role': {
'name': 'add-foo-plugin'
},
'name': 'Import role add-foo-plugin',
'vars': {
'target_image': '%s:%s' % (push_image, tag),
'modified_append_tag': append_tag,
'source_image': '%s:%s' % (image, tag),
'foo_version': '1.0.1'
}
}],
'hosts': 'localhost'
}]
self.uploader.upload_image(image + ':' + tag,
None,
push_destination,
set(),
append_tag,
'add-foo-plugin',
{'foo_version': '1.0.1'})
self.dockermock.assert_called_once_with(
base_url='unix://var/run/docker.sock', version='auto')
self.dockermock.return_value.pull.assert_called_once_with(
image, tag=tag, stream=True)
mock_ansible.assert_called_once_with(
playbook=playbook, work_dir=mock.ANY)
self.dockermock.return_value.tag.assert_not_called()
self.dockermock.return_value.push.assert_called_once_with(
push_image,
tag=tag + append_tag,
stream=True)
@mock.patch('subprocess.Popen')
@mock.patch('tripleo_common.actions.'
'ansible.AnsiblePlaybookAction', autospec=True)
def test_modify_image_failed(self, mock_ansible, mock_popen):
result1 = {
'Digest': 'a'
}
result2 = {
'Digest': 'b'
}
mock_process = mock.Mock()
mock_process.communicate.side_effect = [
(json.dumps(result1), ''),
(json.dumps(result2), ''),
]
mock_process.returncode = 0
mock_popen.return_value = mock_process
image = 'docker.io/tripleomaster/heat-docker-agents-centos'
tag = 'latest'
append_tag = 'modify-123'
push_destination = 'localhost:8787'
error = processutils.ProcessExecutionError(
'', 'ouch', -1, 'ansible-playbook')
mock_ansible.return_value.run.side_effect = error
self.assertRaises(
processutils.ProcessExecutionError,
self.uploader.upload_image,
image + ':' + tag, None, push_destination, set(), append_tag,
'add-foo-plugin', {'foo_version': '1.0.1'}
)
self.dockermock.assert_called_once_with(
base_url='unix://var/run/docker.sock', version='auto')
self.dockermock.return_value.pull.assert_called_once_with(
image, tag=tag, stream=True)
self.dockermock.return_value.tag.assert_not_called()
self.dockermock.return_value.push.assert_not_called()
@mock.patch('requests.get')
def test_is_insecure_registry_known(self, mock_get):
self.assertFalse(

14
tripleo_common/tests/image/test_kolla_builder.py
View File

@ -669,7 +669,9 @@ class TestPrepare(base.TestCase):
'set': mapping_args,
'tag_from_label': 'bar',
'excludes': ['nova', 'neutron'],
'push_destination': '192.0.2.1:8787'
'push_destination': '192.0.2.1:8787',
'modify_role': 'add-foo-plugin',
'modify_vars': {'foo_version': '1.0.1'}
}]
}
}
@ -709,7 +711,10 @@ class TestPrepare(base.TestCase):
pull_source=None,
push_destination=None,
service_filter=None,
tag_from_label='foo'
tag_from_label='foo',
append_tag=None,
modify_role=None,
modify_vars=None
),
mock.call(
excludes=['nova', 'neutron'],
@ -719,7 +724,10 @@ class TestPrepare(base.TestCase):
pull_source=None,
push_destination='192.0.2.1:8787',
service_filter=None,
tag_from_label='bar'
tag_from_label='bar',
append_tag=mock.ANY,
modify_role='add-foo-plugin',
modify_vars={'foo_version': '1.0.1'}
)
])