openstack/tripleo-common
Code Issues Proposed changes

Implement image customization during upload

Browse Source 
This allows an ansible role to be invoked after the image pull and
before the image push during an upload operation.

This mechanism can be used to modify images for tasks such:
- updating to latest packages during a CI run
- modifying an image to add a vendor plugin
- adding local developer code to an image to test deployment

Blueprint: container-prepare-workflow

Change-Id: I2c877a96264b351b4fc8527a3e40b87ddcb4f9a5
This commit is contained in:
Steve Baker
2018-05-03 13:12:51 +12:00
parent 63ab54411e
commit f3f8993681
4 changed files with 235 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
tripleo_common/image/image_uploader.py
View File

@@ -21,10 +21,13 @@ import logging
import netifaces
import os
import requests
import shutil
import six
from six.moves import urllib
import subprocess
import tempfile
import tenacity
import yaml
import docker
try:
@@ -32,6 +35,7 @@ try:
except ImportError:
from docker import Client
from oslo_concurrency import processutils
from tripleo_common.actions import ansible
from tripleo_common.image.base import BaseImageManager
from tripleo_common.image.exception import ImageUploaderException
@@ -96,9 +100,13 @@ class ImageUploadManager(BaseImageManager):
# This updates the parsed upload_images dict with real values
item['push_destination'] = push_destination
append_tag = item.get('append_tag')
modify_role = item.get('modify_role')
modify_vars = item.get('modify_vars')
self.uploader(uploader).add_upload_task(
image_name, pull_source, push_destination)
image_name, pull_source, push_destination,
append_tag, modify_role, modify_vars)
for uploader in self.uploaders.values():
uploader.run_tasks()
@@ -122,7 +130,8 @@ class ImageUploader(object):
pass
@abc.abstractmethod
def add_upload_task(self, image_name, pull_source, push_destination):
def add_upload_task(self, image_name, pull_source, push_destination,
append_tag, modify_role, modify_vars):
"""Add an upload task to be executed later"""
pass
@@ -150,41 +159,90 @@ class DockerImageUploader(ImageUploader):
self.secure_registries = set(SECURE_REGISTRIES)
self.insecure_registries = set()
@staticmethod
def run_modify_playbook(modify_role, modify_vars,
source_image, target_image, append_tag):
vars = {}
if modify_vars:
vars.update(modify_vars)
vars['source_image'] = source_image
vars['target_image'] = target_image
vars['modified_append_tag'] = append_tag
LOG.debug('Playbook variables: \n%s' % yaml.safe_dump(
vars, default_flow_style=False))
playbook = [{
'hosts': 'localhost',
'tasks': [{
'name': 'Import role %s' % modify_role,
'import_role': {
'name': modify_role
},
'vars': vars
}]
}]
LOG.debug('Playbook: \n%s' % yaml.safe_dump(
playbook, default_flow_style=False))
work_dir = tempfile.mkdtemp(prefix='tripleo-modify-image-playbook-')
try:
action = ansible.AnsiblePlaybookAction(
playbook=playbook,
work_dir=work_dir
)
result = action.run(None)
LOG.debug(result.get('stdout', ''))
finally:
shutil.rmtree(work_dir)
@staticmethod
def upload_image(image_name, pull_source, push_destination,
insecure_registries):
insecure_registries, append_tag, modify_role,
modify_vars):
LOG.info('imagename: %s' % image_name)
dockerc = Client(base_url='unix://var/run/docker.sock', version='auto')
if ':' in image_name:
image = image_name.rpartition(':')[0]
tag = image_name.rpartition(':')[2]
source_tag = image_name.rpartition(':')[2]
else:
image = image_name
tag = 'latest'
source_tag = 'latest'
if pull_source:
repo = pull_source + '/' + image
else:
repo = image
full_image = repo + ':' + tag
new_repo = push_destination + '/' + repo.partition('/')[2]
full_new_repo = new_repo + ':' + tag
source_image = repo + ':' + source_tag
target_image_no_tag = push_destination + '/' + repo.partition('/')[2]
append_tag = append_tag or ''
target_tag = source_tag + append_tag
target_image_source_tag = target_image_no_tag + ':' + source_tag
target_image = target_image_no_tag + ':' + target_tag
if DockerImageUploader._images_match(full_image, full_new_repo,
if DockerImageUploader._images_match(source_image, target_image,
insecure_registries):
LOG.info('Skipping upload for image %s' % image_name)
return []
DockerImageUploader._pull(dockerc, repo, tag=tag)
DockerImageUploader._pull(dockerc, repo, tag=source_tag)
response = dockerc.tag(image=full_image, repository=new_repo,
tag=tag, force=True)
LOG.debug(response)
if modify_role:
DockerImageUploader.run_modify_playbook(
modify_role, modify_vars, source_image,
target_image_source_tag, append_tag)
# raise an exception if the playbook didn't tag
# the expected target image
dockerc.inspect_image(target_image)
else:
response = dockerc.tag(
image=source_image, repository=target_image_no_tag,
tag=target_tag, force=True
)
LOG.debug(response)
DockerImageUploader._push(dockerc, new_repo, tag=tag)
DockerImageUploader._push(dockerc, target_image_no_tag, tag=target_tag)
LOG.info('Completed upload for image %s' % image_name)
return full_image, full_new_repo
return source_image, target_image
@staticmethod
@tenacity.retry( # Retry up to 5 times with jittered exponential backoff
@@ -353,7 +411,8 @@ class DockerImageUploader(ImageUploader):
else:
LOG.warning(e)
def add_upload_task(self, image_name, pull_source, push_destination):
def add_upload_task(self, image_name, pull_source, push_destination,
append_tag, modify_role, modify_vars):
# prime self.insecure_registries
if pull_source:
self.is_insecure_registry(self._image_to_url(pull_source).netloc)
@@ -361,7 +420,8 @@ class DockerImageUploader(ImageUploader):
self.is_insecure_registry(self._image_to_url(image_name).netloc)
self.is_insecure_registry(self._image_to_url(push_destination).netloc)
self.upload_tasks.append((image_name, pull_source, push_destination,
self.insecure_registries))
self.insecure_registries, append_tag,
modify_role, modify_vars))
def run_tasks(self):
if not self.upload_tasks:

35
tripleo_common/image/kolla_builder.py
View File

@@ -21,6 +21,7 @@ import re
import subprocess
import sys
import tempfile
import time
import yaml
from tripleo_common.image import base
@@ -114,11 +115,18 @@ def container_images_prepare_multi(environment, roles_data):
env_params = {}
service_filter = build_service_filter(environment, roles_data)
modified_timestamp = time.strftime('-modified-%Y%m%d%H%M%S')
for cip_entry in cip:
mapping_args = cip_entry.get('set')
push_destination = cip_entry.get('push_destination')
pull_source = cip_entry.get('pull_source')
modify_role = cip_entry.get('modify_role')
modify_vars = cip_entry.get('modify_vars')
if modify_role:
append_tag = modified_timestamp
else:
append_tag = None
prepare_data = container_images_prepare(
excludes=cip_entry.get('excludes'),
@@ -129,16 +137,21 @@ def container_images_prepare_multi(environment, roles_data):
output_env_file='image_params',
output_images_file='upload_data',
tag_from_label=cip_entry.get('tag_from_label'),
append_tag=append_tag,
modify_role=modify_role,
modify_vars=modify_vars
)
env_params.update(prepare_data['image_params'])
if push_destination or pull_source:
if push_destination or pull_source or modify_role:
with tempfile.NamedTemporaryFile(mode='w') as f:
yaml.safe_dump({
'container_images': prepare_data['upload_data']
}, f)
uploader = image_uploader.ImageUploadManager(
[f.name], verbose=True)
[f.name],
verbose=True,
)
uploader.upload()
return env_params
@@ -157,7 +170,9 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
excludes=None, service_filter=None,
pull_source=None, push_destination=None,
mapping_args=None, output_env_file=None,
output_images_file=None, tag_from_label=None):
output_images_file=None, tag_from_label=None,
append_tag=None, modify_role=None,
modify_vars=None):
"""Perform container image preparation
:param template_file: path to Jinja2 file containing all image entries
@@ -174,12 +189,18 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
:param output_images_file: key to use for image upload data
:param tag_from_label: string when set will trigger tag discovery on every
image
:param append_tag: string to append to the tag for the destination image
:param modify_role: string of ansible role name to run during upload before
the push to destination
:param modify_vars: dict of variables to pass to modify_role
:returns: dict with entries for the supplied output_env_file or
output_images_file
"""
if mapping_args is None:
mapping_args = {}
if not append_tag:
append_tag = ''
if service_filter:
if 'OS::TripleO::Services::OpenDaylightApi' in service_filter:
@@ -227,9 +248,15 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
# push_destination, since that is where they will be uploaded to
image = imagename.partition('/')[2]
imagename = '/'.join((push_destination, image))
if append_tag:
entry['append_tag'] = append_tag
if modify_role:
entry['modify_role'] = modify_role
if modify_vars:
entry['modify_vars'] = modify_vars
if 'params' in entry:
for p in entry.pop('params'):
params[p] = imagename
params[p] = imagename + append_tag
if 'services' in entry:
del(entry['services'])