From 86f3df58dce3839bc5940a41b697afe8c6f18e5f Mon Sep 17 00:00:00 2001 From: Jeff Peeler Date: Thu, 19 Nov 2015 13:01:12 -0500 Subject: [PATCH] Upload docker image files from yaml config file Very similar to the change of definining a yaml file to build image files, this allows one to specify a file to upload images to a registry specified in the file. Documentation included along with a script to pass the yaml config file to. Change-Id: If0cff96574338052cf1073484486d2850c495228 --- contrib/overcloud_containers.yaml | 29 +++++ doc/source/image/upload.rst | 19 +++ doc/source/uploads.rst | 20 +++ requirements.txt | 1 + tripleo_common/image/base.py | 32 ++--- tripleo_common/image/build.py | 2 +- tripleo_common/image/exception.py | 4 + tripleo_common/image/image_uploader.py | 99 +++++++++++++++ tripleo_common/tests/image/fakes.py | 18 +++ tripleo_common/tests/image/test_base.py | 9 +- .../tests/image/test_image_uploader.py | 119 ++++++++++++++++++ 11 files changed, 331 insertions(+), 21 deletions(-) create mode 100644 contrib/overcloud_containers.yaml create mode 100644 doc/source/image/upload.rst create mode 100644 doc/source/uploads.rst create mode 100644 tripleo_common/image/image_uploader.py create mode 100644 tripleo_common/tests/image/test_image_uploader.py diff --git a/contrib/overcloud_containers.yaml b/contrib/overcloud_containers.yaml new file mode 100644 index 000000000..bd43e70ab --- /dev/null +++ b/contrib/overcloud_containers.yaml @@ -0,0 +1,29 @@ +uploads: + - imagename: tripleoupstream/heat-docker-agents:liberty + uploader: docker + pull_source: docker.io + push_destination: localhost:8787 + - imagename: tripleoupstream/centos-binary-nova-compute:liberty + uploader: docker + pull_source: docker.io + push_destination: localhost:8787 + - imagename: tripleoupstream/centos-binary-nova-libvirt:liberty + uploader: docker + pull_source: docker.io + push_destination: localhost:8787 + - imagename: tripleoupstream/centos-binary-data:liberty + uploader: docker + pull_source: docker.io + push_destination: localhost:8787 + - imagename: tripleoupstream/centos-binary-neutron-openvswitch-agent:liberty + uploader: docker + pull_source: docker.io + push_destination: localhost:8787 + - imagename: tripleoupstream/centos-binary-openvswitch-vswitchd:liberty + uploader: docker + pull_source: docker.io + push_destination: localhost:8787 + - imagename: tripleoupstream/centos-binary-openvswitch-db-server:liberty + uploader: docker + pull_source: docker.io + push_destination: localhost:8787 diff --git a/doc/source/image/upload.rst b/doc/source/image/upload.rst new file mode 100644 index 000000000..df6236f10 --- /dev/null +++ b/doc/source/image/upload.rst @@ -0,0 +1,19 @@ +================ +Uploading images +================ + +Call the image upload manager:: + + manager = ImageUploadManager(['path/to/config.yaml']) + manager.upload() + + +.. autoclass:: tripleo_common.image.build.ImageUploadManager + :members: + +Multiple config files +--------------------- + +Multiple config files can be passed to the ImageUploadManager. +Attributes are set by the first encountered with the 'imagename' attribute +being the primary key. diff --git a/doc/source/uploads.rst b/doc/source/uploads.rst new file mode 100644 index 000000000..57e3947e5 --- /dev/null +++ b/doc/source/uploads.rst @@ -0,0 +1,20 @@ +============= +Image uploads +============= + +.. toctree:: + :glob: + :maxdepth: 2 + + image/upload.rst + + +YAML file format +---------------- +:: + + uploads: + - imagename: namespace/heat-docker-agents:latest + uploader: docker + pull_source: docker.io + push_destination: localhost:8787 diff --git a/requirements.txt b/requirements.txt index 4922d057c..6a41b0f11 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ pbr>=1.6 # Apache-2.0 Babel>=1.3 # BSD +docker-py>=1.6.0 # Apache-2.0 python-heatclient>=0.6.0 # Apache-2.0 oslo.config>=3.7.0 # Apache-2.0 oslo.log>=1.14.0 # Apache-2.0 diff --git a/tripleo_common/image/base.py b/tripleo_common/image/base.py index cf290aec6..0f8a9ad3f 100644 --- a/tripleo_common/image/base.py +++ b/tripleo_common/image/base.py @@ -26,6 +26,7 @@ from tripleo_common.image.exception import ImageSpecificationException class BaseImageManager(object): logger = log.getLogger(__name__ + '.BaseImageManager') APPEND_ATTRIBUTES = ['elements', 'options', 'packages'] + CONFIG_SECTIONS = ['disk_images', 'uploads'] def __init__(self, config_files): self.config_files = config_files @@ -38,39 +39,38 @@ class BaseImageManager(object): except KeyError: existing_image[attribute_name] = attribute - def load_config_files(self): - disk_images = {} + def load_config_files(self, section): + config_data = {} for config_file in self.config_files: if os.path.isfile(config_file): with open(config_file) as cf: - images = yaml.load(cf.read()).get("disk_images") - self.logger.debug( - 'disk_images JSON: %s' % str(disk_images)) - for image in images: - image_name = image.get('imagename') + data = yaml.load(cf.read()).get(section) + if not data: + return None + self.logger.debug('%s JSON: %s' % (section, str(data))) + for item in data: + image_name = item.get('imagename') if image_name is None: msg = 'imagename is required' self.logger.error(msg) raise ImageSpecificationException(msg) - existing_image = disk_images.get(image_name) + existing_image = config_data.get(image_name) if not existing_image: - disk_images[image_name] = image + config_data[image_name] = item continue for attr in self.APPEND_ATTRIBUTES: - self._extend_or_set_attribute(existing_image, image, + self._extend_or_set_attribute(existing_image, item, attr) # If a new key is introduced, add it. - for key, value in six.iteritems(image): + for key, value in six.iteritems(item): if key not in existing_image: - existing_image[key] = image[key] - - disk_images[image_name] = existing_image + existing_image[key] = item[key] + config_data[image_name] = existing_image else: self.logger.error('No config file exists at: %s' % config_file) raise IOError('No config file exists at: %s' % config_file) - - return [x for x in disk_images.values()] + return [x for x in config_data.values()] diff --git a/tripleo_common/image/build.py b/tripleo_common/image/build.py index 82978c395..d6de9d609 100644 --- a/tripleo_common/image/build.py +++ b/tripleo_common/image/build.py @@ -42,7 +42,7 @@ class ImageBuildManager(BaseImageManager): self.logger.info('Using config files: %s' % self.config_files) - disk_images = self.load_config_files() + disk_images = self.load_config_files(self.CONFIG_SECTIONS[0]) for image in disk_images: arch = image.get('arch', 'amd64') diff --git a/tripleo_common/image/exception.py b/tripleo_common/image/exception.py index 4ce7e4b68..83b148005 100644 --- a/tripleo_common/image/exception.py +++ b/tripleo_common/image/exception.py @@ -20,3 +20,7 @@ class ImageBuilderException(Exception): class ImageSpecificationException(Exception): pass + + +class ImageUploaderException(Exception): + pass diff --git a/tripleo_common/image/image_uploader.py b/tripleo_common/image/image_uploader.py new file mode 100644 index 000000000..3bfaaff8c --- /dev/null +++ b/tripleo_common/image/image_uploader.py @@ -0,0 +1,99 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + + +import abc +import logging +import six + +from docker import Client +from tripleo_common.image.base import BaseImageManager +from tripleo_common.image.exception import ImageUploaderException + + +class ImageUploadManager(BaseImageManager): + """Manage the uploading of image files + + Manage the uploading of images from a config file specified in YAML + syntax. Multiple config files can be specified. They will be merged. + """ + logger = logging.getLogger(__name__ + '.ImageUploadManager') + + def __init__(self, config_files, verbose=False, debug=False): + super(ImageUploadManager, self).__init__(config_files) + + def upload(self): + """Start the upload process""" + + self.logger.info('Using config files: %s' % self.config_files) + + upload_images = self.load_config_files(self.CONFIG_SECTIONS[1]) + + for item in upload_images: + image_name = item.get('imagename') + uploader = item.get('uploader') + pull_source = item.get('pull_source') + push_destination = item.get('push_destination') + + self.logger.info('imagename: %s' % image_name) + + uploader = ImageUploader.get_uploader(uploader) + uploader.upload_image(image_name, pull_source, push_destination) + + return upload_images # simply to make test validation easier + + +@six.add_metaclass(abc.ABCMeta) +class ImageUploader(object): + """Base representation of an image uploading method""" + + @staticmethod + def get_uploader(uploader): + if uploader == 'docker': + return DockerImageUploader() + raise ImageUploaderException('Unknown image uploader type') + + @abc.abstractmethod + def upload_image(self, image_name, pull_source, push_destination): + """Upload a disk image""" + pass + + +class DockerImageUploader(ImageUploader): + """Upload images using docker push""" + + logger = logging.getLogger(__name__ + '.DockerImageUploader') + + def upload_image(self, image_name, pull_source, push_destination): + dockerc = Client(base_url='unix://var/run/docker.sock') + image = image_name.rpartition(':')[0] + tag = image_name.rpartition(':')[2] + repo = pull_source + '/' + image + + response = [line for line in dockerc.pull(repo, + tag=tag, stream=True, insecure_registry=True)] + self.logger.debug(response) + + full_image = repo + ':' + tag + new_repo = push_destination + '/' + image + response = dockerc.tag(image=full_image, repository=new_repo, + tag=tag, force=True) + self.logger.debug(response) + + response = [line for line in dockerc.push(new_repo, + tag=tag, stream=True, insecure_registry=True)] + self.logger.debug(response) + + self.logger.info('Completed upload for docker image %s' % image_name) diff --git a/tripleo_common/tests/image/fakes.py b/tripleo_common/tests/image/fakes.py index 5375ec8c5..9a97787ca 100644 --- a/tripleo_common/tests/image/fakes.py +++ b/tripleo_common/tests/image/fakes.py @@ -25,3 +25,21 @@ def create_disk_images(): } return disk_images + + +def create_parsed_upload_images(): + uploads = [ + {'imagename': 'tripleoupstream/heat-docker-agents-centos:latest', + 'push_destination': 'localhost:8787', + 'pull_source': 'docker.io', + 'uploader': 'docker'}, + {'imagename': 'tripleoupstream/centos-binary-nova-compute:liberty', + 'push_destination': 'localhost:8787', + 'pull_source': 'docker.io', + 'uploader': 'docker'}, + {'imagename': 'tripleoupstream/centos-binary-nova-libvirt:liberty', + 'push_destination': 'localhost:8787', + 'pull_source': 'docker.io', + 'uploader': 'docker'} + ] + return uploads diff --git a/tripleo_common/tests/image/test_base.py b/tripleo_common/tests/image/test_base.py index c5311665f..2c4daca2e 100644 --- a/tripleo_common/tests/image/test_base.py +++ b/tripleo_common/tests/image/test_base.py @@ -38,7 +38,7 @@ class TestBaseImageManager(testbase.TestCase): with mock.patch('six.moves.builtins.open', mock_open_context): base_manager = BaseImageManager(['yamlfile']) - disk_images = base_manager.load_config_files() + disk_images = base_manager.load_config_files('disk_images') mock_yaml_load.assert_called_once_with("YAML") self.assertEqual([{ @@ -51,7 +51,8 @@ class TestBaseImageManager(testbase.TestCase): def test_load_config_files_not_found(self): base_manager = BaseImageManager(['file/does/not/exist']) - self.assertRaises(IOError, base_manager.load_config_files) + self.assertRaises(IOError, base_manager.load_config_files, + 'disk_images') @mock.patch('yaml.load', autospec=True) @mock.patch('os.path.isfile', autospec=True) @@ -80,7 +81,7 @@ class TestBaseImageManager(testbase.TestCase): with mock.patch('six.moves.builtins.open', mock_open_context): base_manager = BaseImageManager(['yamlfile1', 'yamlfile2']) - disk_images = base_manager.load_config_files() + disk_images = base_manager.load_config_files('disk_images') self.assertEqual(2, mock_yaml_load.call_count) self.assertEqual([{ @@ -117,4 +118,4 @@ class TestBaseImageManager(testbase.TestCase): with mock.patch('six.moves.builtins.open', mock_open_context): base_manager = BaseImageManager(['yamlfile']) self.assertRaises(ImageSpecificationException, - base_manager.load_config_files) + base_manager.load_config_files, 'disk_images') diff --git a/tripleo_common/tests/image/test_image_uploader.py b/tripleo_common/tests/image/test_image_uploader.py new file mode 100644 index 000000000..8f99ee3ef --- /dev/null +++ b/tripleo_common/tests/image/test_image_uploader.py @@ -0,0 +1,119 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + + +import mock +import operator +import six + +from tripleo_common.image.exception import ImageUploaderException +from tripleo_common.image.image_uploader import DockerImageUploader +from tripleo_common.image.image_uploader import ImageUploader +from tripleo_common.image.image_uploader import ImageUploadManager +from tripleo_common.tests import base +from tripleo_common.tests.image import fakes + + +filedata = six.u( + """uploads: + - imagename: tripleoupstream/heat-docker-agents-centos:latest + uploader: docker + pull_source: docker.io + push_destination: localhost:8787 + - imagename: tripleoupstream/centos-binary-nova-compute:liberty + uploader: docker + pull_source: docker.io + push_destination: localhost:8787 + - imagename: tripleoupstream/centos-binary-nova-libvirt:liberty + uploader: docker + pull_source: docker.io + push_destination: localhost:8787 +""") + + +class TestImageUploadManager(base.TestCase): + def setUp(self): + super(TestImageUploadManager, self).setUp() + files = [] + files.append('testfile') + self.filelist = files + + @mock.patch('tripleo_common.image.base.open', + mock.mock_open(read_data=filedata), create=True) + @mock.patch('os.path.isfile', return_value=True) + @mock.patch('tripleo_common.image.image_uploader.Client') + def test_file_parsing(self, mockpath, mockdocker): + print(filedata) + manager = ImageUploadManager(self.filelist, debug=True) + parsed_data = manager.upload() + mockpath(self.filelist[0]) + + expected_data = fakes.create_parsed_upload_images() + sorted_expected_data = sorted(expected_data, + key=operator.itemgetter('imagename')) + sorted_parsed_data = sorted(parsed_data, + key=operator.itemgetter('imagename')) + self.assertEqual(sorted_parsed_data, sorted_expected_data) + + +class TestImageUploader(base.TestCase): + + def setUp(self): + super(TestImageUploader, self).setUp() + + def test_get_uploader_docker(self): + uploader = ImageUploader.get_uploader('docker') + assert isinstance(uploader, DockerImageUploader) + + def test_get_builder_unknown(self): + self.assertRaises(ImageUploaderException, ImageUploader.get_uploader, + 'unknown') + + +class TestDockerImageUploader(base.TestCase): + + def setUp(self): + super(TestDockerImageUploader, self).setUp() + self.uploader = DockerImageUploader() + self.patcher = mock.patch('tripleo_common.image.image_uploader.Client') + self.dockermock = self.patcher.start() + + def tearDown(self): + super(TestDockerImageUploader, self).tearDown() + self.patcher.stop() + + def test_upload_image(self): + image = 'tripleoupstream/heat-docker-agents-centos' + tag = 'latest' + pull_source = 'docker.io' + push_destination = 'localhost:8787' + + self.uploader.upload_image(image + ':' + tag, + pull_source, + push_destination) + + self.dockermock.assert_called_once_with( + base_url='unix://var/run/docker.sock') + + self.dockermock.return_value.pull.assert_called_once_with( + pull_source + '/' + image, + tag=tag, stream=True, insecure_registry=True) + self.dockermock.return_value.tag.assert_called_once_with( + image=pull_source + '/' + image + ':' + tag, + repository=push_destination + '/' + image, + tag=tag, force=True) + self.dockermock.return_value.push( + push_destination + '/' + image, + tag=tag, stream=True, insecure_registry=True)