openstack
/
tripleo-common
Code Issues Proposed changes

Upload docker image files from yaml config file 

Very similar to the change of definining a yaml file to build image
files, this allows one to specify a file to upload images to a registry
specified in the file. Documentation included along with a script to
pass the yaml config file to.

Change-Id: If0cff96574338052cf1073484486d2850c495228
This commit is contained in:
Jeff Peeler 2015-11-19 13:01:12 -05:00
parent d5b5d35efe
commit 86f3df58dc
11 changed files with 331 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
contrib/overcloud_containers.yaml Normal file
View File

@ -0,0 +1,29 @@
uploads:
- imagename: tripleoupstream/heat-docker-agents:liberty
uploader: docker
pull_source: docker.io
push_destination: localhost:8787
- imagename: tripleoupstream/centos-binary-nova-compute:liberty
uploader: docker
pull_source: docker.io
push_destination: localhost:8787
- imagename: tripleoupstream/centos-binary-nova-libvirt:liberty
uploader: docker
pull_source: docker.io
push_destination: localhost:8787
- imagename: tripleoupstream/centos-binary-data:liberty
uploader: docker
pull_source: docker.io
push_destination: localhost:8787
- imagename: tripleoupstream/centos-binary-neutron-openvswitch-agent:liberty
uploader: docker
pull_source: docker.io
push_destination: localhost:8787
- imagename: tripleoupstream/centos-binary-openvswitch-vswitchd:liberty
uploader: docker
pull_source: docker.io
push_destination: localhost:8787
- imagename: tripleoupstream/centos-binary-openvswitch-db-server:liberty
uploader: docker
pull_source: docker.io
push_destination: localhost:8787

19
doc/source/image/upload.rst Normal file
View File

@ -0,0 +1,19 @@
================
Uploading images
================
Call the image upload manager::
manager = ImageUploadManager(['path/to/config.yaml'])
manager.upload()
.. autoclass:: tripleo_common.image.build.ImageUploadManager
:members:
Multiple config files
---------------------
Multiple config files can be passed to the ImageUploadManager.
Attributes are set by the first encountered with the 'imagename' attribute
being the primary key.

20
doc/source/uploads.rst Normal file
View File

@ -0,0 +1,20 @@
=============
Image uploads
=============
.. toctree::
:glob:
:maxdepth: 2
image/upload.rst
YAML file format
----------------
::
uploads:
- imagename: namespace/heat-docker-agents:latest
uploader: docker
pull_source: docker.io
push_destination: localhost:8787

1
requirements.txt
View File

@ -4,6 +4,7 @@
pbr>=1.6 # Apache-2.0
Babel>=1.3 # BSD
docker-py>=1.6.0 # Apache-2.0
python-heatclient>=0.6.0 # Apache-2.0
oslo.config>=3.7.0 # Apache-2.0
oslo.log>=1.14.0 # Apache-2.0

32
tripleo_common/image/base.py
View File

@ -26,6 +26,7 @@ from tripleo_common.image.exception import ImageSpecificationException
class BaseImageManager(object):
logger = log.getLogger(__name__ + '.BaseImageManager')
APPEND_ATTRIBUTES = ['elements', 'options', 'packages']
CONFIG_SECTIONS = ['disk_images', 'uploads']
def __init__(self, config_files):
self.config_files = config_files
@ -38,39 +39,38 @@ class BaseImageManager(object):
except KeyError:
existing_image[attribute_name] = attribute
def load_config_files(self):
disk_images = {}
def load_config_files(self, section):
config_data = {}
for config_file in self.config_files:
if os.path.isfile(config_file):
with open(config_file) as cf:
images = yaml.load(cf.read()).get("disk_images")
self.logger.debug(
'disk_images JSON: %s' % str(disk_images))
for image in images:
image_name = image.get('imagename')
data = yaml.load(cf.read()).get(section)
if not data:
return None
self.logger.debug('%s JSON: %s' % (section, str(data)))
for item in data:
image_name = item.get('imagename')
if image_name is None:
msg = 'imagename is required'
self.logger.error(msg)
raise ImageSpecificationException(msg)
existing_image = disk_images.get(image_name)
existing_image = config_data.get(image_name)
if not existing_image:
disk_images[image_name] = image
config_data[image_name] = item
continue
for attr in self.APPEND_ATTRIBUTES:
self._extend_or_set_attribute(existing_image, image,
self._extend_or_set_attribute(existing_image, item,
attr)
# If a new key is introduced, add it.
for key, value in six.iteritems(image):
for key, value in six.iteritems(item):
if key not in existing_image:
existing_image[key] = image[key]
disk_images[image_name] = existing_image
existing_image[key] = item[key]
config_data[image_name] = existing_image
else:
self.logger.error('No config file exists at: %s' % config_file)
raise IOError('No config file exists at: %s' % config_file)
return [x for x in disk_images.values()]
return [x for x in config_data.values()]

2
tripleo_common/image/build.py
View File

@ -42,7 +42,7 @@ class ImageBuildManager(BaseImageManager):
self.logger.info('Using config files: %s' % self.config_files)
disk_images = self.load_config_files()
disk_images = self.load_config_files(self.CONFIG_SECTIONS[0])
for image in disk_images:
arch = image.get('arch', 'amd64')

4
tripleo_common/image/exception.py
View File

@ -20,3 +20,7 @@ class ImageBuilderException(Exception):
class ImageSpecificationException(Exception):
pass
class ImageUploaderException(Exception):
pass

99
tripleo_common/image/image_uploader.py Normal file
View File

@ -0,0 +1,99 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import abc
import logging
import six
from docker import Client
from tripleo_common.image.base import BaseImageManager
from tripleo_common.image.exception import ImageUploaderException
class ImageUploadManager(BaseImageManager):
"""Manage the uploading of image files
Manage the uploading of images from a config file specified in YAML
syntax. Multiple config files can be specified. They will be merged.
"""
logger = logging.getLogger(__name__ + '.ImageUploadManager')
def __init__(self, config_files, verbose=False, debug=False):
super(ImageUploadManager, self).__init__(config_files)
def upload(self):
"""Start the upload process"""
self.logger.info('Using config files: %s' % self.config_files)
upload_images = self.load_config_files(self.CONFIG_SECTIONS[1])
for item in upload_images:
image_name = item.get('imagename')
uploader = item.get('uploader')
pull_source = item.get('pull_source')
push_destination = item.get('push_destination')
self.logger.info('imagename: %s' % image_name)
uploader = ImageUploader.get_uploader(uploader)
uploader.upload_image(image_name, pull_source, push_destination)
return upload_images # simply to make test validation easier
@six.add_metaclass(abc.ABCMeta)
class ImageUploader(object):
"""Base representation of an image uploading method"""
@staticmethod
def get_uploader(uploader):
if uploader == 'docker':
return DockerImageUploader()
raise ImageUploaderException('Unknown image uploader type')
@abc.abstractmethod
def upload_image(self, image_name, pull_source, push_destination):
"""Upload a disk image"""
pass
class DockerImageUploader(ImageUploader):
"""Upload images using docker push"""
logger = logging.getLogger(__name__ + '.DockerImageUploader')
def upload_image(self, image_name, pull_source, push_destination):
dockerc = Client(base_url='unix://var/run/docker.sock')
image = image_name.rpartition(':')[0]
tag = image_name.rpartition(':')[2]
repo = pull_source + '/' + image
response = [line for line in dockerc.pull(repo,
tag=tag, stream=True, insecure_registry=True)]
self.logger.debug(response)
full_image = repo + ':' + tag
new_repo = push_destination + '/' + image
response = dockerc.tag(image=full_image, repository=new_repo,
tag=tag, force=True)
self.logger.debug(response)
response = [line for line in dockerc.push(new_repo,
tag=tag, stream=True, insecure_registry=True)]
self.logger.debug(response)
self.logger.info('Completed upload for docker image %s' % image_name)

18
tripleo_common/tests/image/fakes.py
View File

@ -25,3 +25,21 @@ def create_disk_images():
}
return disk_images
def create_parsed_upload_images():
uploads = [
{'imagename': 'tripleoupstream/heat-docker-agents-centos:latest',
'push_destination': 'localhost:8787',
'pull_source': 'docker.io',
'uploader': 'docker'},
{'imagename': 'tripleoupstream/centos-binary-nova-compute:liberty',
'push_destination': 'localhost:8787',
'pull_source': 'docker.io',
'uploader': 'docker'},
{'imagename': 'tripleoupstream/centos-binary-nova-libvirt:liberty',
'push_destination': 'localhost:8787',
'pull_source': 'docker.io',
'uploader': 'docker'}
]
return uploads

9
tripleo_common/tests/image/test_base.py
View File

@ -38,7 +38,7 @@ class TestBaseImageManager(testbase.TestCase):
with mock.patch('six.moves.builtins.open', mock_open_context):
base_manager = BaseImageManager(['yamlfile'])
disk_images = base_manager.load_config_files()
disk_images = base_manager.load_config_files('disk_images')
mock_yaml_load.assert_called_once_with("YAML")
self.assertEqual([{
@ -51,7 +51,8 @@ class TestBaseImageManager(testbase.TestCase):
def test_load_config_files_not_found(self):
base_manager = BaseImageManager(['file/does/not/exist'])
self.assertRaises(IOError, base_manager.load_config_files)
self.assertRaises(IOError, base_manager.load_config_files,
'disk_images')
@mock.patch('yaml.load', autospec=True)
@mock.patch('os.path.isfile', autospec=True)
@ -80,7 +81,7 @@ class TestBaseImageManager(testbase.TestCase):
with mock.patch('six.moves.builtins.open', mock_open_context):
base_manager = BaseImageManager(['yamlfile1', 'yamlfile2'])
disk_images = base_manager.load_config_files()
disk_images = base_manager.load_config_files('disk_images')
self.assertEqual(2, mock_yaml_load.call_count)
self.assertEqual([{
@ -117,4 +118,4 @@ class TestBaseImageManager(testbase.TestCase):
with mock.patch('six.moves.builtins.open', mock_open_context):
base_manager = BaseImageManager(['yamlfile'])
self.assertRaises(ImageSpecificationException,
base_manager.load_config_files)
base_manager.load_config_files, 'disk_images')

119
tripleo_common/tests/image/test_image_uploader.py Normal file
View File

@ -0,0 +1,119 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import mock
import operator
import six
from tripleo_common.image.exception import ImageUploaderException
from tripleo_common.image.image_uploader import DockerImageUploader
from tripleo_common.image.image_uploader import ImageUploader
from tripleo_common.image.image_uploader import ImageUploadManager
from tripleo_common.tests import base
from tripleo_common.tests.image import fakes
filedata = six.u(
"""uploads:
- imagename: tripleoupstream/heat-docker-agents-centos:latest
uploader: docker
pull_source: docker.io
push_destination: localhost:8787
- imagename: tripleoupstream/centos-binary-nova-compute:liberty
uploader: docker
pull_source: docker.io
push_destination: localhost:8787
- imagename: tripleoupstream/centos-binary-nova-libvirt:liberty
uploader: docker
pull_source: docker.io
push_destination: localhost:8787
""")
class TestImageUploadManager(base.TestCase):
def setUp(self):
super(TestImageUploadManager, self).setUp()
files = []
files.append('testfile')
self.filelist = files
@mock.patch('tripleo_common.image.base.open',
mock.mock_open(read_data=filedata), create=True)
@mock.patch('os.path.isfile', return_value=True)
@mock.patch('tripleo_common.image.image_uploader.Client')
def test_file_parsing(self, mockpath, mockdocker):
print(filedata)
manager = ImageUploadManager(self.filelist, debug=True)
parsed_data = manager.upload()
mockpath(self.filelist[0])
expected_data = fakes.create_parsed_upload_images()
sorted_expected_data = sorted(expected_data,
key=operator.itemgetter('imagename'))
sorted_parsed_data = sorted(parsed_data,
key=operator.itemgetter('imagename'))
self.assertEqual(sorted_parsed_data, sorted_expected_data)
class TestImageUploader(base.TestCase):
def setUp(self):
super(TestImageUploader, self).setUp()
def test_get_uploader_docker(self):
uploader = ImageUploader.get_uploader('docker')
assert isinstance(uploader, DockerImageUploader)
def test_get_builder_unknown(self):
self.assertRaises(ImageUploaderException, ImageUploader.get_uploader,
'unknown')
class TestDockerImageUploader(base.TestCase):
def setUp(self):
super(TestDockerImageUploader, self).setUp()
self.uploader = DockerImageUploader()
self.patcher = mock.patch('tripleo_common.image.image_uploader.Client')
self.dockermock = self.patcher.start()
def tearDown(self):
super(TestDockerImageUploader, self).tearDown()
self.patcher.stop()
def test_upload_image(self):
image = 'tripleoupstream/heat-docker-agents-centos'
tag = 'latest'
pull_source = 'docker.io'
push_destination = 'localhost:8787'
self.uploader.upload_image(image + ':' + tag,
pull_source,
push_destination)
self.dockermock.assert_called_once_with(
base_url='unix://var/run/docker.sock')
self.dockermock.return_value.pull.assert_called_once_with(
pull_source + '/' + image,
tag=tag, stream=True, insecure_registry=True)
self.dockermock.return_value.tag.assert_called_once_with(
image=pull_source + '/' + image + ':' + tag,
repository=push_destination + '/' + image,
tag=tag, force=True)
self.dockermock.return_value.push(
push_destination + '/' + image,
tag=tag, stream=True, insecure_registry=True)