diff --git a/doc/source/image/build.rst b/doc/source/image/build.rst new file mode 100644 index 000000000..f921fbe64 --- /dev/null +++ b/doc/source/image/build.rst @@ -0,0 +1,20 @@ +=============== +Building images +=============== + +Call the image build manager:: + + manager = ImageBuildManager(['path/to/config.yaml']) + manager.build() + + +.. autoclass:: tripleo_common.image.build.ImageBuildManager + :members: + +Multiple config files +--------------------- + +Multiple config files can be passed to the ImageBuildManager. Certain attributes +will be merged (currently, 'elements', 'options', and 'packages'), while other +attributes will only be set by the first encountered. The 'imagename' attribute +will be the primary key. \ No newline at end of file diff --git a/doc/source/images.rst b/doc/source/images.rst new file mode 100644 index 000000000..43c3f10bb --- /dev/null +++ b/doc/source/images.rst @@ -0,0 +1,29 @@ +=========== +Disk images +=========== + +.. toctree:: + :glob: + :maxdepth: 2 + + image/* + + +YAML file format +---------------- +:: + + disk_images: + - + imagename: overcloud-compute + builder: dib + arch: amd64 + type: qcow2 + distro: centos7 + elements: + - overcloud-compute + - other-element + packages: + - vim + options: + diff --git a/doc/source/index.rst b/doc/source/index.rst index 55eb7634d..b898c5aa0 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -14,6 +14,7 @@ Contents: readme installation usage + images contributing Indices and tables diff --git a/requirements.txt b/requirements.txt index 693b11179..4922d057c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,6 @@ pbr>=1.6 # Apache-2.0 Babel>=1.3 # BSD python-heatclient>=0.6.0 # Apache-2.0 +oslo.config>=3.7.0 # Apache-2.0 +oslo.log>=1.14.0 # Apache-2.0 +oslo.utils>=3.5.0 # Apache-2.0 diff --git a/scripts/tripleo-build-images b/scripts/tripleo-build-images new file mode 100755 index 000000000..306c4c505 --- /dev/null +++ b/scripts/tripleo-build-images @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import argparse +import os +import subprocess +import sys +import yaml + +from oslo_config import cfg +from oslo_config import types +from oslo_log import log + + +from tripleo_common.image.build import ImageBuildManager + +LOG = log.getLogger(__name__) +env = os.environ.copy() + +CONF = cfg.CONF +log.register_options(CONF) +image_opt_group = cfg.OptGroup(name='image', + title='Image build options') +_opts = [ + cfg.MultiOpt('config-file', + item_type=types.String(), + default=['disk_images.yaml'], + help=("""Path to configuration file. Can be specified """ + """multiple times"""), + ), + cfg.StrOpt('output-directory', + default=env.get('TRIPLEO_ROOT', '.'), + help=("""output directory for images. """ + """Defaults to $TRIPLEO_ROOT, or current directory""") + ), + cfg.BoolOpt('skip', + default=False, + help="""Skip build if cached image exists.""" + ), +] +CONF.register_group(image_opt_group) +CONF.register_cli_opts(_opts, group=image_opt_group) +log.setup(CONF, 'build-overcloud-images') + +def main(argv=sys.argv): + CONF(argv[1:]) + LOG.info('Using config files at: %s' % CONF.image.config_file) + + manager = ImageBuildManager(CONF.image.config_file, + output_directory=CONF.image.output_directory, + skip=CONF.image.skip) + manager.build() + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/setup.cfg b/setup.cfg index 666a4b3c1..852850ada 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,7 @@ packages = scripts = scripts/upgrade-non-controller.sh + scripts/tripleo-build-images data_files = share/tripleo-common/templates = templates/* diff --git a/tripleo_common/image/__init__.py b/tripleo_common/image/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tripleo_common/image/base.py b/tripleo_common/image/base.py new file mode 100644 index 000000000..cf290aec6 --- /dev/null +++ b/tripleo_common/image/base.py @@ -0,0 +1,76 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import six + +import os +import yaml + +from oslo_log import log + +from tripleo_common.image.exception import ImageSpecificationException + + +class BaseImageManager(object): + logger = log.getLogger(__name__ + '.BaseImageManager') + APPEND_ATTRIBUTES = ['elements', 'options', 'packages'] + + def __init__(self, config_files): + self.config_files = config_files + + def _extend_or_set_attribute(self, existing_image, image, attribute_name): + attribute = image.get(attribute_name, []) + if attribute: + try: + existing_image[attribute_name].extend(attribute) + except KeyError: + existing_image[attribute_name] = attribute + + def load_config_files(self): + disk_images = {} + for config_file in self.config_files: + if os.path.isfile(config_file): + with open(config_file) as cf: + images = yaml.load(cf.read()).get("disk_images") + self.logger.debug( + 'disk_images JSON: %s' % str(disk_images)) + for image in images: + image_name = image.get('imagename') + if image_name is None: + msg = 'imagename is required' + self.logger.error(msg) + raise ImageSpecificationException(msg) + + existing_image = disk_images.get(image_name) + if not existing_image: + disk_images[image_name] = image + continue + + for attr in self.APPEND_ATTRIBUTES: + self._extend_or_set_attribute(existing_image, image, + attr) + + # If a new key is introduced, add it. + for key, value in six.iteritems(image): + if key not in existing_image: + existing_image[key] = image[key] + + disk_images[image_name] = existing_image + + else: + self.logger.error('No config file exists at: %s' % config_file) + raise IOError('No config file exists at: %s' % config_file) + + return [x for x in disk_images.values()] diff --git a/tripleo_common/image/build.py b/tripleo_common/image/build.py new file mode 100644 index 000000000..82978c395 --- /dev/null +++ b/tripleo_common/image/build.py @@ -0,0 +1,80 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import os +import re + +from oslo_log import log +from oslo_utils import strutils + +from tripleo_common.image.base import BaseImageManager +from tripleo_common.image.exception import ImageSpecificationException +from tripleo_common.image.image_builder import ImageBuilder + + +class ImageBuildManager(BaseImageManager): + """Manage the building of image files + + Manage the building of images from a config file specified in YAML + syntax. Multiple config files can be specified. They will be merged + """ + logger = log.getLogger(__name__ + '.ImageBuildManager') + + def __init__(self, config_files, output_directory='.', skip=False): + super(ImageBuildManager, self).__init__(config_files) + self.output_directory = re.sub('[/]$', '', output_directory) + self.skip = skip + + def build(self): + """Start the build process""" + + self.logger.info('Using config files: %s' % self.config_files) + + disk_images = self.load_config_files() + + for image in disk_images: + arch = image.get('arch', 'amd64') + image_type = image.get('type', 'qcow2') + image_name = image.get('imagename') + builder = image.get('builder', 'dib') + skip_base = strutils.bool_from_string( + image.get('skip_base', False)) + docker_target = image.get('docker_target') + node_dist = image.get('distro') + if node_dist is None: + raise ImageSpecificationException('distro is required') + self.logger.info('imagename: %s' % image_name) + image_extension = image.get('imageext', image_type) + image_path = '%s/%s.%s' % ( + self.output_directory, image_name, image_extension) + if self.skip: + self.logger.info('looking for image at path: %s' % image_path) + if os.path.exists(image_path): + self.logger.info('Image file exists for image name: %s' % + image_name) + self.logger.info('Skipping image build') + continue + elements = image.get('elements', []) + options = image.get('options', []) + packages = image.get('packages', []) + + extra_options = { + 'skip_base': skip_base, + 'docker_target': docker_target, + } + + builder = ImageBuilder.get_builder(builder) + builder.build_image(image_path, image_type, node_dist, arch, + elements, options, packages, extra_options) diff --git a/tripleo_common/image/exception.py b/tripleo_common/image/exception.py new file mode 100644 index 000000000..4ce7e4b68 --- /dev/null +++ b/tripleo_common/image/exception.py @@ -0,0 +1,22 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + + +class ImageBuilderException(Exception): + pass + + +class ImageSpecificationException(Exception): + pass diff --git a/tripleo_common/image/image_builder.py b/tripleo_common/image/image_builder.py new file mode 100644 index 000000000..4a910d2c1 --- /dev/null +++ b/tripleo_common/image/image_builder.py @@ -0,0 +1,89 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + + +import abc +import logging +import os +import six +import subprocess + +from tripleo_common.image.exception import ImageBuilderException + + +@six.add_metaclass(abc.ABCMeta) +class ImageBuilder(object): + """Base representation of an image building method""" + + @staticmethod + def get_builder(builder): + if builder == 'dib': + return DibImageBuilder() + raise ImageBuilderException('Unknown image builder type') + + @abc.abstractmethod + def build_image(self, image_path, image_type, node_dist, arch, elements, + options, packages, extra_options={}): + """Build a disk image""" + pass + + +class DibImageBuilder(ImageBuilder): + """Build images using diskimage-builder""" + + logger = logging.getLogger(__name__ + '.DibImageBuilder') + + def build_image(self, image_path, image_type, node_dist, arch, elements, + options, packages, extra_options={}): + env = os.environ.copy() + + elements_path = env.get('ELEMENTS_PATH') + if elements_path is None: + env['ELEMENTS_PATH'] = os.pathsep.join([ + "/usr/share/tripleo-puppet-elements", + "/usr/share/instack-undercloud", + '/usr/share/tripleo-image-elements', + '/usr/share/openstack-heat-templates/' + 'software-config/elements', + ]) + os.environ.update(env) + + cmd = ['disk-image-create', '-a', arch, '-o', image_path, + '-t', image_type] + + if packages: + cmd.append('-p') + cmd.append(','.join(packages)) + + if options: + for option in options: + cmd.extend(option.split(' ')) + + skip_base = extra_options.get('skip_base', False) + if skip_base: + cmd.append('-n') + + docker_target = extra_options.get('docker_target') + if docker_target: + cmd.append('--docker-target') + cmd.append(docker_target) + + if node_dist: + cmd.append(node_dist) + + cmd.extend(elements) + + self.logger.info('Running %s' % cmd) + subprocess.check_call(cmd) diff --git a/tripleo_common/tests/image/__init__.py b/tripleo_common/tests/image/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tripleo_common/tests/image/fakes.py b/tripleo_common/tests/image/fakes.py new file mode 100644 index 000000000..5375ec8c5 --- /dev/null +++ b/tripleo_common/tests/image/fakes.py @@ -0,0 +1,27 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def create_disk_images(): + disk_images = { + 'disk_images': [{ + 'arch': 'amd64', + 'distro': 'some_awesome_os', + 'imagename': 'overcloud', + 'type': 'qcow2', + 'elements': ['image_element'] + }] + } + + return disk_images diff --git a/tripleo_common/tests/image/test_base.py b/tripleo_common/tests/image/test_base.py new file mode 100644 index 000000000..c5311665f --- /dev/null +++ b/tripleo_common/tests/image/test_base.py @@ -0,0 +1,120 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + + +import mock + +from tripleo_common.image.base import BaseImageManager +from tripleo_common.image.exception import ImageSpecificationException +from tripleo_common.tests import base as testbase +from tripleo_common.tests.image import fakes + + +class TestBaseImageManager(testbase.TestCase): + def setUp(self): + super(TestBaseImageManager, self).setUp() + + @mock.patch('yaml.load', autospec=True) + @mock.patch('os.path.isfile', autospec=True) + def test_load_config_files(self, mock_os_path_isfile, mock_yaml_load): + mock_yaml_load.return_value = fakes.create_disk_images() + + mock_os_path_isfile.return_value = True + + mock_open_context = mock.mock_open() + mock_open_context().read.return_value = "YAML" + + with mock.patch('six.moves.builtins.open', mock_open_context): + base_manager = BaseImageManager(['yamlfile']) + disk_images = base_manager.load_config_files() + + mock_yaml_load.assert_called_once_with("YAML") + self.assertEqual([{ + 'arch': 'amd64', + 'distro': 'some_awesome_os', + 'imagename': 'overcloud', + 'type': 'qcow2', + 'elements': ['image_element'] + }], disk_images) + + def test_load_config_files_not_found(self): + base_manager = BaseImageManager(['file/does/not/exist']) + self.assertRaises(IOError, base_manager.load_config_files) + + @mock.patch('yaml.load', autospec=True) + @mock.patch('os.path.isfile', autospec=True) + def test_load_config_files_multiple_files(self, mock_os_path_isfile, + mock_yaml_load): + mock_yaml_load.side_effect = [{ + 'disk_images': [{ + 'arch': 'amd64', + 'imagename': 'overcloud', + 'distro': 'some_awesome_distro', + 'type': 'qcow2', + 'elements': ['image_element'] + }]}, + { + 'disk_images': [{ + 'imagename': 'overcloud', + 'elements': ['another_image_element'], + 'packages': ['a_package'], + 'otherkey': 'some_other_key', + }]}] + + mock_os_path_isfile.return_value = True + + mock_open_context = mock.mock_open() + mock_open_context().read.return_value = "YAML" + + with mock.patch('six.moves.builtins.open', mock_open_context): + base_manager = BaseImageManager(['yamlfile1', 'yamlfile2']) + disk_images = base_manager.load_config_files() + + self.assertEqual(2, mock_yaml_load.call_count) + self.assertEqual([{ + 'arch': 'amd64', + 'distro': 'some_awesome_distro', + 'imagename': 'overcloud', + 'type': 'qcow2', + 'elements': ['image_element', 'another_image_element'], + 'packages': ['a_package'], + 'otherkey': 'some_other_key', + }], disk_images) + + @mock.patch('yaml.load', autospec=True) + @mock.patch('os.path.isfile', autospec=True) + def test_load_config_files_missing_image_name(self, mock_os_path_isfile, + mock_yaml_load): + mock_yaml_load.return_value = { + 'disk_images': [{ + 'arch': 'amd64', + 'imagename': 'overcloud', + 'type': 'qcow2', + 'elements': ['image_element'] + }, { + 'arch': 'amd64', + 'type': 'qcow2', + }] + } + + mock_os_path_isfile.return_value = True + + mock_open_context = mock.mock_open() + mock_open_context().read.return_value = "YAML" + + with mock.patch('six.moves.builtins.open', mock_open_context): + base_manager = BaseImageManager(['yamlfile']) + self.assertRaises(ImageSpecificationException, + base_manager.load_config_files) diff --git a/tripleo_common/tests/image/test_build.py b/tripleo_common/tests/image/test_build.py new file mode 100644 index 000000000..feb7f06e9 --- /dev/null +++ b/tripleo_common/tests/image/test_build.py @@ -0,0 +1,80 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +import mock + +from tripleo_common.image.build import ImageBuildManager +from tripleo_common.image.exception import ImageSpecificationException +from tripleo_common.image.image_builder import ImageBuilder +from tripleo_common.tests import base +from tripleo_common.tests.image import fakes + + +class TestImageBuildManager(base.TestCase): + def setUp(self): + super(TestImageBuildManager, self).setUp() + + @mock.patch.object(ImageBuilder, 'get_builder') + @mock.patch('tripleo_common.image.base.BaseImageManager.load_config_files', + autospec=True) + def test_build(self, mock_load_config_files, mock_get_builder): + mock_load_config_files.return_value = fakes.create_disk_images().get( + 'disk_images') + + mock_builder = mock.Mock() + mock_get_builder.return_value = mock_builder + + build_manager = ImageBuildManager(['config/file']) + build_manager.build() + + self.assertEqual(1, mock_load_config_files.call_count) + + mock_builder.build_image.assert_called_with( + './overcloud.qcow2', 'qcow2', 'some_awesome_os', 'amd64', + ['image_element'], [], [], + {'skip_base': False, 'docker_target': None}) + + @mock.patch.object(ImageBuilder, 'get_builder') + @mock.patch('tripleo_common.image.base.BaseImageManager.load_config_files', + autospec=True) + def test_build_no_distro(self, mock_load_config_files, mock_get_builder): + mock_load_config_files.return_value = [{ + 'imagename': 'overcloud', + }] + + mock_builder = mock.Mock() + mock_get_builder.return_value = mock_builder + + build_manager = ImageBuildManager(['config/file']) + self.assertRaises(ImageSpecificationException, build_manager.build) + + @mock.patch('os.path.exists', autospec=True) + @mock.patch.object(ImageBuilder, 'get_builder') + @mock.patch('tripleo_common.image.base.BaseImageManager.load_config_files', + autospec=True) + def test_build_with_skip(self, mock_load_config_files, mock_get_builder, + mock_os_path_exists): + mock_load_config_files.return_value = fakes.create_disk_images().get( + 'disk_images') + + mock_builder = mock.Mock() + mock_get_builder.return_value = mock_builder + + mock_os_path_exists.return_value = True + + build_manager = ImageBuildManager(['config/file'], skip=True) + build_manager.build() + + self.assertEqual(1, mock_os_path_exists.call_count) diff --git a/tripleo_common/tests/image/test_image_builder.py b/tripleo_common/tests/image/test_image_builder.py new file mode 100644 index 000000000..725ddfe4e --- /dev/null +++ b/tripleo_common/tests/image/test_image_builder.py @@ -0,0 +1,54 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + + +import mock + +from tripleo_common.image.exception import ImageBuilderException +from tripleo_common.image.image_builder import DibImageBuilder +from tripleo_common.image.image_builder import ImageBuilder +from tripleo_common.tests import base + + +class TestImageBuilder(base.TestCase): + + def test_get_builder_dib(self): + builder = ImageBuilder.get_builder('dib') + assert isinstance(builder, DibImageBuilder) + + def test_get_builder_unknown(self): + self.assertRaises(ImageBuilderException, ImageBuilder.get_builder, + 'unknown') + + +class TestDibImageBuilder(base.TestCase): + + def setUp(self): + super(TestDibImageBuilder, self).setUp() + self.builder = DibImageBuilder() + + @mock.patch('subprocess.check_call') + def test_build_image(self, mock_check_call): + self.builder.build_image('image/path', 'imgtype', 'node_dist', 'arch', + ['element1', 'element2'], ['options'], + ['package1', 'package2'], + {'skip_base': True, + 'docker_target': 'docker-target'}) + mock_check_call.assert_called_once_with( + ['disk-image-create', '-a', 'arch', '-o', 'image/path', + '-t', 'imgtype', + '-p', 'package1,package2', 'options', '-n', + '--docker-target', 'docker-target', 'node_dist', + 'element1', 'element2'])