Build image files from definitions in yaml

This change introduces a new way of defining image files. Using yaml,
the image file definitions can now be externalized, making updates to
the images being built easier. This will also allow for easier
customization of the images without having to change the actual code.

Change-Id: I321fa12f6a85c6a71cbd6e4e5ed9db3bfa4cb1c7

doc/source/image/build.rst

@@ -0,0 +1,20 @@
+===============
+Building images
+===============
+
+Call the image build manager::
+
+    manager = ImageBuildManager(['path/to/config.yaml'])
+    manager.build()
+
+
+.. autoclass:: tripleo_common.image.build.ImageBuildManager
+   :members:
+
+Multiple config files
+---------------------
+
+Multiple config files can be passed to the ImageBuildManager. Certain attributes
+will be merged (currently, 'elements', 'options', and 'packages'), while other
+attributes will only be set by the first encountered. The 'imagename' attribute
+will be the primary key.

doc/source/images.rst

@@ -0,0 +1,29 @@
+===========
+Disk images
+===========
+
+.. toctree::
+   :glob:
+   :maxdepth: 2
+
+   image/*
+
+
+YAML file format
+----------------
+::
+
+    disk_images:
+      -
+         imagename: overcloud-compute
+         builder: dib
+         arch: amd64
+         type: qcow2
+         distro: centos7
+         elements:
+           - overcloud-compute
+           - other-element
+         packages:
+           - vim
+         options:
+

doc/source/index.rst

@@ -14,6 +14,7 @@ Contents:
    readme
    installation
    usage
+   images
    contributing
 
 Indices and tables

requirements.txt

@@ -5,3 +5,6 @@
 pbr>=1.6 # Apache-2.0
 Babel>=1.3 # BSD
 python-heatclient>=0.6.0 # Apache-2.0
+oslo.config>=3.7.0 # Apache-2.0
+oslo.log>=1.14.0 # Apache-2.0
+oslo.utils>=3.5.0 # Apache-2.0

scripts/tripleo-build-images

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import argparse
+import os
+import subprocess
+import sys
+import yaml
+
+from oslo_config import cfg
+from oslo_config import types
+from oslo_log import log
+
+
+from tripleo_common.image.build import ImageBuildManager
+
+LOG = log.getLogger(__name__)
+env = os.environ.copy()
+
+CONF = cfg.CONF
+log.register_options(CONF)
+image_opt_group = cfg.OptGroup(name='image',
+                               title='Image build options')
+_opts = [
+    cfg.MultiOpt('config-file',
+                 item_type=types.String(),
+                 default=['disk_images.yaml'],
+                 help=("""Path to configuration file. Can be specified """
+                       """multiple times"""),
+                ),
+    cfg.StrOpt('output-directory',
+               default=env.get('TRIPLEO_ROOT', '.'),
+               help=("""output directory for images. """
+                     """Defaults to $TRIPLEO_ROOT, or current directory""")
+              ),
+    cfg.BoolOpt('skip',
+                default=False,
+                help="""Skip build if cached image exists."""
+               ),
+]
+CONF.register_group(image_opt_group)
+CONF.register_cli_opts(_opts, group=image_opt_group)
+log.setup(CONF, 'build-overcloud-images')
+
+def main(argv=sys.argv):
+    CONF(argv[1:])
+    LOG.info('Using config files at: %s' % CONF.image.config_file)
+
+    manager = ImageBuildManager(CONF.image.config_file,
+                                output_directory=CONF.image.output_directory,
+                                skip=CONF.image.skip)
+    manager.build()
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

setup.cfg

@@ -24,6 +24,7 @@ packages =
 
 scripts =
     scripts/upgrade-non-controller.sh
+    scripts/tripleo-build-images
 
 data_files =
     share/tripleo-common/templates = templates/*

tripleo_common/image/base.py

@@ -0,0 +1,76 @@
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import six
+
+import os
+import yaml
+
+from oslo_log import log
+
+from tripleo_common.image.exception import ImageSpecificationException
+
+
+class BaseImageManager(object):
+    logger = log.getLogger(__name__ + '.BaseImageManager')
+    APPEND_ATTRIBUTES = ['elements', 'options', 'packages']
+
+    def __init__(self, config_files):
+        self.config_files = config_files
+
+    def _extend_or_set_attribute(self, existing_image, image, attribute_name):
+        attribute = image.get(attribute_name, [])
+        if attribute:
+            try:
+                existing_image[attribute_name].extend(attribute)
+            except KeyError:
+                existing_image[attribute_name] = attribute
+
+    def load_config_files(self):
+        disk_images = {}
+        for config_file in self.config_files:
+            if os.path.isfile(config_file):
+                with open(config_file) as cf:
+                    images = yaml.load(cf.read()).get("disk_images")
+                    self.logger.debug(
+                        'disk_images JSON: %s' % str(disk_images))
+                for image in images:
+                    image_name = image.get('imagename')
+                    if image_name is None:
+                        msg = 'imagename is required'
+                        self.logger.error(msg)
+                        raise ImageSpecificationException(msg)
+
+                    existing_image = disk_images.get(image_name)
+                    if not existing_image:
+                        disk_images[image_name] = image
+                        continue
+
+                    for attr in self.APPEND_ATTRIBUTES:
+                        self._extend_or_set_attribute(existing_image, image,
+                                                      attr)
+
+                    # If a new key is introduced, add it.
+                    for key, value in six.iteritems(image):
+                        if key not in existing_image:
+                            existing_image[key] = image[key]
+
+                    disk_images[image_name] = existing_image
+
+            else:
+                self.logger.error('No config file exists at: %s' % config_file)
+                raise IOError('No config file exists at: %s' % config_file)
+
+        return [x for x in disk_images.values()]

tripleo_common/image/build.py

@@ -0,0 +1,80 @@
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import os
+import re
+
+from oslo_log import log
+from oslo_utils import strutils
+
+from tripleo_common.image.base import BaseImageManager
+from tripleo_common.image.exception import ImageSpecificationException
+from tripleo_common.image.image_builder import ImageBuilder
+
+
+class ImageBuildManager(BaseImageManager):
+    """Manage the building of image files
+
+       Manage the building of images from a config file specified in YAML
+       syntax. Multiple config files can be specified. They will be merged
+       """
+    logger = log.getLogger(__name__ + '.ImageBuildManager')
+
+    def __init__(self, config_files, output_directory='.', skip=False):
+        super(ImageBuildManager, self).__init__(config_files)
+        self.output_directory = re.sub('[/]$', '', output_directory)
+        self.skip = skip
+
+    def build(self):
+        """Start the build process"""
+
+        self.logger.info('Using config files: %s' % self.config_files)
+
+        disk_images = self.load_config_files()
+
+        for image in disk_images:
+            arch = image.get('arch', 'amd64')
+            image_type = image.get('type', 'qcow2')
+            image_name = image.get('imagename')
+            builder = image.get('builder', 'dib')
+            skip_base = strutils.bool_from_string(
+                image.get('skip_base', False))
+            docker_target = image.get('docker_target')
+            node_dist = image.get('distro')
+            if node_dist is None:
+                raise ImageSpecificationException('distro is required')
+            self.logger.info('imagename: %s' % image_name)
+            image_extension = image.get('imageext', image_type)
+            image_path = '%s/%s.%s' % (
+                self.output_directory, image_name, image_extension)
+            if self.skip:
+                self.logger.info('looking for image at path: %s' % image_path)
+                if os.path.exists(image_path):
+                    self.logger.info('Image file exists for image name: %s' %
+                                     image_name)
+                    self.logger.info('Skipping image build')
+                    continue
+            elements = image.get('elements', [])
+            options = image.get('options', [])
+            packages = image.get('packages', [])
+
+            extra_options = {
+                'skip_base': skip_base,
+                'docker_target': docker_target,
+            }
+
+            builder = ImageBuilder.get_builder(builder)
+            builder.build_image(image_path, image_type, node_dist, arch,
+                                elements, options, packages, extra_options)

tripleo_common/image/exception.py

@@ -0,0 +1,22 @@
+#   Copyright 2015 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+
+class ImageBuilderException(Exception):
+    pass
+
+
+class ImageSpecificationException(Exception):
+    pass

tripleo_common/image/image_builder.py

@@ -0,0 +1,89 @@
+#   Copyright 2015 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+
+import abc
+import logging
+import os
+import six
+import subprocess
+
+from tripleo_common.image.exception import ImageBuilderException
+
+
+@six.add_metaclass(abc.ABCMeta)
+class ImageBuilder(object):
+    """Base representation of an image building method"""
+
+    @staticmethod
+    def get_builder(builder):
+        if builder == 'dib':
+            return DibImageBuilder()
+        raise ImageBuilderException('Unknown image builder type')
+
+    @abc.abstractmethod
+    def build_image(self, image_path, image_type, node_dist, arch, elements,
+                    options, packages, extra_options={}):
+        """Build a disk image"""
+        pass
+
+
+class DibImageBuilder(ImageBuilder):
+    """Build images using diskimage-builder"""
+
+    logger = logging.getLogger(__name__ + '.DibImageBuilder')
+
+    def build_image(self, image_path, image_type, node_dist, arch, elements,
+                    options, packages, extra_options={}):
+        env = os.environ.copy()
+
+        elements_path = env.get('ELEMENTS_PATH')
+        if elements_path is None:
+            env['ELEMENTS_PATH'] = os.pathsep.join([
+                "/usr/share/tripleo-puppet-elements",
+                "/usr/share/instack-undercloud",
+                '/usr/share/tripleo-image-elements',
+                '/usr/share/openstack-heat-templates/'
+                'software-config/elements',
+            ])
+            os.environ.update(env)
+
+        cmd = ['disk-image-create', '-a', arch, '-o', image_path,
+               '-t', image_type]
+
+        if packages:
+            cmd.append('-p')
+            cmd.append(','.join(packages))
+
+        if options:
+            for option in options:
+                cmd.extend(option.split(' '))
+
+        skip_base = extra_options.get('skip_base', False)
+        if skip_base:
+            cmd.append('-n')
+
+        docker_target = extra_options.get('docker_target')
+        if docker_target:
+            cmd.append('--docker-target')
+            cmd.append(docker_target)
+
+        if node_dist:
+            cmd.append(node_dist)
+
+        cmd.extend(elements)
+
+        self.logger.info('Running %s' % cmd)
+        subprocess.check_call(cmd)

tripleo_common/tests/image/fakes.py

@@ -0,0 +1,27 @@
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+def create_disk_images():
+    disk_images = {
+        'disk_images': [{
+            'arch': 'amd64',
+            'distro': 'some_awesome_os',
+            'imagename': 'overcloud',
+            'type': 'qcow2',
+            'elements': ['image_element']
+        }]
+    }
+
+    return disk_images

tripleo_common/tests/image/test_base.py

@@ -0,0 +1,120 @@
+#   Copyright 2015 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+
+import mock
+
+from tripleo_common.image.base import BaseImageManager
+from tripleo_common.image.exception import ImageSpecificationException
+from tripleo_common.tests import base as testbase
+from tripleo_common.tests.image import fakes
+
+
+class TestBaseImageManager(testbase.TestCase):
+    def setUp(self):
+        super(TestBaseImageManager, self).setUp()
+
+    @mock.patch('yaml.load', autospec=True)
+    @mock.patch('os.path.isfile', autospec=True)
+    def test_load_config_files(self, mock_os_path_isfile, mock_yaml_load):
+        mock_yaml_load.return_value = fakes.create_disk_images()
+
+        mock_os_path_isfile.return_value = True
+
+        mock_open_context = mock.mock_open()
+        mock_open_context().read.return_value = "YAML"
+
+        with mock.patch('six.moves.builtins.open', mock_open_context):
+            base_manager = BaseImageManager(['yamlfile'])
+            disk_images = base_manager.load_config_files()
+
+        mock_yaml_load.assert_called_once_with("YAML")
+        self.assertEqual([{
+            'arch': 'amd64',
+            'distro': 'some_awesome_os',
+            'imagename': 'overcloud',
+            'type': 'qcow2',
+            'elements': ['image_element']
+        }], disk_images)
+
+    def test_load_config_files_not_found(self):
+        base_manager = BaseImageManager(['file/does/not/exist'])
+        self.assertRaises(IOError, base_manager.load_config_files)
+
+    @mock.patch('yaml.load', autospec=True)
+    @mock.patch('os.path.isfile', autospec=True)
+    def test_load_config_files_multiple_files(self, mock_os_path_isfile,
+                                              mock_yaml_load):
+        mock_yaml_load.side_effect = [{
+            'disk_images': [{
+                'arch': 'amd64',
+                'imagename': 'overcloud',
+                'distro': 'some_awesome_distro',
+                'type': 'qcow2',
+                'elements': ['image_element']
+            }]},
+            {
+            'disk_images': [{
+                'imagename': 'overcloud',
+                'elements': ['another_image_element'],
+                'packages': ['a_package'],
+                'otherkey': 'some_other_key',
+            }]}]
+
+        mock_os_path_isfile.return_value = True
+
+        mock_open_context = mock.mock_open()
+        mock_open_context().read.return_value = "YAML"
+
+        with mock.patch('six.moves.builtins.open', mock_open_context):
+            base_manager = BaseImageManager(['yamlfile1', 'yamlfile2'])
+            disk_images = base_manager.load_config_files()
+
+        self.assertEqual(2, mock_yaml_load.call_count)
+        self.assertEqual([{
+            'arch': 'amd64',
+            'distro': 'some_awesome_distro',
+            'imagename': 'overcloud',
+            'type': 'qcow2',
+            'elements': ['image_element', 'another_image_element'],
+            'packages': ['a_package'],
+            'otherkey': 'some_other_key',
+        }], disk_images)
+
+    @mock.patch('yaml.load', autospec=True)
+    @mock.patch('os.path.isfile', autospec=True)
+    def test_load_config_files_missing_image_name(self, mock_os_path_isfile,
+                                                  mock_yaml_load):
+        mock_yaml_load.return_value = {
+            'disk_images': [{
+                'arch': 'amd64',
+                'imagename': 'overcloud',
+                'type': 'qcow2',
+                'elements': ['image_element']
+            }, {
+                'arch': 'amd64',
+                'type': 'qcow2',
+            }]
+        }
+
+        mock_os_path_isfile.return_value = True
+
+        mock_open_context = mock.mock_open()
+        mock_open_context().read.return_value = "YAML"
+
+        with mock.patch('six.moves.builtins.open', mock_open_context):
+            base_manager = BaseImageManager(['yamlfile'])
+            self.assertRaises(ImageSpecificationException,
+                              base_manager.load_config_files)

tripleo_common/tests/image/test_build.py

@@ -0,0 +1,80 @@
+#   Copyright 2015 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+import mock
+
+from tripleo_common.image.build import ImageBuildManager
+from tripleo_common.image.exception import ImageSpecificationException
+from tripleo_common.image.image_builder import ImageBuilder
+from tripleo_common.tests import base
+from tripleo_common.tests.image import fakes
+
+
+class TestImageBuildManager(base.TestCase):
+    def setUp(self):
+        super(TestImageBuildManager, self).setUp()
+
+    @mock.patch.object(ImageBuilder, 'get_builder')
+    @mock.patch('tripleo_common.image.base.BaseImageManager.load_config_files',
+                autospec=True)
+    def test_build(self, mock_load_config_files, mock_get_builder):
+        mock_load_config_files.return_value = fakes.create_disk_images().get(
+            'disk_images')
+
+        mock_builder = mock.Mock()
+        mock_get_builder.return_value = mock_builder
+
+        build_manager = ImageBuildManager(['config/file'])
+        build_manager.build()
+
+        self.assertEqual(1, mock_load_config_files.call_count)
+
+        mock_builder.build_image.assert_called_with(
+            './overcloud.qcow2', 'qcow2', 'some_awesome_os', 'amd64',
+            ['image_element'], [], [],
+            {'skip_base': False, 'docker_target': None})
+
+    @mock.patch.object(ImageBuilder, 'get_builder')
+    @mock.patch('tripleo_common.image.base.BaseImageManager.load_config_files',
+                autospec=True)
+    def test_build_no_distro(self, mock_load_config_files, mock_get_builder):
+        mock_load_config_files.return_value = [{
+            'imagename': 'overcloud',
+        }]
+
+        mock_builder = mock.Mock()
+        mock_get_builder.return_value = mock_builder
+
+        build_manager = ImageBuildManager(['config/file'])
+        self.assertRaises(ImageSpecificationException, build_manager.build)
+
+    @mock.patch('os.path.exists', autospec=True)
+    @mock.patch.object(ImageBuilder, 'get_builder')
+    @mock.patch('tripleo_common.image.base.BaseImageManager.load_config_files',
+                autospec=True)
+    def test_build_with_skip(self, mock_load_config_files, mock_get_builder,
+                             mock_os_path_exists):
+        mock_load_config_files.return_value = fakes.create_disk_images().get(
+            'disk_images')
+
+        mock_builder = mock.Mock()
+        mock_get_builder.return_value = mock_builder
+
+        mock_os_path_exists.return_value = True
+
+        build_manager = ImageBuildManager(['config/file'], skip=True)
+        build_manager.build()
+
+        self.assertEqual(1, mock_os_path_exists.call_count)

tripleo_common/tests/image/test_image_builder.py

@@ -0,0 +1,54 @@
+#   Copyright 2015 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+
+import mock
+
+from tripleo_common.image.exception import ImageBuilderException
+from tripleo_common.image.image_builder import DibImageBuilder
+from tripleo_common.image.image_builder import ImageBuilder
+from tripleo_common.tests import base
+
+
+class TestImageBuilder(base.TestCase):
+
+    def test_get_builder_dib(self):
+        builder = ImageBuilder.get_builder('dib')
+        assert isinstance(builder, DibImageBuilder)
+
+    def test_get_builder_unknown(self):
+        self.assertRaises(ImageBuilderException, ImageBuilder.get_builder,
+                          'unknown')
+
+
+class TestDibImageBuilder(base.TestCase):
+
+    def setUp(self):
+        super(TestDibImageBuilder, self).setUp()
+        self.builder = DibImageBuilder()
+
+    @mock.patch('subprocess.check_call')
+    def test_build_image(self, mock_check_call):
+        self.builder.build_image('image/path', 'imgtype', 'node_dist', 'arch',
+                                 ['element1', 'element2'], ['options'],
+                                 ['package1', 'package2'],
+                                 {'skip_base': True,
+                                  'docker_target': 'docker-target'})
+        mock_check_call.assert_called_once_with(
+            ['disk-image-create', '-a', 'arch', '-o', 'image/path',
+             '-t', 'imgtype',
+             '-p', 'package1,package2', 'options', '-n',
+             '--docker-target', 'docker-target', 'node_dist',
+             'element1', 'element2'])