Build image files from definitions in yaml

This change introduces a new way of defining image files. Using yaml,
the image file definitions can now be externalized, making updates to
the images being built easier. This will also allow for easier
customization of the images without having to change the actual code.

Change-Id: I321fa12f6a85c6a71cbd6e4e5ed9db3bfa4cb1c7
This commit is contained in:
Brad P. Crochet 2015-12-08 12:33:09 -05:00 committed by Jeff Peeler
parent e7441531db
commit 981c084014
16 changed files with 670 additions and 0 deletions

View File

@ -0,0 +1,20 @@
===============
Building images
===============
Call the image build manager::
manager = ImageBuildManager(['path/to/config.yaml'])
manager.build()
.. autoclass:: tripleo_common.image.build.ImageBuildManager
:members:
Multiple config files
---------------------
Multiple config files can be passed to the ImageBuildManager. Certain attributes
will be merged (currently, 'elements', 'options', and 'packages'), while other
attributes will only be set by the first encountered. The 'imagename' attribute
will be the primary key.

29
doc/source/images.rst Normal file
View File

@ -0,0 +1,29 @@
===========
Disk images
===========
.. toctree::
:glob:
:maxdepth: 2
image/*
YAML file format
----------------
::
disk_images:
-
imagename: overcloud-compute
builder: dib
arch: amd64
type: qcow2
distro: centos7
elements:
- overcloud-compute
- other-element
packages:
- vim
options:

View File

@ -14,6 +14,7 @@ Contents:
readme
installation
usage
images
contributing
Indices and tables

View File

@ -5,3 +5,6 @@
pbr>=1.6 # Apache-2.0
Babel>=1.3 # BSD
python-heatclient>=0.6.0 # Apache-2.0
oslo.config>=3.7.0 # Apache-2.0
oslo.log>=1.14.0 # Apache-2.0
oslo.utils>=3.5.0 # Apache-2.0

68
scripts/tripleo-build-images Executable file
View File

@ -0,0 +1,68 @@
#!/usr/bin/env python
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import os
import subprocess
import sys
import yaml
from oslo_config import cfg
from oslo_config import types
from oslo_log import log
from tripleo_common.image.build import ImageBuildManager
LOG = log.getLogger(__name__)
env = os.environ.copy()
CONF = cfg.CONF
log.register_options(CONF)
image_opt_group = cfg.OptGroup(name='image',
title='Image build options')
_opts = [
cfg.MultiOpt('config-file',
item_type=types.String(),
default=['disk_images.yaml'],
help=("""Path to configuration file. Can be specified """
"""multiple times"""),
),
cfg.StrOpt('output-directory',
default=env.get('TRIPLEO_ROOT', '.'),
help=("""output directory for images. """
"""Defaults to $TRIPLEO_ROOT, or current directory""")
),
cfg.BoolOpt('skip',
default=False,
help="""Skip build if cached image exists."""
),
]
CONF.register_group(image_opt_group)
CONF.register_cli_opts(_opts, group=image_opt_group)
log.setup(CONF, 'build-overcloud-images')
def main(argv=sys.argv):
CONF(argv[1:])
LOG.info('Using config files at: %s' % CONF.image.config_file)
manager = ImageBuildManager(CONF.image.config_file,
output_directory=CONF.image.output_directory,
skip=CONF.image.skip)
manager.build()
if __name__ == '__main__':
sys.exit(main(sys.argv))

View File

@ -24,6 +24,7 @@ packages =
scripts =
scripts/upgrade-non-controller.sh
scripts/tripleo-build-images
data_files =
share/tripleo-common/templates = templates/*

View File

View File

@ -0,0 +1,76 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import six
import os
import yaml
from oslo_log import log
from tripleo_common.image.exception import ImageSpecificationException
class BaseImageManager(object):
logger = log.getLogger(__name__ + '.BaseImageManager')
APPEND_ATTRIBUTES = ['elements', 'options', 'packages']
def __init__(self, config_files):
self.config_files = config_files
def _extend_or_set_attribute(self, existing_image, image, attribute_name):
attribute = image.get(attribute_name, [])
if attribute:
try:
existing_image[attribute_name].extend(attribute)
except KeyError:
existing_image[attribute_name] = attribute
def load_config_files(self):
disk_images = {}
for config_file in self.config_files:
if os.path.isfile(config_file):
with open(config_file) as cf:
images = yaml.load(cf.read()).get("disk_images")
self.logger.debug(
'disk_images JSON: %s' % str(disk_images))
for image in images:
image_name = image.get('imagename')
if image_name is None:
msg = 'imagename is required'
self.logger.error(msg)
raise ImageSpecificationException(msg)
existing_image = disk_images.get(image_name)
if not existing_image:
disk_images[image_name] = image
continue
for attr in self.APPEND_ATTRIBUTES:
self._extend_or_set_attribute(existing_image, image,
attr)
# If a new key is introduced, add it.
for key, value in six.iteritems(image):
if key not in existing_image:
existing_image[key] = image[key]
disk_images[image_name] = existing_image
else:
self.logger.error('No config file exists at: %s' % config_file)
raise IOError('No config file exists at: %s' % config_file)
return [x for x in disk_images.values()]

View File

@ -0,0 +1,80 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import re
from oslo_log import log
from oslo_utils import strutils
from tripleo_common.image.base import BaseImageManager
from tripleo_common.image.exception import ImageSpecificationException
from tripleo_common.image.image_builder import ImageBuilder
class ImageBuildManager(BaseImageManager):
"""Manage the building of image files
Manage the building of images from a config file specified in YAML
syntax. Multiple config files can be specified. They will be merged
"""
logger = log.getLogger(__name__ + '.ImageBuildManager')
def __init__(self, config_files, output_directory='.', skip=False):
super(ImageBuildManager, self).__init__(config_files)
self.output_directory = re.sub('[/]$', '', output_directory)
self.skip = skip
def build(self):
"""Start the build process"""
self.logger.info('Using config files: %s' % self.config_files)
disk_images = self.load_config_files()
for image in disk_images:
arch = image.get('arch', 'amd64')
image_type = image.get('type', 'qcow2')
image_name = image.get('imagename')
builder = image.get('builder', 'dib')
skip_base = strutils.bool_from_string(
image.get('skip_base', False))
docker_target = image.get('docker_target')
node_dist = image.get('distro')
if node_dist is None:
raise ImageSpecificationException('distro is required')
self.logger.info('imagename: %s' % image_name)
image_extension = image.get('imageext', image_type)
image_path = '%s/%s.%s' % (
self.output_directory, image_name, image_extension)
if self.skip:
self.logger.info('looking for image at path: %s' % image_path)
if os.path.exists(image_path):
self.logger.info('Image file exists for image name: %s' %
image_name)
self.logger.info('Skipping image build')
continue
elements = image.get('elements', [])
options = image.get('options', [])
packages = image.get('packages', [])
extra_options = {
'skip_base': skip_base,
'docker_target': docker_target,
}
builder = ImageBuilder.get_builder(builder)
builder.build_image(image_path, image_type, node_dist, arch,
elements, options, packages, extra_options)

View File

@ -0,0 +1,22 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
class ImageBuilderException(Exception):
pass
class ImageSpecificationException(Exception):
pass

View File

@ -0,0 +1,89 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import abc
import logging
import os
import six
import subprocess
from tripleo_common.image.exception import ImageBuilderException
@six.add_metaclass(abc.ABCMeta)
class ImageBuilder(object):
"""Base representation of an image building method"""
@staticmethod
def get_builder(builder):
if builder == 'dib':
return DibImageBuilder()
raise ImageBuilderException('Unknown image builder type')
@abc.abstractmethod
def build_image(self, image_path, image_type, node_dist, arch, elements,
options, packages, extra_options={}):
"""Build a disk image"""
pass
class DibImageBuilder(ImageBuilder):
"""Build images using diskimage-builder"""
logger = logging.getLogger(__name__ + '.DibImageBuilder')
def build_image(self, image_path, image_type, node_dist, arch, elements,
options, packages, extra_options={}):
env = os.environ.copy()
elements_path = env.get('ELEMENTS_PATH')
if elements_path is None:
env['ELEMENTS_PATH'] = os.pathsep.join([
"/usr/share/tripleo-puppet-elements",
"/usr/share/instack-undercloud",
'/usr/share/tripleo-image-elements',
'/usr/share/openstack-heat-templates/'
'software-config/elements',
])
os.environ.update(env)
cmd = ['disk-image-create', '-a', arch, '-o', image_path,
'-t', image_type]
if packages:
cmd.append('-p')
cmd.append(','.join(packages))
if options:
for option in options:
cmd.extend(option.split(' '))
skip_base = extra_options.get('skip_base', False)
if skip_base:
cmd.append('-n')
docker_target = extra_options.get('docker_target')
if docker_target:
cmd.append('--docker-target')
cmd.append(docker_target)
if node_dist:
cmd.append(node_dist)
cmd.extend(elements)
self.logger.info('Running %s' % cmd)
subprocess.check_call(cmd)

View File

View File

@ -0,0 +1,27 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
def create_disk_images():
disk_images = {
'disk_images': [{
'arch': 'amd64',
'distro': 'some_awesome_os',
'imagename': 'overcloud',
'type': 'qcow2',
'elements': ['image_element']
}]
}
return disk_images

View File

@ -0,0 +1,120 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import mock
from tripleo_common.image.base import BaseImageManager
from tripleo_common.image.exception import ImageSpecificationException
from tripleo_common.tests import base as testbase
from tripleo_common.tests.image import fakes
class TestBaseImageManager(testbase.TestCase):
def setUp(self):
super(TestBaseImageManager, self).setUp()
@mock.patch('yaml.load', autospec=True)
@mock.patch('os.path.isfile', autospec=True)
def test_load_config_files(self, mock_os_path_isfile, mock_yaml_load):
mock_yaml_load.return_value = fakes.create_disk_images()
mock_os_path_isfile.return_value = True
mock_open_context = mock.mock_open()
mock_open_context().read.return_value = "YAML"
with mock.patch('six.moves.builtins.open', mock_open_context):
base_manager = BaseImageManager(['yamlfile'])
disk_images = base_manager.load_config_files()
mock_yaml_load.assert_called_once_with("YAML")
self.assertEqual([{
'arch': 'amd64',
'distro': 'some_awesome_os',
'imagename': 'overcloud',
'type': 'qcow2',
'elements': ['image_element']
}], disk_images)
def test_load_config_files_not_found(self):
base_manager = BaseImageManager(['file/does/not/exist'])
self.assertRaises(IOError, base_manager.load_config_files)
@mock.patch('yaml.load', autospec=True)
@mock.patch('os.path.isfile', autospec=True)
def test_load_config_files_multiple_files(self, mock_os_path_isfile,
mock_yaml_load):
mock_yaml_load.side_effect = [{
'disk_images': [{
'arch': 'amd64',
'imagename': 'overcloud',
'distro': 'some_awesome_distro',
'type': 'qcow2',
'elements': ['image_element']
}]},
{
'disk_images': [{
'imagename': 'overcloud',
'elements': ['another_image_element'],
'packages': ['a_package'],
'otherkey': 'some_other_key',
}]}]
mock_os_path_isfile.return_value = True
mock_open_context = mock.mock_open()
mock_open_context().read.return_value = "YAML"
with mock.patch('six.moves.builtins.open', mock_open_context):
base_manager = BaseImageManager(['yamlfile1', 'yamlfile2'])
disk_images = base_manager.load_config_files()
self.assertEqual(2, mock_yaml_load.call_count)
self.assertEqual([{
'arch': 'amd64',
'distro': 'some_awesome_distro',
'imagename': 'overcloud',
'type': 'qcow2',
'elements': ['image_element', 'another_image_element'],
'packages': ['a_package'],
'otherkey': 'some_other_key',
}], disk_images)
@mock.patch('yaml.load', autospec=True)
@mock.patch('os.path.isfile', autospec=True)
def test_load_config_files_missing_image_name(self, mock_os_path_isfile,
mock_yaml_load):
mock_yaml_load.return_value = {
'disk_images': [{
'arch': 'amd64',
'imagename': 'overcloud',
'type': 'qcow2',
'elements': ['image_element']
}, {
'arch': 'amd64',
'type': 'qcow2',
}]
}
mock_os_path_isfile.return_value = True
mock_open_context = mock.mock_open()
mock_open_context().read.return_value = "YAML"
with mock.patch('six.moves.builtins.open', mock_open_context):
base_manager = BaseImageManager(['yamlfile'])
self.assertRaises(ImageSpecificationException,
base_manager.load_config_files)

View File

@ -0,0 +1,80 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import mock
from tripleo_common.image.build import ImageBuildManager
from tripleo_common.image.exception import ImageSpecificationException
from tripleo_common.image.image_builder import ImageBuilder
from tripleo_common.tests import base
from tripleo_common.tests.image import fakes
class TestImageBuildManager(base.TestCase):
def setUp(self):
super(TestImageBuildManager, self).setUp()
@mock.patch.object(ImageBuilder, 'get_builder')
@mock.patch('tripleo_common.image.base.BaseImageManager.load_config_files',
autospec=True)
def test_build(self, mock_load_config_files, mock_get_builder):
mock_load_config_files.return_value = fakes.create_disk_images().get(
'disk_images')
mock_builder = mock.Mock()
mock_get_builder.return_value = mock_builder
build_manager = ImageBuildManager(['config/file'])
build_manager.build()
self.assertEqual(1, mock_load_config_files.call_count)
mock_builder.build_image.assert_called_with(
'./overcloud.qcow2', 'qcow2', 'some_awesome_os', 'amd64',
['image_element'], [], [],
{'skip_base': False, 'docker_target': None})
@mock.patch.object(ImageBuilder, 'get_builder')
@mock.patch('tripleo_common.image.base.BaseImageManager.load_config_files',
autospec=True)
def test_build_no_distro(self, mock_load_config_files, mock_get_builder):
mock_load_config_files.return_value = [{
'imagename': 'overcloud',
}]
mock_builder = mock.Mock()
mock_get_builder.return_value = mock_builder
build_manager = ImageBuildManager(['config/file'])
self.assertRaises(ImageSpecificationException, build_manager.build)
@mock.patch('os.path.exists', autospec=True)
@mock.patch.object(ImageBuilder, 'get_builder')
@mock.patch('tripleo_common.image.base.BaseImageManager.load_config_files',
autospec=True)
def test_build_with_skip(self, mock_load_config_files, mock_get_builder,
mock_os_path_exists):
mock_load_config_files.return_value = fakes.create_disk_images().get(
'disk_images')
mock_builder = mock.Mock()
mock_get_builder.return_value = mock_builder
mock_os_path_exists.return_value = True
build_manager = ImageBuildManager(['config/file'], skip=True)
build_manager.build()
self.assertEqual(1, mock_os_path_exists.call_count)

View File

@ -0,0 +1,54 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import mock
from tripleo_common.image.exception import ImageBuilderException
from tripleo_common.image.image_builder import DibImageBuilder
from tripleo_common.image.image_builder import ImageBuilder
from tripleo_common.tests import base
class TestImageBuilder(base.TestCase):
def test_get_builder_dib(self):
builder = ImageBuilder.get_builder('dib')
assert isinstance(builder, DibImageBuilder)
def test_get_builder_unknown(self):
self.assertRaises(ImageBuilderException, ImageBuilder.get_builder,
'unknown')
class TestDibImageBuilder(base.TestCase):
def setUp(self):
super(TestDibImageBuilder, self).setUp()
self.builder = DibImageBuilder()
@mock.patch('subprocess.check_call')
def test_build_image(self, mock_check_call):
self.builder.build_image('image/path', 'imgtype', 'node_dist', 'arch',
['element1', 'element2'], ['options'],
['package1', 'package2'],
{'skip_base': True,
'docker_target': 'docker-target'})
mock_check_call.assert_called_once_with(
['disk-image-create', '-a', 'arch', '-o', 'image/path',
'-t', 'imgtype',
'-p', 'package1,package2', 'options', '-n',
'--docker-target', 'docker-target', 'node_dist',
'element1', 'element2'])