New command "overcloud container image prepare"

This new command allows the user to generate their own
overcloud_containers.yaml file in preperation for calling
"overcloud container image build" or
"overcloud container image upload".

Arguments allow the caller to customise the resulting
overcloud_containers.yaml file with the aim of not requiring any
further manual editing for the majority of cases.

The file generated by the --env-file option adopts the convention
established in change Ieaedac33f0a25a352ab432cdb00b5c888be4ba27 where
the DockerNamespace parameter is not used and image names are fully
qualified.

Partial-Bug: #1696598
Change-Id: I6af1828ea2b26f3c6295981fe40fe825d8ccef37

setup.cfg

@@ -66,6 +66,7 @@ openstack.tripleoclient.v1 =
     overcloud_netenv_validate = tripleoclient.v1.overcloud_netenv_validate:ValidateOvercloudNetenv
     overcloud_container_image_upload = tripleoclient.v1.container_image:UploadImage
     overcloud_container_image_build = tripleoclient.v1.container_image:BuildImage
+    overcloud_container_image_prepare = tripleoclient.v1.container_image:PrepareImageFiles
     overcloud_delete = tripleoclient.v1.overcloud_delete:DeleteOvercloud
     overcloud_credentials = tripleoclient.v1.overcloud_credentials:OvercloudCredentials
     overcloud_deploy = tripleoclient.v1.overcloud_deploy:DeployOvercloud

tripleoclient/tests/fakes.py

@@ -30,6 +30,7 @@ class FakeApp(object):
         self.stdout = _stdout or sys.stdout
         self.stderr = sys.stderr
         self.restapi = None
+        self.command_options = None
 
 
 class FakeClientManager(object):

tripleoclient/tests/v1/overcloud_image/test_container_image.py

@@ -14,6 +14,10 @@
 #
 
 import mock
+import os
+import shutil
+import tempfile
+import yaml
 
 from tripleoclient.tests.v1.test_plugin import TestPluginV1
 from tripleoclient.v1 import container_image
@@ -60,6 +64,106 @@ class TestContainerImageUpload(TestPluginV1):
         mock_manager.return_value.upload.assert_called_once_with()
 
 
+class TestContainerImagePrepare(TestPluginV1):
+
+    def setUp(self):
+        super(TestContainerImagePrepare, self).setUp()
+
+        # Get the command object to test
+        self.cmd = container_image.PrepareImageFiles(self.app, None)
+
+    @mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder')
+    def test_container_image_prepare_noargs(self, mock_builder):
+        arglist = []
+        verifylist = []
+        cift = mock.MagicMock()
+        cift.return_value = {}
+
+        mock_builder.return_value.container_images_from_template = cift
+
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        self.cmd.take_action(parsed_args)
+
+        mock_builder.assert_called_once_with([parsed_args.template_file])
+        cift.assert_called_once_with(
+            filter=mock.ANY,
+            name_prefix='centos-binary-',
+            name_suffix='',
+            namespace='tripleoupstream',
+            tag='latest'
+        )
+
+    @mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder')
+    def test_container_image_prepare(self, mock_builder):
+
+        temp = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, temp)
+        images_file = os.path.join(temp, 'overcloud_containers.yaml')
+        env_file = os.path.join(temp, 'containers_env.yaml')
+        tmpl_file = os.path.join(temp, 'overcloud_containers.yaml.j2')
+
+        arglist = [
+            '--template-file',
+            tmpl_file,
+            '--tag',
+            'passed-ci',
+            '--namespace',
+            'tripleo',
+            '--prefix',
+            'os-',
+            '--suffix',
+            'foo',
+            '--images-file',
+            images_file,
+            '--env-file',
+            env_file,
+        ]
+        self.cmd.app.command_options = arglist
+        verifylist = []
+        cift = mock.MagicMock()
+        cift.return_value = [{
+            'imagename': 'tripleo/os-aodh-apifoo:passed-ci',
+            'params': ['DockerAodhApiImage', 'DockerAodhConfigImage'],
+        }, {
+            'imagename': 'tripleo/os-heat-apifoo:passed-ci',
+            'params': ['DockerHeatApiImage'],
+        }]
+
+        mock_builder.return_value.container_images_from_template = cift
+
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        self.cmd.take_action(parsed_args)
+
+        mock_builder.assert_called_once_with([tmpl_file])
+        cift.assert_called_once_with(
+            filter=mock.ANY,
+            name_prefix='os-',
+            name_suffix='foo',
+            namespace='tripleo',
+            tag='passed-ci'
+        )
+        ci_data = {
+            'container_images': [{
+                'imagename': 'tripleo/os-aodh-apifoo:passed-ci',
+            }, {
+                'imagename': 'tripleo/os-heat-apifoo:passed-ci',
+            }]
+        }
+        env_data = {
+            'parameter_defaults': {
+                'DockerAodhApiImage': 'tripleo/os-aodh-apifoo:passed-ci',
+                'DockerAodhConfigImage': 'tripleo/os-aodh-apifoo:passed-ci',
+                'DockerHeatApiImage': 'tripleo/os-heat-apifoo:passed-ci',
+            }
+        }
+        with open(images_file) as f:
+            self.assertEqual(ci_data, yaml.safe_load(f))
+        with open(env_file) as f:
+            self.assertEqual(env_data, yaml.safe_load(f))
+
+
 class TestContainerImageBuild(TestPluginV1):
 
     def setUp(self):

tripleoclient/v1/container_image.py

@@ -13,10 +13,15 @@
 #   under the License.
 #
 
+import datetime
 import logging
+import os
+import re
+import sys
 
 from osc_lib.command import command
 from osc_lib.i18n import _
+import yaml
 
 from tripleo_common.image import image_uploader
 from tripleo_common.image import kolla_builder
@@ -88,3 +93,152 @@ class BuildImage(command.Command):
         self.log.debug("take_action(%s)" % parsed_args)
         builder = kolla_builder.KollaImageBuilder(parsed_args.config_files)
         builder.build_images(parsed_args.kolla_config_files)
+
+
+class PrepareImageFiles(command.Command):
+    """Generate files defining the images, tags and registry."""
+
+    auth_required = False
+    log = logging.getLogger(__name__ + ".PrepareImageFiles")
+
+    def get_parser(self, prog_name):
+        parser = super(PrepareImageFiles, self).get_parser(prog_name)
+        template_file = os.path.join(sys.prefix, 'share', 'tripleo-common',
+                                     'container-images',
+                                     'overcloud_containers.yaml.j2')
+        parser.add_argument(
+            "--template-file",
+            dest="template_file",
+            default=template_file,
+            metavar='<yaml template file>',
+            help=_("YAML template file which the images config file will be "
+                   "built from.\n"
+                   "Default: %s") % template_file,
+        )
+        parser.add_argument(
+            "--pull-source",
+            dest="pull_source",
+            metavar='<location>',
+            help=_("Location of image registry to pull images from. "
+                   "If specified, a pull_source will be set for every image "
+                   "entry."),
+        )
+        parser.add_argument(
+            "--push-destination",
+            dest="push_destination",
+            metavar='<location>',
+            help=_("Location of image registry to push images to. "
+                   "If specified, a push_destination will be set for every "
+                   "image entry."),
+        )
+        parser.add_argument(
+            "--tag",
+            dest="tag",
+            default="latest",
+            metavar='<tag>',
+            help=_("Override the default tag substitution.\n"
+                   "Default: latest"),
+        )
+        parser.add_argument(
+            "--namespace",
+            dest="namespace",
+            default="tripleoupstream",
+            metavar='<namespace>',
+            help=_("Override the default namespace substitution.\n"
+                   "Default: tripleoupstream"),
+        )
+        parser.add_argument(
+            "--prefix",
+            dest="prefix",
+            default="centos-binary-",
+            metavar='<prefix>',
+            help=_("Override the default name prefix substitution.\n"
+                   "Default: centos-binary-"),
+        )
+        parser.add_argument(
+            "--suffix",
+            dest="suffix",
+            default="",
+            metavar='<suffix>',
+            help=_("Override the default name suffix substitution.\n"
+                   "Default is empty."),
+        )
+        parser.add_argument(
+            "--exclude",
+            dest="excludes",
+            metavar='<regex>',
+            default=[],
+            action="append",
+            help=_("Pattern to match against resulting imagename entries to "
+                   "exclude from the final output. Can be specified multiple "
+                   "times."),
+        )
+        parser.add_argument(
+            "--images-file",
+            dest="images_file",
+            metavar='<file path>',
+            help=_("File to write resulting image entries to, as well as "
+                   "stdout. Any existing file will be overwritten."),
+        )
+        parser.add_argument(
+            "--env-file",
+            dest="env_file",
+            metavar='<file path>',
+            help=_("File to write heat environment file which specifies all "
+                   "image parameters. Any existing file will be overwritten."),
+        )
+        return parser
+
+    def write_env_file(self, result, env_file):
+        params = {}
+        for entry in result:
+            imagename = entry.get('imagename', '')
+            if 'params' in entry:
+                for p in entry.pop('params'):
+                    params[p] = imagename
+
+        with os.fdopen(os.open(env_file,
+                       os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o666),
+                       'w') as f:
+            f.write('# Generated with the following on %s\n#\n' %
+                    datetime.datetime.now().isoformat())
+            f.write('#   %s\n#\n\n' % ' '.join(self.app.command_options))
+
+            yaml.safe_dump({'parameter_defaults': params}, f,
+                           default_flow_style=False)
+
+    def take_action(self, parsed_args):
+        self.log.debug("take_action(%s)" % parsed_args)
+        subs = {
+            'tag': parsed_args.tag,
+            'namespace': parsed_args.namespace,
+            'name_prefix': parsed_args.prefix,
+            'name_suffix': parsed_args.suffix,
+        }
+
+        def ffunc(entry):
+            imagename = entry.get('imagename', '')
+            for p in parsed_args.excludes:
+                if re.search(p, imagename):
+                    return None
+            if parsed_args.pull_source:
+                entry['pull_source'] = parsed_args.pull_source
+            if parsed_args.push_destination:
+                entry['push_destination'] = parsed_args.push_destination
+            return entry
+
+        builder = kolla_builder.KollaImageBuilder([parsed_args.template_file])
+        result = builder.container_images_from_template(filter=ffunc, **subs)
+
+        if parsed_args.env_file:
+            self.write_env_file(result, parsed_args.env_file)
+
+        result_str = yaml.safe_dump({'container_images': result},
+                                    default_flow_style=False)
+        sys.stdout.write(result_str)
+
+        if parsed_args.images_file:
+            with os.fdopen(os.open(parsed_args.images_file,
+                           os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o666),
+                           'w') as f:
+                f.write(result_str)