Make overcloud_containers.yaml template driven

Currently container based CI jobs are implementing their own logic to
transform overcloud_containers.yaml to a file that reflects the
container images and registry required to run their job [1] unless the
job requires the exact images which the static
overcloud_containers.yaml provides[2].

This change adds a template overcloud_containers.yaml.j2 which will
exist in parallel to overcloud_containers.yaml until all downstream
users of overcloud_containers.yaml switch to a templated approach.

This change is the first step to write commands which allow
overcloud_containers.yaml to be built for a specific enviroment,
including version based tags and reducing the image list to those
needed by the deployment.

The test test_container_images_yaml_in_sync will ensure that any
changes made to overcloud_containers.yaml won't pass until they're
made to overcloud_containers.yaml.j2 as well. This will keep them in
sync until overcloud_containers.yaml is removed.

[1] https://github.com/rdo-infra/ansible-role-rdo-kolla-build/blob/master/tasks/main.yml#L39-L75
[2] https://github.com/openstack/tripleo-quickstart-extras/blob/master/roles/overcloud-prep-containers/templates/overcloud-prep-containers.sh.j2#L18

Change-Id: I4fe18565362f9f308dd9957aeb9bc0f7498590a6
Partial-Bug: #1696598

container-images/overcloud_containers.yaml.j2

@@ -0,0 +1,80 @@
+{% set namespace=namespace or "tripleoupstream" %}
+{% set name_prefix=name_prefix or "centos-binary-" %}
+{% set name_suffix=name_suffix or "" %}
+{% set tag=tag or "latest" %}
+container_images_template:
+- imagename: "{{namespace}}/{{name_prefix}}aodh-api{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}aodh-evaluator{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}aodh-listener{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}aodh-notifier{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}ceilometer-central{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}ceilometer-compute{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}ceilometer-notification{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}ceilometer-ipmi{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}cinder-api{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}cinder-backup{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}cinder-scheduler{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}cinder-volume{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}collectd{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}congress-api{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}ec2-api{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}etcd{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}glance-api{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}gnocchi-api{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}gnocchi-metricd{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}gnocchi-statsd{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}haproxy{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}heat-api-cfn{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}heat-api{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}heat-engine{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}horizon{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}ironic-api{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}ironic-conductor{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}ironic-pxe{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}iscsid{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}keystone{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}manila-api{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}manila-base{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}manila-scheduler{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}mariadb{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}memcached{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}mistral-api{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}mistral-engine{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}mistral-executor{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}mistral-event-engine{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}mongodb{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}multipathd{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}neutron-dhcp-agent{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}neutron-l3-agent{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}neutron-metadata-agent{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}neutron-openvswitch-agent{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}neutron-sriov-agent{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}neutron-server{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}nova-api{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}nova-base{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}nova-compute-ironic{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}nova-compute{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}nova-conductor{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}nova-consoleauth{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}nova-libvirt{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}nova-novncproxy{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}nova-placement-api{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}nova-scheduler{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}octavia-base{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}octavia-api{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}octavia-health-manager{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}octavia-housekeeping{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}octavia-worker{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}panko-api{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}rabbitmq{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}redis{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}sahara-api{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}sahara-engine{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}sensu-client{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}swift-account{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}swift-container{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}swift-object{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}swift-proxy-server{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}tacker{{name_suffix}}:{{tag}}"
+- imagename: "{{namespace}}/{{name_prefix}}zaqar{{name_suffix}}:{{tag}}"
+- imagename: "ceph/daemon:tag-build-master-jewel-centos-7"

tripleo_common/image/base.py

@@ -12,6 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import collections
 import json
 import os
 import yaml
@@ -25,9 +26,11 @@ class BaseImageManager(object):
     logger = log.getLogger(__name__ + '.BaseImageManager')
     APPEND_ATTRIBUTES = ['elements', 'options', 'packages']
     CONFIG_SECTIONS = (
-        DISK_IMAGES, UPLOADS, CONTAINER_IMAGES
+        DISK_IMAGES, UPLOADS, CONTAINER_IMAGES,
+        CONTAINER_IMAGES_TEMPLATE
     ) = (
-        'disk_images', 'uploads', 'container_images'
+        'disk_images', 'uploads', 'container_images',
+        'container_images_template'
     )
 
     def __init__(self, config_files, images=None):
@@ -45,7 +48,7 @@ class BaseImageManager(object):
                 existing_image[attribute_name] = attribute
 
     def load_config_files(self, section):
-        config_data = {}
+        config_data = collections.OrderedDict()
         for config_file in self.config_files:
             if os.path.isfile(config_file):
                 with open(config_file) as cf:

tripleo_common/image/kolla_builder.py

@@ -14,19 +14,16 @@
 #
 
 
+import jinja2
 import logging
 import os
 import re
 import subprocess
 import sys
+import yaml
 
 from tripleo_common.image import base
 
-if sys.version_info[0] < 3:
-    import codecs
-    _open = open
-    open = codecs.open
-
 
 class KollaImageBuilder(base.BaseImageManager):
     """Build images using kolla-build"""
@@ -53,6 +50,44 @@ class KollaImageBuilder(base.BaseImageManager):
         # what results should be acceptable as a regex to build one image
         return imagename
 
+    def container_images_from_template(self, filter=None, **kwargs):
+        '''Build container_images data from container_images_template.
+
+        Any supplied keyword arguments are used for the substitution mapping to
+        transform the data in the config file container_images_template
+        section.
+
+        The resulting data resembles a config file which contains a valid
+        populated container_images section.
+
+        If a function is passed to the filter argument, this will be used to
+        modify the entry after substitution. If the filter function returns
+        None then the entry will not be added to the resulting list.
+
+        Defaults are applied so that when no arguments are provided the
+        resulting entries have the form:
+        - imagename: tripleoupstream/centos-binary-<name>:latest
+        '''
+        mapping = dict(kwargs)
+
+        result = []
+
+        if len(self.config_files) != 1:
+            raise ValueError('A single config file must be specified')
+        config_file = self.config_files[0]
+        with open(config_file) as cf:
+            template = jinja2.Template(cf.read())
+
+        rendered = template.render(mapping)
+        rendered_dict = yaml.safe_load(rendered)
+        for i in rendered_dict[self.CONTAINER_IMAGES_TEMPLATE]:
+            entry = dict(i)
+            if filter:
+                entry = filter(entry)
+            if entry is not None:
+                result.append(entry)
+        return result
+
     def build_images(self, kolla_config_files=None):
 
         cmd = ['kolla-build']

tripleo_common/tests/image/test_kolla_builder.py

@@ -18,24 +18,45 @@ import mock
 import os
 import six
 import subprocess
+import sys
+import tempfile
+import yaml
 
 from tripleo_common.image import kolla_builder as kb
 from tripleo_common.tests import base
 
 
-filedata = six.u(
+filedata = six.u("""container_images:
-    """container_images:
+- imagename: tripleoupstream/heat-docker-agents-centos:latest
-    - imagename: tripleoupstream/heat-docker-agents-centos:latest
+  push_destination: localhost:8787
-      push_destination: localhost:8787
+- imagename: tripleoupstream/centos-binary-nova-compute:liberty
-    - imagename: tripleoupstream/centos-binary-nova-compute:liberty
+  uploader: docker
-      uploader: docker
+  pull_source: docker.io
-      pull_source: docker.io
+  push_destination: localhost:8787
-      push_destination: localhost:8787
+- imagename: tripleoupstream/centos-binary-nova-libvirt:liberty
-    - imagename: tripleoupstream/centos-binary-nova-libvirt:liberty
+  uploader: docker
-      uploader: docker
+  pull_source: docker.io
-      pull_source: docker.io
+- imagename: tripleoupstream/image-with-missing-tag
-    - imagename: tripleoupstream/image-with-missing-tag
+  push_destination: localhost:8787
-      push_destination: localhost:8787
+""")
+
+template_filedata = six.u("""
+{% set namespace=namespace or "tripleoupstream" %}
+{% set name_prefix=name_prefix or "centos-binary-" %}
+{% set name_suffix=name_suffix or "" %}
+{% set tag=tag or "latest" %}
+container_images_template:
+- imagename: "{{namespace}}/heat-docker-agents-centos:latest"
+  push_destination: "{{push_destination}}"
+- imagename: "{{namespace}}/{{name_prefix}}nova-compute{{name_suffix}}:{{tag}}"
+  uploader: "docker"
+  pull_source: "{{pull_source}}"
+  push_destination: "{{push_destination}}"
+- imagename: "{{namespace}}/{{name_prefix}}nova-libvirt{{name_suffix}}:{{tag}}"
+  uploader: "docker"
+  pull_source: "{{pull_source}}"
+- imagename: "{{namespace}}/image-with-missing-tag"
+  push_destination: "{{push_destination}}"
 """)
 
 
@@ -108,3 +129,80 @@ class TestKollaImageBuilder(base.TestCase):
         mock_popen.assert_called_once_with([
             'kolla-build',
         ], env=env)
+
+
+class TestKollaImageBuilderTemplate(base.TestCase):
+
+    def setUp(self):
+        super(TestKollaImageBuilderTemplate, self).setUp()
+        with tempfile.NamedTemporaryFile(delete=False) as imagefile:
+            self.addCleanup(os.remove, imagefile.name)
+            self.filelist = [imagefile.name]
+            with open(imagefile.name, 'w') as f:
+                f.write(template_filedata)
+
+    def test_container_images_from_template(self):
+        builder = kb.KollaImageBuilder(self.filelist)
+        result = builder.container_images_from_template(
+            pull_source='docker.io',
+            push_destination='localhost:8787',
+            tag='liberty'
+        )
+        # template substitution on the container_images_template section should
+        # be identical to the container_images section
+        container_images = yaml.safe_load(filedata)['container_images']
+        self.assertEqual(container_images, result)
+
+    def test_container_images_from_template_filter(self):
+        builder = kb.KollaImageBuilder(self.filelist)
+
+        def filter(entry):
+
+            # do not want heat-agents image
+            if 'heat-docker-agents' in entry.get('imagename'):
+                return
+
+            # set source and destination on all entries
+            entry['pull_source'] = 'docker.io'
+            entry['push_destination'] = 'localhost:8787'
+            return entry
+
+        result = builder.container_images_from_template(
+            filter=filter,
+            tag='liberty'
+        )
+        container_images = [{
+            'imagename': 'tripleoupstream/centos-binary-nova-compute:liberty',
+            'pull_source': 'docker.io',
+            'push_destination': 'localhost:8787',
+            'uploader': 'docker'
+        }, {
+            'imagename': 'tripleoupstream/centos-binary-nova-libvirt:liberty',
+            'pull_source': 'docker.io',
+            'push_destination': 'localhost:8787',
+            'uploader': 'docker'
+        }, {
+            'imagename': 'tripleoupstream/image-with-missing-tag',
+            'pull_source': 'docker.io',
+            'push_destination': 'localhost:8787'
+        }]
+        self.assertEqual(container_images, result)
+
+    def test_container_images_yaml_in_sync(self):
+        '''Confirm overcloud_containers.tpl.yaml equals overcloud_containers.yaml
+
+        TODO(sbaker) remove when overcloud_containers.yaml is deleted
+        '''
+        mod_dir = os.path.dirname(sys.modules[__name__].__file__)
+        project_dir = os.path.abspath(os.path.join(mod_dir, '../../../'))
+        files_dir = os.path.join(project_dir, 'container-images')
+
+        oc_tmpl_file = os.path.join(files_dir, 'overcloud_containers.yaml.j2')
+        tmpl_builder = kb.KollaImageBuilder([oc_tmpl_file])
+        result = tmpl_builder.container_images_from_template()
+
+        oc_yaml_file = os.path.join(files_dir, 'overcloud_containers.yaml')
+        yaml_builder = kb.KollaImageBuilder([oc_yaml_file])
+        container_images = yaml_builder.load_config_files(
+            yaml_builder.CONTAINER_IMAGES)
+        self.assertSequenceEqual(container_images, result)