Merge "Split off Ceph related container images" into stable/train

container-images/ceph.j2

@@ -0,0 +1,41 @@
+- imagename: "{{ceph_namespace}}/{{ceph_image}}:{{ceph_tag}}"
+  image_source: ceph
+  params:
+  - ContainerCephDaemonImage
+  services:
+  - OS::TripleO::Services::CephClient
+  - OS::TripleO::Services::CephExternal
+  - OS::TripleO::Services::CephMds
+  - OS::TripleO::Services::CephMgr
+  - OS::TripleO::Services::CephMon
+  - OS::TripleO::Services::CephOSD
+  - OS::TripleO::Services::CephRgw
+  - OS::TripleO::Services::CephRbdMirror
+
+- imagename: "{{ceph_prometheus_namespace}}/{{ceph_prometheus_image}}:{{ceph_prometheus_tag}}"
+  image_source: prom
+  params:
+  - PrometheusContainerImage
+  services:
+  - OS::TripleO::Services::CephGrafana
+
+- imagename: "{{ceph_alertmanager_namespace}}/{{ceph_alertmanager_image}}:{{ceph_alertmanager_tag}}"
+  image_source: prom
+  params:
+  - AlertManagerContainerImage
+  services:
+  - OS::TripleO::Services::CephGrafana
+
+- imagename: "{{ceph_node_exporter_namespace}}/{{ceph_node_exporter_image}}:{{ceph_node_exporter_tag}}"
+  image_source: prom
+  params:
+  - NodeExporterContainerImage
+  services:
+  - OS::TripleO::Services::CephGrafana
+
+- imagename: "{{ceph_grafana_namespace}}/{{ceph_grafana_image}}:{{ceph_grafana_tag}}"
+  image_source: grafana
+  params:
+  - GrafanaContainerImage
+  services:
+  - OS::TripleO::Services::CephGrafana

container-images/tripleo_containers.yaml.j2

@@ -871,47 +871,9 @@ container_images_template:
   services:
   - OS::TripleO::Services::Zaqar
 
-- imagename: "{{ceph_namespace}}/{{ceph_image}}:{{ceph_tag}}"
-  image_source: ceph
-  params:
-  - ContainerCephDaemonImage
-  services:
-  - OS::TripleO::Services::CephClient
-  - OS::TripleO::Services::CephExternal
-  - OS::TripleO::Services::CephMds
-  - OS::TripleO::Services::CephMgr
-  - OS::TripleO::Services::CephMon
-  - OS::TripleO::Services::CephOSD
-  - OS::TripleO::Services::CephRgw
-  - OS::TripleO::Services::CephRbdMirror
-
-- imagename: "{{ceph_prometheus_namespace}}/{{ceph_prometheus_image}}:{{ceph_prometheus_tag}}"
-  image_source: prom
-  params:
-  - PrometheusContainerImage
-  services:
-  - OS::TripleO::Services::CephGrafana
-
-- imagename: "{{ceph_alertmanager_namespace}}/{{ceph_alertmanager_image}}:{{ceph_alertmanager_tag}}"
-  image_source: prom
-  params:
-  - AlertManagerContainerImage
-  services:
-  - OS::TripleO::Services::CephGrafana
-
-- imagename: "{{ceph_node_exporter_namespace}}/{{ceph_node_exporter_image}}:{{ceph_node_exporter_tag}}"
-  image_source: prom
-  params:
-  - NodeExporterContainerImage
-  services:
-  - OS::TripleO::Services::CephGrafana
-
-- imagename: "{{ceph_grafana_namespace}}/{{ceph_grafana_image}}:{{ceph_grafana_tag}}"
-  image_source: grafana
-  params:
-  - GrafanaContainerImage
-  services:
-  - OS::TripleO::Services::CephGrafana
+{% if ceph_images | default(true) %}
+{% include 'ceph.j2' %}
+{% endif %}
 
 {% if ceph3_namespace is defined %}
 - imagename: "{{ceph3_namespace}}/{{ceph3_image}}:{{ceph3_tag}}"

releasenotes/notes/split_off_ceph_containers-e1a66fa39076c2cf.yaml

@@ -0,0 +1,30 @@
+---
+features:
+  - |
+    Split off Ceph related container images
+
+    This change releases an update on the default tripleo_containers jinja
+    template, splitting off the Ceph related container images.
+    With this new approach pulling the ceph containers is optional, and can
+    be avoided by setting the `ceph_images` boolean to False.
+    e.g., passing something like the following::
+
+
+      parameter_defaults:
+        ContainerImagePrepare:
+        -   push_destination: true
+            set:
+                name_prefix: openstack-
+                name_suffix: ''
+                namespace: quay.io/tripleomaster
+                neutron_driver: ovn
+                rhel_containers: false
+                tag: current-tripleo
+                ceph_images: false
+        ContainerImagePrepareDebug: true
+        ContainerImageRegistryCredentials: {}
+
+    avoid the ceph containers being pulled in the undercloud.
+    To make this possible, a new jinja template processing approach
+    has been introduced, and a template basedir parameter (required
+    by the jinja loader) has been added to the BaseImageManager.

tripleo_common/actions/container_images.py

@@ -41,8 +41,9 @@ def default_image_params():
     template_file = os.path.join(sys.prefix, 'share', 'tripleo-common',
                                  'container-images',
                                  'tripleo_containers.yaml.j2')
-
-    builder = kolla_builder.KollaImageBuilder([template_file])
+    template_dir = os.path.join(sys.prefix, 'share', 'tripleo-common',
+                                'container-images')
+    builder = kolla_builder.KollaImageBuilder([template_file], template_dir)
     result = builder.container_images_from_template(filter=ffunc)
 
     params = {}

tripleo_common/image/base.py

@@ -33,9 +33,10 @@ class BaseImageManager(object):
         'container_images_template'
     )
 
-    def __init__(self, config_files, images=None):
+    def __init__(self, config_files, template_dir=None, images=None):
         self.config_files = config_files
         self.images = images
+        self.template_dir = template_dir
 
     def _extend_or_set_attribute(self, existing_image, image, attribute_name):
         attribute = image.get(attribute_name)

tripleo_common/image/kolla_builder.py

@@ -57,12 +57,13 @@ def init_prepare_defaults(defaults_file):
         CONTAINER_IMAGES_DEFAULTS['namespace'] += 'centos8'
 
 
-DEFAULT_TEMPLATE_FILE = os.path.join(sys.prefix, 'share', 'tripleo-common',
-                                     'container-images',
+DEFAULT_TEMPLATE_DIR = os.path.join(sys.prefix, 'share', 'tripleo-common',
+                                    'container-images')
+
+DEFAULT_TEMPLATE_FILE = os.path.join(DEFAULT_TEMPLATE_DIR,
                                      'tripleo_containers.yaml.j2')
 
-DEFAULT_PREPARE_FILE = os.path.join(sys.prefix, 'share', 'tripleo-common',
-                                    'container-images',
+DEFAULT_PREPARE_FILE = os.path.join(DEFAULT_TEMPLATE_DIR,
                                     'container_image_prepare_defaults.yaml')
 
 if os.path.isfile(DEFAULT_PREPARE_FILE):
@@ -256,6 +257,7 @@ def container_images_prepare_defaults():
 
 
 def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
+                             template_dir=DEFAULT_TEMPLATE_DIR,
                              excludes=None, includes=None, service_filter=None,
                              pull_source=None, push_destination=None,
                              mapping_args=None, output_env_file=None,
@@ -268,6 +270,7 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
     """Perform container image preparation
 
     :param template_file: path to Jinja2 file containing all image entries
+    :param template_dir: path to Jinja2 files included in the main template
     :param excludes: list of image name substrings to use for exclude filter
     :param includes: list of image name substrings, at least one must match.
                      All excludes are ignored if includes is specified.
@@ -331,7 +334,7 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
                     return None
         return entry
 
-    builder = KollaImageBuilder([template_file])
+    builder = KollaImageBuilder([template_file], template_dir)
     result = builder.container_images_from_template(
         filter=ffunc, **mapping_args)
 
@@ -506,8 +509,14 @@ class KollaImageBuilder(base.BaseImageManager):
         if len(self.config_files) != 1:
             raise ValueError('A single config file must be specified')
         config_file = self.config_files[0]
+        template_dir = self.template_dir
+
         with open(config_file) as cf:
-            template = jinja2.Template(cf.read())
+            if template_dir is not None:
+                template = jinja2.Environment(loader=jinja2.FileSystemLoader(
+                    template_dir)).from_string(cf.read())
+            else:
+                template = jinja2.Template(cf.read())
 
         rendered = template.render(mapping)
         rendered_dict = yaml.safe_load(rendered)
@@ -530,6 +539,7 @@ class KollaImageBuilder(base.BaseImageManager):
 
         if len(self.config_files) == 0:
             self.config_files = [DEFAULT_TEMPLATE_FILE]
+            self.template_dir = DEFAULT_TEMPLATE_DIR
             container_images = self.container_images_from_template()
         else:
             container_images = self.load_config_files(self.CONTAINER_IMAGES) \

tripleo_common/tests/actions/test_container_images.py

@@ -13,9 +13,6 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import os
-import sys
-
 import mock
 from swiftclient import exceptions as swiftexceptions
 import yaml
@@ -61,10 +58,6 @@ class PrepareContainerImageEnvTest(base.TestCase):
             container='overcloud')
         self.assertEqual(final_env, action.run(self.ctx))
 
-        kib.assert_called_once_with(
-            [os.path.join(sys.prefix, 'share', 'tripleo-common',
-                          'container-images', 'tripleo_containers.yaml.j2')]
-        )
         params = {
             'ContainerNovaComputeImage': 't/cb-nova-compute:liberty',
             'ContainerNovaLibvirtConfigImage': 't/cb-nova-compute:liberty',

tripleo_common/tests/image/test_kolla_builder.py

@@ -37,6 +37,10 @@ TEMPLATE_PATH = os.path.join(os.path.dirname(__file__),
 DEFAULTS_PATH = os.path.join(os.path.dirname(__file__),
                              '..', '..', '..', 'container-images',
                              'container_image_prepare_defaults.yaml')
+
+TEMPLATE_DIR_PATH = os.path.join(os.path.dirname(__file__),
+                                 '..', '..', '..', 'container-images')
+
 kb.init_prepare_defaults(DEFAULTS_PATH)
 
 
@@ -417,7 +421,7 @@ class TestKollaImageBuilderTemplate(base.TestCase):
         files_dir = os.path.join(project_dir, 'container-images')
 
         oc_tmpl_file = os.path.join(files_dir, 'tripleo_containers.yaml.j2')
-        tmpl_builder = kb.KollaImageBuilder([oc_tmpl_file])
+        tmpl_builder = kb.KollaImageBuilder([oc_tmpl_file], files_dir)
 
         def ffunc(entry):
             if 'params' in entry:
@@ -433,7 +437,7 @@ class TestKollaImageBuilderTemplate(base.TestCase):
                 rhel_containers=rhel_containers)
 
         oc_yaml_file = os.path.join(files_dir, 'tripleo_containers.yaml')
-        yaml_builder = kb.KollaImageBuilder([oc_yaml_file])
+        yaml_builder = kb.KollaImageBuilder([oc_yaml_file], files_dir)
         container_images = yaml_builder.load_config_files(
             yaml_builder.CONTAINER_IMAGES)
 
@@ -589,7 +593,8 @@ class TestPrepare(base.TestCase):
     def test_prepare_noargs(self, mock_insecure):
         self.assertEqual(
             {},
-            kb.container_images_prepare(template_file=TEMPLATE_PATH)
+            kb.container_images_prepare(template_file=TEMPLATE_PATH,
+                                        template_dir=TEMPLATE_DIR_PATH)
         )
 
     @mock.patch('tripleo_common.image.kolla_builder.'
@@ -608,6 +613,7 @@ class TestPrepare(base.TestCase):
             }},
             kb.container_images_prepare(
                 template_file=TEMPLATE_PATH,
+                template_dir=TEMPLATE_DIR_PATH,
                 output_env_file=constants.CONTAINER_DEFAULTS_ENVIRONMENT,
                 output_images_file='container_images.yaml',
                 service_filter=['OS::TripleO::Services::NovaLibvirt'],
@@ -635,6 +641,7 @@ class TestPrepare(base.TestCase):
             }},
             kb.container_images_prepare(
                 template_file=TEMPLATE_PATH,
+                template_dir=TEMPLATE_DIR_PATH,
                 output_env_file=constants.CONTAINER_DEFAULTS_ENVIRONMENT,
                 output_images_file='container_images.yaml',
                 includes=['libvirt'],
@@ -662,6 +669,7 @@ class TestPrepare(base.TestCase):
             }},
             kb.container_images_prepare(
                 template_file=TEMPLATE_PATH,
+                template_dir=TEMPLATE_DIR_PATH,
                 output_env_file=constants.CONTAINER_DEFAULTS_ENVIRONMENT,
                 output_images_file='container_images.yaml',
                 includes=['libvirt'],
@@ -696,6 +704,7 @@ class TestPrepare(base.TestCase):
             }},
             kb.container_images_prepare(
                 template_file=TEMPLATE_PATH,
+                template_dir=TEMPLATE_DIR_PATH,
                 output_env_file=constants.CONTAINER_DEFAULTS_ENVIRONMENT,
                 output_images_file='container_images.yaml',
                 service_filter=['OS::TripleO::Services::NovaApi'],
@@ -732,6 +741,7 @@ class TestPrepare(base.TestCase):
             }},
             kb.container_images_prepare(
                 template_file=TEMPLATE_PATH,
+                template_dir=TEMPLATE_DIR_PATH,
                 output_env_file=constants.CONTAINER_DEFAULTS_ENVIRONMENT,
                 output_images_file='container_images.yaml',
                 service_filter=['OS::TripleO::Services::NovaApi'],
@@ -758,6 +768,7 @@ class TestPrepare(base.TestCase):
             }},
             kb.container_images_prepare(
                 template_file=TEMPLATE_PATH,
+                template_dir=TEMPLATE_DIR_PATH,
                 output_env_file=constants.CONTAINER_DEFAULTS_ENVIRONMENT,
                 output_images_file='container_images.yaml',
                 service_filter=['OS::TripleO::Services::CephMon'],
@@ -783,6 +794,7 @@ class TestPrepare(base.TestCase):
             }},
             kb.container_images_prepare(
                 template_file=TEMPLATE_PATH,
+                template_dir=TEMPLATE_DIR_PATH,
                 output_env_file=constants.CONTAINER_DEFAULTS_ENVIRONMENT,
                 output_images_file='container_images.yaml',
                 service_filter=[
@@ -816,6 +828,7 @@ class TestPrepare(base.TestCase):
             }},
             kb.container_images_prepare(
                 template_file=TEMPLATE_PATH,
+                template_dir=TEMPLATE_DIR_PATH,
                 output_env_file=constants.CONTAINER_DEFAULTS_ENVIRONMENT,
                 output_images_file='container_images.yaml',
                 service_filter=[
@@ -851,6 +864,7 @@ class TestPrepare(base.TestCase):
             }},
             kb.container_images_prepare(
                 template_file=TEMPLATE_PATH,
+                template_dir=TEMPLATE_DIR_PATH,
                 output_env_file=constants.CONTAINER_DEFAULTS_ENVIRONMENT,
                 output_images_file='container_images.yaml',
                 service_filter=[
@@ -879,6 +893,7 @@ class TestPrepare(base.TestCase):
 
         kb.container_images_prepare(
             template_file=TEMPLATE_PATH,
+            template_dir=TEMPLATE_DIR_PATH,
             output_env_file=constants.CONTAINER_DEFAULTS_ENVIRONMENT,
             output_images_file='container_images.yaml',
             mapping_args={},
@@ -889,6 +904,7 @@ class TestPrepare(base.TestCase):
 
         kb.container_images_prepare(
             template_file=TEMPLATE_PATH,
+            template_dir=TEMPLATE_DIR_PATH,
             output_env_file=constants.CONTAINER_DEFAULTS_ENVIRONMENT,
             output_images_file='container_images.yaml',
             mapping_args={"tag": "master"},