Split off Ceph related container images

This commit updates the default tripleo_containers jinja template splitting off the Ceph related container images. With this new approach pulling the ceph containers is optional, and can be avoided by setting the 'ceph_images' boolean to False. To make this possible, a new jinja template processing approach has been introduced, and a template basedir parameter (required by the jinja loader) has been added to the BaseImageManager. In particular: - the 'template_dir' parameter represents the location path to the j2 templates that can be included in the main tripleo containers template; a default location (which matches with the default j2 path) has been added, but if nothing is passed the old behavior is maintained; This is not clean cherry-pick as long as the CephIngress daemon is not part of the wallaby cherry-pick. Change-Id: I7e337596b653cf635f07a36606e9f673044402a3 (cherry picked from commit 08a8118f1b)
2022-01-12 13:24:13 +01:00 · 2022-01-12 13:24:13 +01:00 · 4e7965ad25
commit 4e7965ad25
parent d71101a39d
7 changed files with 114 additions and 52 deletions
--- a/container-images/ceph.j2
+++ b/container-images/ceph.j2
@ -0,0 +1,41 @@
+- imagename: "{{ceph_namespace}}/{{ceph_image}}:{{ceph_tag}}"
+  image_source: ceph
+  params:
+  - ContainerCephDaemonImage
+  services:
+  - OS::TripleO::Services::CephClient
+  - OS::TripleO::Services::CephExternal
+  - OS::TripleO::Services::CephMds
+  - OS::TripleO::Services::CephMgr
+  - OS::TripleO::Services::CephMon
+  - OS::TripleO::Services::CephOSD
+  - OS::TripleO::Services::CephRgw
+  - OS::TripleO::Services::CephRbdMirror
+
+- imagename: "{{ceph_prometheus_namespace}}/{{ceph_prometheus_image}}:{{ceph_prometheus_tag}}"
+  image_source: prom
+  params:
+  - PrometheusContainerImage
+  services:
+  - OS::TripleO::Services::CephGrafana
+
+- imagename: "{{ceph_alertmanager_namespace}}/{{ceph_alertmanager_image}}:{{ceph_alertmanager_tag}}"
+  image_source: prom
+  params:
+  - AlertManagerContainerImage
+  services:
+  - OS::TripleO::Services::CephGrafana
+
+- imagename: "{{ceph_node_exporter_namespace}}/{{ceph_node_exporter_image}}:{{ceph_node_exporter_tag}}"
+  image_source: prom
+  params:
+  - NodeExporterContainerImage
+  services:
+  - OS::TripleO::Services::CephGrafana
+
+- imagename: "{{ceph_grafana_namespace}}/{{ceph_grafana_image}}:{{ceph_grafana_tag}}"
+  image_source: grafana
+  params:
+  - GrafanaContainerImage
+  services:
+  - OS::TripleO::Services::CephGrafana
--- a/container-images/tripleo_containers.yaml.j2
+++ b/container-images/tripleo_containers.yaml.j2
@ -727,47 +727,9 @@ container_images_template:
  - OS::TripleO::Services::SwiftRingBuilder
  - OS::TripleO::Services::SwiftStorage

- imagename: "{{ceph_namespace}}/{{ceph_image}}:{{ceph_tag}}"
-  image_source: ceph
-  params:
-  - ContainerCephDaemonImage
-  services:
-  - OS::TripleO::Services::CephClient
-  - OS::TripleO::Services::CephExternal
-  - OS::TripleO::Services::CephMds
-  - OS::TripleO::Services::CephMgr
-  - OS::TripleO::Services::CephMon
-  - OS::TripleO::Services::CephOSD
-  - OS::TripleO::Services::CephRgw
-  - OS::TripleO::Services::CephRbdMirror
-
- imagename: "{{ceph_prometheus_namespace}}/{{ceph_prometheus_image}}:{{ceph_prometheus_tag}}"
-  image_source: prom
-  params:
-  - PrometheusContainerImage
-  services:
-  - OS::TripleO::Services::CephGrafana
-
- imagename: "{{ceph_alertmanager_namespace}}/{{ceph_alertmanager_image}}:{{ceph_alertmanager_tag}}"
-  image_source: prom
-  params:
-  - AlertManagerContainerImage
-  services:
-  - OS::TripleO::Services::CephGrafana
-
- imagename: "{{ceph_node_exporter_namespace}}/{{ceph_node_exporter_image}}:{{ceph_node_exporter_tag}}"
-  image_source: prom
-  params:
-  - NodeExporterContainerImage
-  services:
-  - OS::TripleO::Services::CephGrafana
-
- imagename: "{{ceph_grafana_namespace}}/{{ceph_grafana_image}}:{{ceph_grafana_tag}}"
-  image_source: grafana
-  params:
-  - GrafanaContainerImage
-  services:
-  - OS::TripleO::Services::CephGrafana
+{% if ceph_images | default(true) %}
+{% include 'ceph.j2' %}
+{% endif %}

 - imagename: "{{namespace}}/{{name_prefix}}rsyslog{{name_suffix}}:{{tag}}"
  image_source: tripleo
@ -805,3 +767,4 @@ container_images_template:
  - ContainerTripleoClientImage
  services:
  - OS::TripleO::Services::TripleoClient
+
--- a/releasenotes/notes/split_off_ceph_containers-e1a66fa39076c2cf.yaml
+++ b/releasenotes/notes/split_off_ceph_containers-e1a66fa39076c2cf.yaml
@ -0,0 +1,30 @@
+---
+features:
+  - |
+    Split off Ceph related container images
+
+    This change releases an update on the default tripleo_containers jinja
+    template, splitting off the Ceph related container images.
+    With this new approach pulling the ceph containers is optional, and can
+    be avoided by setting the `ceph_images` boolean to False.
+    e.g., passing something like the following::
+
+
+      parameter_defaults:
+        ContainerImagePrepare:
+        -   push_destination: true
+            set:
+                name_prefix: openstack-
+                name_suffix: ''
+                namespace: quay.io/tripleomaster
+                neutron_driver: ovn
+                rhel_containers: false
+                tag: current-tripleo
+                ceph_images: false
+        ContainerImagePrepareDebug: true
+        ContainerImageRegistryCredentials: {}
+
+    avoid the ceph containers being pulled in the undercloud.
+    To make this possible, a new jinja template processing approach
+    has been introduced, and a template basedir parameter (required
+    by the jinja loader) has been added to the BaseImageManager.
--- a/tripleo_common/image/base.py
+++ b/tripleo_common/image/base.py
@ -33,9 +33,10 @@ class BaseImageManager(object):
        'container_images_template'
    )

-    def __init__(self, config_files, images=None):
+    def __init__(self, config_files, template_dir=None, images=None):
        self.config_files = config_files
        self.images = images
+        self.template_dir = template_dir

    def _extend_or_set_attribute(self, existing_image, image, attribute_name):
        attribute = image.get(attribute_name)
--- a/tripleo_common/image/kolla_builder.py
+++ b/tripleo_common/image/kolla_builder.py
@ -50,12 +50,13 @@ def init_prepare_defaults(defaults_file):
    CONTAINER_IMAGES_DEFAULTS = CONTAINER_IMAGE_PREPARE_PARAM[0]['set']


-DEFAULT_TEMPLATE_FILE = os.path.join(sys.prefix, 'share', 'tripleo-common',
-                                     'container-images',
+DEFAULT_TEMPLATE_DIR = os.path.join(sys.prefix, 'share', 'tripleo-common',
+                                    'container-images')
+
+DEFAULT_TEMPLATE_FILE = os.path.join(DEFAULT_TEMPLATE_DIR,
                                     'tripleo_containers.yaml.j2')

-DEFAULT_PREPARE_FILE = os.path.join(sys.prefix, 'share', 'tripleo-common',
-                                    'container-images',
+DEFAULT_PREPARE_FILE = os.path.join(DEFAULT_TEMPLATE_DIR,
                                    'container_image_prepare_defaults.yaml')

 if os.path.isfile(DEFAULT_PREPARE_FILE):
@ -269,6 +270,7 @@ def container_images_prepare_defaults():


 def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
+                             template_dir=DEFAULT_TEMPLATE_DIR,
                             excludes=None, includes=None, service_filter=None,
                             pull_source=None, push_destination=None,
                             mapping_args=None, output_env_file=None,
@ -281,6 +283,7 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
    """Perform container image preparation

    :param template_file: path to Jinja2 file containing all image entries
+    :param template_dir: path to Jinja2 files included in the main template
    :param excludes: list of image name substrings to use for exclude filter
    :param includes: list of image name substrings, at least one must match.
                     All excludes are ignored if includes is specified.
@ -344,7 +347,7 @@ def container_images_prepare(template_file=DEFAULT_TEMPLATE_FILE,
                    return None
        return entry

-    builder = KollaImageBuilder([template_file])
+    builder = KollaImageBuilder([template_file], template_dir)
    result = builder.container_images_from_template(
        filter=ffunc, **mapping_args)

@ -513,8 +516,14 @@ class KollaImageBuilder(base.BaseImageManager):
        if len(self.config_files) != 1:
            raise ValueError('A single config file must be specified')
        config_file = self.config_files[0]
+        template_dir = self.template_dir
+
        with open(config_file) as cf:
-            template = jinja2.Template(cf.read())
+            if template_dir is not None:
+                template = jinja2.Environment(loader=jinja2.FileSystemLoader(
+                    template_dir)).from_string(cf.read())
+            else:
+                template = jinja2.Template(cf.read())

        rendered = template.render(mapping)
        rendered_dict = yaml.safe_load(rendered)
@ -537,6 +546,7 @@ class KollaImageBuilder(base.BaseImageManager):

        if len(self.config_files) == 0:
            self.config_files = [DEFAULT_TEMPLATE_FILE]
+            self.template_dir = DEFAULT_TEMPLATE_DIR
            container_images = self.container_images_from_template()
        else:
            container_images = self.load_config_files(self.CONTAINER_IMAGES) \
--- a/tripleo_common/tests/image/test_kolla_builder.py
+++ b/tripleo_common/tests/image/test_kolla_builder.py
@ -34,6 +34,10 @@ TEMPLATE_PATH = os.path.join(os.path.dirname(__file__),
 DEFAULTS_PATH = os.path.join(os.path.dirname(__file__),
                             '..', '..', '..', 'container-images',
                             'container_image_prepare_defaults.yaml')
+
+TEMPLATE_DIR_PATH = os.path.join(os.path.dirname(__file__),
+                                 '..', '..', '..', 'container-images')
+
 kb.init_prepare_defaults(DEFAULTS_PATH)
 KB_DEFAULT_TAG = kb.CONTAINER_IMAGES_DEFAULTS['tag']
 KB_DEFAULT_PREFIX = kb.CONTAINER_IMAGES_DEFAULTS['name_prefix']
@ -365,7 +369,7 @@ class TestKollaImageBuilderTemplate(base.TestCase):
        files_dir = os.path.join(project_dir, 'container-images')

        oc_tmpl_file = os.path.join(files_dir, 'tripleo_containers.yaml.j2')
-        tmpl_builder = kb.KollaImageBuilder([oc_tmpl_file])
+        tmpl_builder = kb.KollaImageBuilder([oc_tmpl_file], files_dir)

        def ffunc(entry):
            if 'params' in entry:
@ -379,7 +383,7 @@ class TestKollaImageBuilderTemplate(base.TestCase):
            rhel_containers=rhel_containers)

        oc_yaml_file = os.path.join(files_dir, 'tripleo_containers.yaml')
-        yaml_builder = kb.KollaImageBuilder([oc_yaml_file])
+        yaml_builder = kb.KollaImageBuilder([oc_yaml_file], files_dir)
        container_images = yaml_builder.load_config_files(
            yaml_builder.CONTAINER_IMAGES)

@ -462,7 +466,8 @@ class TestPrepare(base.TestCase):
    def test_prepare_noargs(self, mock_insecure):
        self.assertEqual(
            {},
-            kb.container_images_prepare(template_file=TEMPLATE_PATH)
+            kb.container_images_prepare(template_file=TEMPLATE_PATH,
+                                        template_dir=TEMPLATE_DIR_PATH)
        )

    @mock.patch('tripleo_common.image.kolla_builder.'
@ -481,6 +486,7 @@ class TestPrepare(base.TestCase):
            }},
            kb.container_images_prepare(
                template_file=TEMPLATE_PATH,
+                template_dir=TEMPLATE_DIR_PATH,
                output_env_file=CONTAINER_DEFAULTS_ENVIRONMENT,
                output_images_file='container_images.yaml',
                service_filter=['OS::TripleO::Services::NovaLibvirt'],
@ -508,6 +514,7 @@ class TestPrepare(base.TestCase):
            }},
            kb.container_images_prepare(
                template_file=TEMPLATE_PATH,
+                template_dir=TEMPLATE_DIR_PATH,
                output_env_file=CONTAINER_DEFAULTS_ENVIRONMENT,
                output_images_file='container_images.yaml',
                includes=['libvirt'],
@ -535,6 +542,7 @@ class TestPrepare(base.TestCase):
            }},
            kb.container_images_prepare(
                template_file=TEMPLATE_PATH,
+                template_dir=TEMPLATE_DIR_PATH,
                output_env_file=CONTAINER_DEFAULTS_ENVIRONMENT,
                output_images_file='container_images.yaml',
                includes=['libvirt'],
@ -569,6 +577,7 @@ class TestPrepare(base.TestCase):
            }},
            kb.container_images_prepare(
                template_file=TEMPLATE_PATH,
+                template_dir=TEMPLATE_DIR_PATH,
                output_env_file=CONTAINER_DEFAULTS_ENVIRONMENT,
                output_images_file='container_images.yaml',
                service_filter=['OS::TripleO::Services::NovaApi'],
@ -605,6 +614,7 @@ class TestPrepare(base.TestCase):
            }},
            kb.container_images_prepare(
                template_file=TEMPLATE_PATH,
+                template_dir=TEMPLATE_DIR_PATH,
                output_env_file=CONTAINER_DEFAULTS_ENVIRONMENT,
                output_images_file='container_images.yaml',
                service_filter=['OS::TripleO::Services::NovaApi'],
@ -631,6 +641,7 @@ class TestPrepare(base.TestCase):
            }},
            kb.container_images_prepare(
                template_file=TEMPLATE_PATH,
+                template_dir=TEMPLATE_DIR_PATH,
                output_env_file=CONTAINER_DEFAULTS_ENVIRONMENT,
                output_images_file='container_images.yaml',
                service_filter=['OS::TripleO::Services::CephMon'],
@ -656,6 +667,7 @@ class TestPrepare(base.TestCase):
            }},
            kb.container_images_prepare(
                template_file=TEMPLATE_PATH,
+                template_dir=TEMPLATE_DIR_PATH,
                output_env_file=CONTAINER_DEFAULTS_ENVIRONMENT,
                output_images_file='container_images.yaml',
                service_filter=[
@ -689,6 +701,7 @@ class TestPrepare(base.TestCase):
            }},
            kb.container_images_prepare(
                template_file=TEMPLATE_PATH,
+                template_dir=TEMPLATE_DIR_PATH,
                output_env_file=CONTAINER_DEFAULTS_ENVIRONMENT,
                output_images_file='container_images.yaml',
                service_filter=[
@ -717,6 +730,7 @@ class TestPrepare(base.TestCase):

        kb.container_images_prepare(
            template_file=TEMPLATE_PATH,
+            template_dir=TEMPLATE_DIR_PATH,
            output_env_file=CONTAINER_DEFAULTS_ENVIRONMENT,
            output_images_file='container_images.yaml',
            mapping_args={},
@ -727,6 +741,7 @@ class TestPrepare(base.TestCase):

        kb.container_images_prepare(
            template_file=TEMPLATE_PATH,
+            template_dir=TEMPLATE_DIR_PATH,
            output_env_file=CONTAINER_DEFAULTS_ENVIRONMENT,
            output_images_file='container_images.yaml',
            mapping_args={"tag": "master"},
--- a/tripleo_common/utils/plan.py
+++ b/tripleo_common/utils/plan.py
@ -35,7 +35,9 @@ def default_image_params():
    template_file = os.path.join(sys.prefix, 'share', 'tripleo-common',
                                 'container-images',
                                 'tripleo_containers.yaml.j2')
-    builder = kolla_builder.KollaImageBuilder([template_file])
+    template_dir = os.path.join(sys.prefix, 'share', 'tripleo-common',
+                                'container-images')
+    builder = kolla_builder.KollaImageBuilder([template_file], template_dir)
    result = builder.container_images_from_template(filter=ffunc)

    params = {}