k8s: Allow use of a default image version besides latest

This change allows us to specify a default image version that is
something other than latest via the ZUUL_IMAGE_VERSION environment
variable.  We also include modifications to the Helm chart that will
link that image version to the application version as specified by the
chart so that the images match up as expected.

Allowing the image version to float at latest can be dangerous for
production configurations because pods being rescheduled to a
different node may result in different (potentially incompatible) pods
being pulled with a different image version than what was originally
deployed for the application.

Change-Id: I80f96614cff5d15662742ed1fa0ad1a32d62e8bd

deploy/operator.yaml

@@ -17,3 +17,6 @@ spec:
         - name: operator
           image: "docker.io/zuul/zuul-operator"
           imagePullPolicy: "IfNotPresent"
+          env:
+            - name: ZUUL_IMAGE_VERSION
+              value: latest

helm/zuul-operator/templates/090-Deployment.yaml

@@ -52,6 +52,8 @@ spec:
             - name: ZUUL_NAMESPACE_WATCH_LIST
               value: /etc/config/namespaces
             {{- end }}
+            - name: ZUUL_IMAGE_VERSION
+              value: "{{ .Values.zuulImageVersion | default .Chart.AppVersion }}"
           {{- if .Values.watchNamespaces }}
           volumeMounts:
             - name: config-volume

helm/zuul-operator/values.yaml

@@ -77,3 +77,5 @@ pxc-operator:
   operatorManaged: false
 
 watchNamespaces: []
+zuulImageVersion: ""
+

playbooks/zuul-operator-functional/install_helm.yaml

@@ -12,7 +12,7 @@
   tasks:
     - name: Install Operator
       when: install_operator
-      command: helm install --set image.tag=latest,cert-manager.enabled=true,pxc-operator.enabled=true test helm/zuul-operator
+      command: helm install --set zuulImageVersion=latest,image.tag=latest,cert-manager.enabled=true,pxc-operator.enabled=true test helm/zuul-operator
       args:
         chdir: "{{ zuul_work_dir }}"
 

zuul_operator/operator.py

@@ -60,6 +60,7 @@ def memoize_secrets(memo, logger):
 def startup(memo, logger, **kwargs):
     logger.info("Loading operator config")
     memo.operator_config = {
+        'image_version': os.environ.get('ZUUL_IMAGE_VERSION', 'latest'),
         'install_cert_manager': distutils.util.strtobool(
             os.environ.get('ZUUL_INSTALL_CERT_MANAGER', '1')),
         'install_pxc': distutils.util.strtobool(
@@ -114,7 +115,8 @@ def update_secret(name, namespace, logger, memo, **kwargs):
             logger.info(f"Affects zuul {zuul_namespace}/{zuul_name}")
             zuul_obj = objects.ZuulObject.objects(api).filter(
                 namespace=zuul_namespace).get(name=zuul_name)
-            zuul = Zuul(namespace, zuul_name, logger, zuul_obj.obj['spec'])
+            zuul = Zuul(namespace, zuul_name, logger, zuul_obj.obj['spec'],
+                        memo.operator_config['image_version'])
             if resource.attr == 'spec.scheduler.config.secretName':
                 zuul.smart_reconfigure()
             if resource.attr == 'spec.launcher.config.secretName':
@@ -131,7 +133,8 @@ def when_update_zuul(name, namespace, memo, logger, **_):
 def create_fn(spec, name, namespace, logger, memo, **kwargs):
     logger.info(f"Create zuul {namespace}/{name}")
 
-    zuul = Zuul(namespace, name, logger, spec)
+    zuul = Zuul(
+        namespace, name, logger, spec, memo.operator_config['image_version'])
     # Get DB installation started first; it's slow and has no
     # dependencies.
     zuul.install_db(install_pxc=memo.operator_config['install_pxc'])
@@ -161,7 +164,8 @@ def update_fn(name, namespace, logger, old, new, memo, **kwargs):
     old = old['spec']
     new = new['spec']
 
-    zuul = Zuul(namespace, name, logger, new)
+    zuul = Zuul(
+        namespace, name, logger, new, memo.operator_config['image_version'])
     conf_changed = False
     spec_changed = False
     if new.get('database') != old.get('database'):

zuul_operator/zuul.py

@@ -29,7 +29,7 @@ from . import zookeeper
 
 
 class Zuul:
-    def __init__(self, namespace, name, logger, spec):
+    def __init__(self, namespace, name, logger, spec, zuul_image_version):
         self.api = pykube.HTTPClient(pykube.KubeConfig.from_env())
         self.namespace = namespace
         self.name = name
@@ -88,10 +88,10 @@ class Zuul:
 
         self.spec.setdefault('imagePrefix', 'docker.io/zuul')
         self.spec.setdefault('imagePullSecrets', [])
-        self.spec.setdefault('zuulImageVersion', 'latest')
-        self.spec.setdefault('zuulPreviewImageVersion', 'latest')
-        self.spec.setdefault('zuulRegistryImageVersion', 'latest')
-        self.spec.setdefault('nodepoolImageVersion', 'latest')
+        self.spec.setdefault('zuulImageVersion', zuul_image_version)
+        self.spec.setdefault('zuulPreviewImageVersion', zuul_image_version)
+        self.spec.setdefault('zuulRegistryImageVersion', zuul_image_version)
+        self.spec.setdefault('nodepoolImageVersion', zuul_image_version)
 
         default_env = {
             'KUBECONFIG': '/etc/kubernetes/kube.config'