CI: Template out kolla-build.conf using Ansible role

Change-Id: I79ec11c013fb1d7381fc152ab92e3d607ae822b9
Signed-off-by: Michal Nasiadka <mnasiadka@gmail.com>

tests/run.yml

@@ -78,20 +78,19 @@
   vars:
     kolla_build_config:
       DEFAULT:
-        profile: gate
-        logs_dir: /tmp/logs/build
-        quiet: true
-        # NOTE(yoctozepto): we cannot build and push at the same time on debian
-        # buster see https://github.com/docker/for-linux/issues/711.
-        push: "{{ base_distro != 'debian' }}"
         base: "{{ base_distro }}"
-        registry: "primary:4000"
+        engine: "{{ container_engine }}"
+        logs_dir: /tmp/logs/build
         namespace: lokolla
+        network_mode: host
+        profile: gate
+        push: true
+        quiet: true
+        registry: "primary:4000"
         tag: "{{ build_image_tag }}"
         template_override: /etc/kolla/template_overrides.j2
-        # NOTE(yoctozepto): to avoid issues with IPv6 not enabled in the docker daemon
-        # and since we don't need isolated networks here, use host networking
-        network_mode: host
+      profiles:
+        gate: "{{ (scenario_images_core + (scenario_images_extra | default([]))) | select | join(',') }}"
   tasks:
     - name: Detect whether need build images
       set_fact:
@@ -102,6 +101,15 @@
         - item.project.short_name not in ["ansible-collection-kolla", "kayobe", "kolla-ansible", "tenks"]
       with_items: "{{ zuul['items'] }}"
 
+    - name: Use base_image from mirror on Debian/Ubuntu
+      vars:
+        base_image_append:
+          DEFAULT:
+            base_image: "quay.io/opendevmirror/{{ base_distro }}"
+      set_fact:
+        kolla_build_config: "{{ kolla_build_config | combine(base_image_append, recursive=True) }}"
+      when: ansible_facts.os_family == "Debian"
+
     # NOTE(yoctozepto): required to template template_overrides.j2 for Zuul
     - name: Include kolla Zuul vars if building new images
       include_vars:

tests/setup_gate.sh

@@ -12,87 +12,6 @@ function prepare_images {
         return
     fi
 
-    if [[ $SCENARIO != "bifrost" ]]; then
-        GATE_IMAGES="^cron,^fluentd,^glance,^haproxy,^proxysql,^keepalived,^keystone,^kolla-toolbox,^mariadb,^memcached,^neutron,^nova-,^openvswitch,^rabbitmq,^horizon,^heat,^placement"
-    else
-        GATE_IMAGES="bifrost"
-    fi
-
-    if [[ $SCENARIO == "cephadm" ]]; then
-        GATE_IMAGES+=",^cinder,^redis"
-    fi
-
-    if [[ $SCENARIO == "cells" ]]; then
-        GATE_IMAGES+=",^proxysql"
-    fi
-
-    if [[ $SCENARIO == "zun" ]]; then
-        GATE_IMAGES+=",^zun,^kuryr,^etcd,^cinder,^iscsid"
-        if [[ $BASE_DISTRO != "centos" && $BASE_DISTRO != "rocky" ]]; then
-            GATE_IMAGES+=",^tgtd"
-        fi
-    fi
-
-    if [[ $SCENARIO == "scenario_nfv" ]]; then
-        GATE_IMAGES+=",^aodh,^tacker,^mistral,^redis,^barbican"
-    fi
-    if [[ $SCENARIO == "ironic" ]]; then
-        GATE_IMAGES+=",^dnsmasq,^ironic,^iscsid,^prometheus"
-    fi
-    if [[ $SCENARIO == "magnum" ]]; then
-        GATE_IMAGES+=",^designate,^magnum,^trove"
-    fi
-    if [[ $SCENARIO == "octavia" ]]; then
-        GATE_IMAGES+=",^redis,^octavia"
-    fi
-    if [[ $SCENARIO == "masakari" ]]; then
-        GATE_IMAGES+=",^masakari-,^hacluster-"
-    fi
-
-    if [[ $SCENARIO == "ovn" ]]; then
-        GATE_IMAGES+=",^redis,^octavia,^ovn"
-    fi
-
-    if [[ $SCENARIO == "mariadb" ]]; then
-        GATE_IMAGES="^cron,^fluentd,^haproxy,^proxysql,^keepalived,^kolla-toolbox,^mariadb"
-    fi
-
-    if [[ $SCENARIO == "lets-encrypt" ]]; then
-        GATE_IMAGES+=",^letsencrypt,^haproxy"
-    fi
-
-    if [[ $SCENARIO == "prometheus-opensearch" ]]; then
-        GATE_IMAGES="^cron,^fluentd,^grafana,^haproxy,^proxysql,^keepalived,^kolla-toolbox,^mariadb,^memcached,^opensearch,^prometheus,^rabbitmq"
-    fi
-
-    if [[ $SCENARIO == "ipv6" ]]; then
-        GATE_IMAGES+=",^prometheus"
-    fi
-
-    if [[ $SCENARIO == "venus" ]]; then
-        GATE_IMAGES="^cron,^opensearch,^fluentd,^haproxy,^proxysql,^keepalived,^keystone,^kolla-toolbox,^mariadb,^memcached,^rabbitmq,^venus"
-    fi
-
-    if [[ $SCENARIO == "skyline" || $SCENARIO == "skyline-sso" ]]; then
-        GATE_IMAGES+=",^skyline"
-    fi
-
-    sudo tee -a /etc/kolla/kolla-build.conf <<EOF
-[DEFAULT]
-engine = ${CONTAINER_ENGINE}
-EOF
-
-    if [[ $BASE_DISTRO == "debian" || $BASE_DISTRO == "ubuntu" ]]; then
-        sudo tee -a /etc/kolla/kolla-build.conf <<EOF
-base_image = quay.io/openstack.kolla/${BASE_DISTRO}
-EOF
-    fi
-
-    sudo tee -a /etc/kolla/kolla-build.conf <<EOF
-[profiles]
-gate = ${GATE_IMAGES}
-EOF
-
     sudo mkdir -p /tmp/logs/build
     sudo mkdir -p /opt/kolla_registry
 
@@ -109,14 +28,6 @@ EOF
 
     sudo ~/kolla-venv/bin/kolla-build
 
-    # NOTE(yoctozepto): due to debian buster we push after images are built
-    # see https://github.com/docker/for-linux/issues/711
-    if [[ "debian" == $BASE_DISTRO ]]; then
-        for img in $(sudo ${CONTAINER_ENGINE} image ls --format '{{ .Repository }}:{{ .Tag }}' | grep lokolla/); do
-            sudo $CONTAINER_ENGINE push $img;
-        done
-    fi
-
     deactivate
 }
 

zuul.d/base.yaml

@@ -30,6 +30,24 @@
       neutron_tenant_network_prefix_length: "24"
       previous_release: "2025.1"
       scenario: core
+      scenario_images_core:
+        - ^cron
+        - ^fluentd
+        - ^glance
+        - ^haproxy
+        - ^heat
+        - ^horizon
+        - ^keepalived
+        - ^keystone
+        - ^kolla-toolbox
+        - ^mariadb
+        - ^memcached
+        - ^neutron
+        - ^nova-
+        - ^openvswitch
+        - ^placement
+        - ^proxysql
+        - ^rabbitmq
       tls_enabled: false
       virt_type: qemu
 
@@ -113,6 +131,8 @@
       - ^tests/test-bifrost.sh
     vars:
       scenario: bifrost
+      scenario_images_core:
+        - ^bifrost
 
 - job:
     name: kolla-ansible-ironic-base
@@ -129,6 +149,11 @@
       - openstack/tenks
     vars:
       scenario: ironic
+      scenario_images_extra:
+        - ^dnsmasq
+        - ^ironic
+        - ^iscsid
+        - ^prometheus
 
 - job:
     name: kolla-ansible-zun-base
@@ -142,6 +167,29 @@
       - ^tests/test-dashboard.sh
     vars:
       scenario: zun
+      scenario_images_extra:
+        - ^zun
+        - ^kuryr
+        - ^etcd
+        - ^cinder
+        - ^iscsid
+        - ^tgtd
+
+- job:
+    name: kolla-ansible-swift-base
+    parent: kolla-ansible-scenario-base
+    voting: false
+    files: !inherit
+      - ^ansible/roles/(glance|swift)/
+      - ^tests/setup_disks.sh
+      - ^tests/init-swift.sh
+      - ^tests/test-core-openstack.sh
+      - ^tests/test-dashboard.sh
+      - ^tests/test-swift.sh
+    vars:
+      scenario: swift
+      scenario_images_extra:
+        - ^swift
 
 - job:
     name: kolla-ansible-cephadm-base
@@ -149,6 +197,9 @@
     voting: false
     vars:
       scenario: cephadm
+      scenario_images_extra:
+        - ^cinder
+        - ^redis
 
 - job:
     name: kolla-ansible-magnum-base
@@ -160,6 +211,10 @@
       - ^tests/test-magnum.sh
     vars:
       scenario: magnum
+      scenario_images_extra:
+        - ^designate
+        - ^magnum
+        - ^trove
 
 - job:
     name: kolla-ansible-octavia-base
@@ -171,6 +226,9 @@
       - ^tests/test-octavia.sh
     vars:
       scenario: octavia
+      scenario_images_extra:
+        - ^redis
+        - ^octavia
 
 - job:
     name: kolla-ansible-masakari-base
@@ -183,6 +241,9 @@
       - ^tests/test-dashboard.sh
     vars:
       scenario: masakari
+      scenario_images_extra:
+        - ^masakari
+        - ^hacluster
 
 - job:
     name: kolla-ansible-mariadb-base
@@ -193,6 +254,14 @@
       - ^tests/test-mariadb.sh
     vars:
       scenario: mariadb
+      scenario_images_core:
+        - ^cron
+        - ^fluentd
+        - ^haproxy
+        - ^keepalived
+        - ^kolla-toolbox
+        - ^mariadb
+        - ^proxysql
 
 - job:
     name: kolla-ansible-scenario-nfv-base
@@ -204,6 +273,12 @@
       - ^tests/test-dashboard.sh
     vars:
       scenario: scenario_nfv
+      scenario_images_extra:
+        - ^aodh
+        - ^tacker
+        - ^mistral
+        - ^redis
+        - ^barbican
 
 - job:
     name: kolla-ansible-cells-base
@@ -217,6 +292,8 @@
       - ^tests/test-proxysql.sh
     vars:
       scenario: cells
+      scenario_images_extra:
+        - ^proxysql
 
 - job:
     name: kolla-ansible-ovn-base
@@ -229,6 +306,10 @@
       - ^tests/reconfigure.sh
     vars:
       scenario: ovn
+      scenario_images_extra:
+        - ^redis
+        - ^octavia
+        - ^ovn
 
 - job:
     name: kolla-ansible-prometheus-opensearch-base
@@ -239,6 +320,19 @@
       - ^tests/test-prometheus-opensearch.sh
     vars:
       scenario: prometheus-opensearch
+      scenario_images_core:
+        - ^cron
+        - ^fluentd
+        - ^grafana
+        - ^haproxy
+        - ^keepalived
+        - ^kolla-toolbox
+        - ^mariadb
+        - ^memcached
+        - ^opensearch
+        - ^prometheus
+        - ^proxysql
+        - ^rabbitmq
 
 - job:
     name: kolla-ansible-venus-base
@@ -249,6 +343,18 @@
       - ^tests/test-venus.sh
     vars:
       scenario: venus
+      scenario_images_core:
+        - ^cron
+        - ^opensearch
+        - ^fluentd
+        - ^haproxy
+        - ^keepalived
+        - ^keystone
+        - ^kolla-toolbox
+        - ^mariadb
+        - ^memcached
+        - ^rabbitmq
+        - ^venus
 
 - job:
     name: kolla-ansible-hashi-vault-base
@@ -293,6 +399,9 @@
       - ^tests/deploy.sh
     vars:
       scenario: lets-encrypt
+      scenario_images_extra:
+        - ^letsencrypt
+        - ^haproxy
       tls_enabled: true
       le_enabled: true
 
@@ -305,6 +414,8 @@
       - ^tests/test-skyline.sh
     vars:
       scenario: skyline
+      scenario_images_extra:
+        - ^skyline
 
 - job:
     name: kolla-ansible-skyline-sso-base
@@ -315,6 +426,8 @@
       - ^tests/test-skyline-sso.sh
     vars:
       scenario: skyline-sso
+      scenario_images_extra:
+        - ^skyline
 
 - job:
     name: kolla-ansible-container-engine-migration-base