[CI] Support building source images with in-review changes

Following this change it is possible to add 'Depends-On' lines to a
kolla commit message that point to reviews in other projects. For
example, this could be used to test building and deploying container
images containing a change to the nova project that is currently in
review.

It also allows to introduce jobs testing against Kolla in other
projects.

The kolla-build-config Ansible role can also be used by other projects,
such as kolla-ansible, to generate the necessary configuration.

This takes us closer to the Kolla Builder - Next Generation concept [1].

[1] http://lists.openstack.org/pipermail/openstack-discuss/2020-April/014255.html

Change-Id: Ia2efa6b5d7278b75ab8dec23207f92d6bd7c58f0
Co-Authored-By: Radosław Piliszek <radoslaw.piliszek@gmail.com>
(cherry picked from commit 6bfe9b8749)

roles/kolla-build-config/defaults/main.yml

@@ -0,0 +1,134 @@
+---
+# Path to the kolla config file.
+kolla_build_config_path: /etc/kolla/kolla-build.conf
+
+# Dict of configuration to add to kolla-build.conf.
+# Format:
+# kolla_build_config:
+#   "section":
+#     "option name": "option value"
+kolla_build_config: {}
+
+# Dict mapping project names to sections in kolla-build.conf.
+kolla_build_sources:
+  openstack/requirements: openstack-base
+
+  openstack/aodh: aodh-base
+  openstack/barbican: barbican-base
+  openstack/bifrost: bifrost-base
+  openstack/blazar: blazar-base
+  openstack/blazar-nova: nova-base-plugin-blazar
+  openstack/ceilometer: ceilometer-base
+  openstack/cinder: cinder-base
+  openstack/cloudkitty: cloudkitty-base
+  openstack/designate: designate-base
+  openstack/ec2-api: ec2-api
+  openstack/freezer: freezer-base
+  openstack/freezer-api: freezer-api
+  openstack/glance: glance-base
+  openstack/gnocchi: gnocchi-base
+  openstack/heat: heat-base
+  openstack/ironic: ironic-base
+  openstack/ironic-inspector: ironic-inspector
+  openstack/karbor: karbor-base
+  openstack/keystone: keystone-base
+  openstack/kuryr: kuryr-base
+  openstack/kuryr-libnetwork: kuryr-libnetwork
+  openstack/magnum: magnum-base
+  openstack/manila: manila-base
+  openstack/mistral: mistral-base
+  openstack/monasca-agent: monasca-agent
+  openstack/monasca-api: monasca-api
+  openstack/monasca-notification: monasca-notification
+  openstack/monasca-persister: monasca-persister
+  openstack/monasca-statsd: monasca-statsd
+  openstack/monasca-common: monasca-thresh-additions-monasca-common
+  openstack/murano: murano-base
+  openstack/neutron: neutron-base
+  openstack/networking-ansible: neutron-base-plugin-networking-ansible
+  openstack/networking-baremetal: neutron-base-plugin-networking-baremetal
+  openstack/neutron-dynamic-routing:
+    - neutron-bgp-dragent
+    - neutron-server-plugin-neutron-dynamic-routing
+  openstack/networking-infoblox: neutron-server-plugin-networking-infoblox
+  openstack/networking-generic-switch: neutron-base-plugin-networking-generic-switch
+  openstack/networking-sfc:
+    - neutron-base-plugin-networking-sfc
+    - tacker-base-plugin-networking-sfc
+  openstack/vmware-nsx: neutron-base-plugin-vmware-nsx
+  openstack/vmware-nsxlib: neutron-server-plugin-vmware-nsxlib
+  openstack/neutron-vpnaas:
+    - neutron-vpnaas-agent
+    - neutron-base-plugin-vpnaas-agent
+  openstack/nova: nova-base
+  openstack/nova-mksproxy: nova-base-plugin-mksproxy
+  openstack/novajoin: novajoin-base
+  openstack/octavia: octavia-base
+  openstack/panko:
+    - ceilometer-base-plugin-panko
+    - panko-base
+  openstack/rally: rally
+  openstack/sahara: sahara-base
+  openstack/searchlight: searchlight-base
+  openstack/senlin: senlin-base
+  openstack/solum: solum-base
+  swift-base: openstack/swift
+  openstack/tacker:
+    - tacker-base
+    - mistral-base-plugin-tacker
+  openstack/trove: trove-base
+  openstack/vitrage: vitrage-base
+  openstack/vmtp: vmtp
+  openstack/watcher: watcher-base
+  openstack/zaqar: zaqar
+  openstack/zun: zun-base
+
+  # Horizon and its plugins
+  openstack/horizon: horizon
+  openstack/blazar-dashboard: horizon-plugin-blazar-dashboard
+  openstack/cloudkitty-dashboard: horizon-plugin-cloudkitty-dashboard
+  openstack/designate-dashboard: horizon-plugin-designate-dashboard
+  openstack/freezer-web-ui: horizon-plugin-freezer-web-ui
+  openstack/heat-dashboard: horizon-plugin-heat-dashboard
+  openstack/ironic-ui: horizon-plugin-ironic-ui
+  openstack/karbor-dashboard: horizon-plugin-karbor-dashboard
+  openstack/magnum-ui: horizon-plugin-magnum-ui
+  openstack/manila-ui: horizon-plugin-manila-ui
+  openstack/mistral-dashboard: horizon-plugin-mistral-dashboard
+  openstack/monasca-ui: horizon-plugin-monasca-ui
+  openstack/murano-dashboard: horizon-plugin-murano-dashboard
+  openstack/neutron-vpnaas-dashboard: horizon-plugin-neutron-vpnaas-dashboard
+  openstack/octavia-dashboard: horizon-plugin-octavia-dashboard
+  openstack/sahara-dashboard: horizon-plugin-sahara-dashboard
+  openstack/searchlight-ui: horizon-plugin-searchlight-ui
+  openstack/senlin-dashboard: horizon-plugin-senlin-dashboard
+  openstack/solum-dashboard: horizon-plugin-solum-dashboard
+  openstack/tacker-horizon: horizon-plugin-tacker-dashboard
+  openstack/trove-dashboard: horizon-plugin-trove-dashboard
+  openstack/vitrage-dashboard: horizon-plugin-vitrage-dashboard
+  openstack/watcher-dashboard: horizon-plugin-watcher-dashboard
+  openstack/zaqar-ui: horizon-plugin-zaqar-ui
+  openstack/zun-ui: horizon-plugin-zun-ui
+
+  # Tempest and its plugins
+  openstack/tempest: tempest
+  openstack/patrole: tempest-plugin-patrole
+  openstack/python-tempestconf: tempest-plugin-tempest-conf
+  openstack/barbican-tempest-plugin: tempest-plugin-barbican
+  openstack/blazar-tempest-plugin: tempest-plugin-blazar
+  openstack/cinder-tempest-plugin: tempest-plugin-cinder
+  openstack/ec2api-tempest-plugin: tempest-plugin-ec2api
+  openstack/heat-tempest-plugin: tempest-plugin-heat
+  openstack/ironic-tempest-plugin: tempest-plugin-ironic
+  openstack/keystone-tempest-plugin: tempest-plugin-keystone
+  openstack/magnum-tempest-plugin: tempest-plugin-magnum
+  openstack/manila-tempest-plugin: tempest-plugin-manila
+  openstack/mistral-tempest-plugin: tempest-plugin-mistral
+  openstack/monasca-tempest-plugin: tempest-plugin-monasca
+  openstack/murano-tempest-plugin: tempest-plugin-murano
+  openstack/neutron-tempest-plugin: tempest-plugin-neutron
+  openstack/telemetry-tempest-plugin: tempest-plugin-telemetry
+  openstack/trove-tempest-plugin: tempest-plugin-trove
+  openstack/vitrage-tempest-plugin: tempest-plugin-vitrage
+  openstack/watcher-tempest-plugin: tempest-plugin-watcher
+  openstack/zaqar-tempest-plugin: tempest-plugin-zaqar

roles/kolla-build-config/tasks/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Ensure kolla-build.conf exists
+  template:
+    src: kolla-build.conf.j2
+    dest: "{{ kolla_build_config_path }}"

roles/kolla-build-config/templates/kolla-build.conf.j2

@@ -0,0 +1,27 @@
+{# Generate requested sections and options #}
+
+{% for section_name, section in kolla_build_config.items() %}
+[{{ section_name }}]
+{% for option_name, option in section.items() %}
+{{ option_name }} = {{ option }}
+{% endfor %}
+{% endfor %}
+
+{# Generate a section for every build source for which there is a Zuul change. #}
+
+{% for zuul_item in zuul['items'] %}
+{% if zuul_item.project.name in kolla_build_sources %}
+{% set source_names = kolla_build_sources[zuul_item.project.name] %}
+{% if source_names is string %}
+[{{ source_names }}]
+type = local
+location = {{ ansible_env.PWD ~ '/' ~ zuul_item.project.src_dir }}
+{% else %}
+{% for source_name in source_names %}
+[{{ source_name }}]
+type = local
+location = {{ ansible_env.PWD ~ '/' ~ zuul_item.project.src_dir }}
+{% endfor %}
+{% endif %}
+{% endif %}
+{% endfor %}

tests/playbooks/run.yml

@@ -2,6 +2,18 @@
 - hosts: all
   vars_files:
     - ../vars/zuul.yml
+  vars:
+    tag_suffix: "{{ '-aarch64' if ansible_architecture == 'aarch64' else '' }}"
+    kolla_build_config:
+      DEFAULT:
+        debug: true
+        logs_dir: "{{ kolla_build_logs_dir }}"
+        base: "{{ base_distro }}"
+        install_type: "{{ install_type }}"
+        template_override: /etc/kolla/template_overrides.j2
+        # NOTE(yoctozepto): to avoid issues with IPv6 not enabled in the docker daemon
+        # and since we don't need isolated networks here, use host networking
+        network_mode: host
   tasks:
     - name: Ensure /etc/kolla exists
       file:
@@ -10,10 +22,19 @@
         mode: 0777
       become: true
 
-    - name: Template kolla-build.conf
-      template:
-        src: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}/tests/templates/kolla-build.conf.j2"
-        dest: /etc/kolla/kolla-build.conf
+    - name: Add publisher config
+      vars:
+        kolla_publisher_config:
+          DEFAULT:
+            namespace: "{{ kolla_namespace }}"
+            tag: "{{ (zuul.tag if zuul.pipeline == 'release' else zuul.branch | basename) ~ tag_suffix }}"
+      set_fact:
+        kolla_build_config: "{{ kolla_build_config | combine(kolla_publisher_config, recursive=True) }}"
+      when:
+        - publisher
+
+    - import_role:
+        name: kolla-build-config
 
     - name: Template template_overrides.j2
       template:

tests/templates/kolla-build.conf.j2

@@ -1,18 +0,0 @@
-[DEFAULT]
-debug = true
-{% if publisher %}
-namespace = {{ kolla_namespace }}
-{% set tag_suffix = '-aarch64' if ansible_architecture == 'aarch64' else '' %}
-tag = {{ (zuul.tag if zuul.pipeline == "release" else zuul.branch | basename) ~ tag_suffix }}
-{% endif %}
-base = {{ base_distro }}
-{% if base_tag is defined %}
-base_tag = {{ base_tag }}
-{% endif %}
-install_type = {{ install_type }}
-push = false
-logs_dir = {{ kolla_build_logs_dir }}
-template_override = /etc/kolla/template_overrides.j2
-# NOTE(yoctozepto): to avoid issues with IPv6 not enabled in the docker daemon
-# and since we don't need isolated networks here, use host networking
-network_mode = host