build containers: multiple platfrom support

- rearrange container build job and parenting so we can fit different
parameters for different distribution
- Explicitly upgrade python decorator module for platform that already
have it installed (pip ansible module doesnt' update)
- Parametrize some hard coded values
- ansible_distribution for kolla base does not work with all
distribution

Task: https://tree.taiga.io/project/tripleo-ci-board/task/994
Change-Id: I4c6c6effc9572b07c341d0cfaf452845835aeaeb

playbooks/tripleo-buildcontainers/pre.yaml

@@ -1,12 +1,17 @@
 - hosts: all
   name: TripleO Setup Container Registry and repos mirror
-  roles:
-    - role: tripleo-repos
-      override_repos: "{{ buildcontainers_override_repos | default('') }}"
-      tripleo_repos_repository: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/tripleo-repos'].src_dir }}"
-    - role: bindep
-      bindep_dir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/python-tripleoclient'].src_dir }}"
   tasks:
+    - name: Include tripleo-repos
+      include_role:
+          name: tripleo-repos
+      vars:
+        override_repos: "{{ buildcontainers_override_repos | default('') }}"
+        tripleo_repos_repository: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/tripleo-repos'].src_dir }}"
+    - name: Include bindep role
+      include_role:
+        name: bindep
+      vars:
+        bindep_dir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/python-tripleoclient'].src_dir }}"
     - name: Run build containers pre tasks
       include_role:
         name: build-containers

roles/build-containers/defaults/main.yaml

@@ -1,6 +1,16 @@
 push_registry_port: 8787
 push_registry: "127.0.0.1:{{ push_registry_port }}"
 push_containers: "false"
+kolla_base: "{{ ansible_distribution|lower }}"
+kolla_base_image: "{{ kolla_base }}"
+# DO NOT define a default for kolla_base_tag,
+# this is here to notify of its availability as a parameter
+# Kolla sets the default kolla tag on its own based on dstro
+# and setting a different default breaks other distro dependent
+# default (like the package manager)
+#kolla_base_tag: latest
+kolla_base_pull: true
 use_buildah: false
 buildcontainers_override_repos: ""
 buildcontainers_rpm_setup_config: "http://172.17.0.1/delorean.repo,http://172.17.0.1/delorean-{{ ci_branch }}-testing.repo"
+openstack_repo_name: delorean

roles/build-containers/tasks/main.yaml

@@ -24,11 +24,11 @@
     push_containers: "{{ push_containers | default(false) | bool }}"
     container_config: ""
 
-- name: Get contents of delorean repo baseurl for the version hash
+- name: Get contents of openstack repo baseurl for the version hash
   become: true
   shell: >
     set -o pipefail &&
-    cat /etc/yum.repos.d/delorean.repo |awk -F= '/baseurl/ {print $2}'
+    cat /etc/yum.repos.d/{{ openstack_repo_name }}.repo |awk -F= '/baseurl/ {print $2}'
   register: baseurl
 
 - name: Set version_hash fact
@@ -63,6 +63,7 @@
       - "file://{{ openstack_git_root }}/kolla"
       - "file://{{ openstack_git_root }}/tripleo-common"
       - "file://{{ openstack_git_root }}/python-tripleoclient"
+      - decorator
     virtualenv: "{{ workspace }}/venv_build"
     virtualenv_python: "{{ virtualenv_python[ansible_pkg_mgr] }}"
     virtualenv_site_packages: true

roles/build-containers/tasks/pre.yaml

@@ -68,10 +68,10 @@
         name: httpd
         state: started
 
-    - name: Fetch delorean repos
+    - name: Fetch delorean repos if present
       shell: |
           set -ex
-          cp /etc/yum.repos.d/delorean* /var/www/html/
+          find /etc/yum.repos.d/ -name delorean\* | xargs -I {} cp {} /var/www/html/
 
     - name: Add http iptables rules from containers
       command: iptables -I openstack-INPUT 1 -p tcp --dport 80 -s 172.17.0.0/16 -j ACCEPT

roles/build-containers/templates/build.sh.j2

@@ -2,6 +2,8 @@
 source {{ workspace }}/venv_build/bin/activate
 TRIPLEO_COMMON_PATH="{{ openstack_git_root }}/tripleo-common"
 
+pip install -U decorator
+
 openstack overcloud container image build {{ container_config }}\
     --kolla-config-file {{ workspace }}/kolla-build.conf \
     {% if ci_branch != 'rocky' %}

roles/build-containers/templates/kolla-build.conf.j2

@@ -1,5 +1,9 @@
 [DEFAULT]
-base={{ ansible_distribution|lower }}
+base={{ kolla_base }}
+base_image={{ kolla_base_image }}
+{% if kolla_base_tag is defined %}
+base_tag={{ kolla_base_tag }}
+{% endif %}
 type=binary
 registry={{ push_registry }}
 tag={{ version_hash }}
@@ -7,5 +11,6 @@ template_override={{ openstack_git_root }}/tripleo-common/container-images/tripl
 # we use the repos off the main host as they will already have the correct mirror info in them
 rpm_setup_config="{{ buildcontainers_rpm_setup_config }}"
 push={{ push_containers }}
+pull={{ kolla_base_pull }}
 namespace=tripleo{{ ci_branch }}
 debug=True

zuul.d/base.yaml

@@ -160,13 +160,11 @@
 # Please do not add files: to tripleo-build-containers-base otherwise the periodic job wont run
 # See https://storyboard.openstack.org/#!/story/2005040
 - job:
-    name: tripleo-build-containers-base
+    name: tripleo-build-containers-base-noreposetup
     abstract: true
     description: |
         Base abstract job for standalone TripleO CI build-containers jobs
     parent: tripleo-ci-base-singlenode
-    pre-run:
-      - playbooks/tripleo-buildcontainers/pre.yaml
     run: playbooks/tripleo-buildcontainers/run.yaml
     post-run: playbooks/tripleo-buildcontainers/post.yaml
     timeout: 7200
@@ -179,6 +177,15 @@
       - opendev.org/openstack/tripleo-common
       - opendev.org/openstack/tripleo-repos
 
+- job:
+    name: tripleo-build-containers-base
+    abstract: true
+    description: |
+        Base abstract job for standalone TripleO CI build-containers jobs running repo setup
+    parent: tripleo-build-containers-base-noreposetup
+    pre-run:
+      - playbooks/tripleo-buildcontainers/pre.yaml
+
 - job:
     # Has file filters in order to be used with non-periodic triggers due to:
     # https://storyboard.openstack.org/#!/story/2005040