Fetch non tripleo containers in provider job and consume it

When provider_job is set to true, it will pull the container
images from quay.io and will push it to local
registry using non_tripleo_containers.yml.

When consumer_job is set to true in child job, then
custom_ceph_parameters will be setted to true and ceph containers
related namespace will be changed to local registry
to pull the ceph containers so that container built and pushed
to local registry in provider job, will be consumed here.

It also adds standalone job which depends on content provider
job and updates the ceph tag.

Setting +standalone_container_ceph_updates to false as with in
tasks it is setted default to false.

Moved local registry host and port to extra-commons role
to easily share between roles.

It also fix the non_tripleo_containers playbook to exclude kolla
containers for train release.

Note to self:
tripleo-standalone-scenarios-pipeline will be added later
once standalone-upgrade job is ported to centent provider.

Change-Id: Ie3f7bafe3b6ee8a7879ebb3d1d5b47b20181b452
Signed-off-by: Chandan Kumar (raukadah) <chkumar@redhat.com>
Co-authored-by: Alex Schultz <aschultz@redhat.com>

roles/container-build/defaults/main.yaml

@@ -14,10 +14,11 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-local_registry_ip: 127.0.0.1
-local_registry_port: 5001
+content_provider_registry_ip: 127.0.0.1
+content_provider_registry_port: 5001
 container_registry_namespace: tripleomaster
 container_tag: current-tripleo
+container_file: /usr/share/openstack-tripleo-common/container-images/tripleo_containers.yaml
 # Notes(chandankumar): openstack- prefix is used from victoria release onwards
 docker_prep_prefix: >-
   {% if (job.build_container_images|default(false)|bool and

roles/container-build/tasks/main.yaml

@@ -14,22 +14,22 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-- name: Deploy the local registry
+- name: Deploy the content provider registry
   become: true
   podman_container:
     name: docker_registry
     image: quay.io/tripleoci/registry:2
     publish:
-      - "{{ local_registry_port }}:{{ local_registry_port }}"
+      - "{{ content_provider_registry_port }}:{{ content_provider_registry_port }}"
     env:
-      REGISTRY_HTTP_ADDR: "0.0.0.0:{{ local_registry_port }}"
+      REGISTRY_HTTP_ADDR: "0.0.0.0:{{ content_provider_registry_port }}"
   tags:
     - skip_ansible_lint
 
-- name: Open port for local registry
+- name: Open port for content provider registry
   become: true
   command: >-
-    iptables -I INPUT -p tcp --dport {{ local_registry_port }} -j ACCEPT
+    iptables -I INPUT -p tcp --dport {{ content_provider_registry_port }} -j ACCEPT
   changed_when: true
 
 - name: Set container_image_build_volumes fact
@@ -60,8 +60,12 @@
     tripleo_container_image_build_namespace: "{{ container_registry_namespace }}"
     tripleo_container_image_build_tag: "{{ container_tag }}"
     tripleo_container_image_build_push: true
-    tripleo_container_image_build_registry: "{{ local_registry_ip }}:{{ local_registry_port }}"
+    tripleo_container_image_build_registry: "{{ content_provider_registry_ip }}:{{ content_provider_registry_port }}"
     tripleo_container_image_build_rhel_modules: "{{ rhel_modules|default('') }}"
     tripleo_container_image_build_volumes: "{{ container_image_build_volumes }}"
     tripleo_container_image_build_work_dir: "{{ working_dir }}/container-builds"
     tripleo_container_image_build_prefix: "{{ docker_prep_prefix }}"
+
+- name: Pull and push non *tripleo* container images to local registry
+  import_tasks: non_tripleo_containers.yml
+  when: job.provider_job|default(false)|bool

roles/container-build/tasks/non_tripleo_containers.yml

@@ -0,0 +1,55 @@
+---
+- name: Fetch tripleo containers
+  command: cat "{{ container_file }}"
+  register: all_containers
+  changed_when: false
+
+- name: Store tripleo containers
+  set_fact:
+    containers: "{{ all_containers.stdout | from_yaml }}"
+
+- name: Filter out non-tripleo containers
+  set_fact:
+    non_tripleo_containers: "{{ containers.container_images | json_query('[?image_source != `tripleo`]') }}"
+
+- name: Filter out non-tripleo containers excluding kolla image_source
+  set_fact:
+    non_tripleo_containers: "{{ non_tripleo_containers.container_images | json_query('[?image_source != `kolla`]') }}"
+  when: release in ["train"]
+
+- name: Store non-tripleo containers
+  set_fact:
+    non_tripleo_containers: "{{ non_tripleo_containers | map(attribute='imagename') | list }}"
+
+- name: Print non-tripleo containers
+  debug:
+    msg: "{{ non_tripleo_containers }}"
+
+- name: Pull non-tripleo containers to the content provider registry
+  become: true
+  block:
+
+    - name: Make sure buildah is installed
+      package:
+        name: "buildah"
+        state: present
+
+    # NOTE(mwhahaha): use buildah because --format can only be specified with buildah push.
+    # Additionally we could switch to oci, but v2v2 is what we convert on the undercloud.
+    - name: Pull non-tripleo containers (ceph, alertmanager, prometheus) to the content provider registry
+      shell: >
+        buildah images;
+        buildah pull {{ item }};
+        buildah push --format=v2s2 --tls-verify=False --log-level debug {{ item }} \
+          docker://{{ content_provider_registry_ip }}:{{ content_provider_registry_port }}/{{ docker_registry_namespace }}/{{ item.split('/')[-1] }};
+      with_items: "{{ non_tripleo_containers }}"
+      register: non_tripleo
+      changed_when: false
+      tags:
+        - skip_ansible_lint
+
+- name: Dump non-tripleo containers output to file
+  copy:
+    content: "{{ non_tripleo }}"
+    dest: "{{ ansible_user_dir }}/non_tripleo_containers.log"
+  when: non_tripleo is defined