Always build requirements image in the pipeline

Without this patch, the loci builder automatically consumes a
previously published requirements image when the project is not
"requirements". This also means the first step in checks/gates
for non requirement images would be to pull a requirement.

Currently there is neither a "pipeline" nor consumption of
artifacts between jobs. Which means the "requirements" build
and the other image build have to be considered as independant.

However, by default, the "non-requirements" image built is
consuming previously published "requirements" image, but not
immediately previous built requirements image.

This is a problem, as it leaves us open to a race condition
where a new build of "requirements" and "non-requirements"
can succeed in a commit, but would cause all the next
"non-requirements" builds to fail, as it is possible the
newly published "requirements" to be incompatible with
the next "non-requirements", as they are not tested together.

This should fix the problem by ensuring that "requirements"
are always built and consumed in the "non-requirement"
building process, instead of re-using the previously built
requirements. For this, project needs to be overriden to
build the requirement image, and therefore a new profile
'requirements' was added to the buildargs of the distros.
Requirements repo also needs to be cloned in the gating
environment, and therefore was added to all the required
projects.

This will also allow new branches to be created, as there
would be no need of a previous build and publish
of requirements in the gating.

Change-Id: I093e4dfc0eef03c47d2d029011fdb5429242ae79

.zuul.d/cinder.yaml

@@ -16,6 +16,7 @@
       project: cinder
     required-projects:
       - openstack/loci
+      - openstack/requirements
       - openstack/cinder
 
 - job:

.zuul.d/glance.yaml

@@ -16,6 +16,7 @@
       project: glance
     required-projects:
       - openstack/loci
+      - openstack/requirements
       - openstack/glance
 
 - job:

.zuul.d/heat.yaml

@@ -16,6 +16,7 @@
       project: heat
     required-projects:
       - openstack/loci
+      - openstack/requirements
       - openstack/heat
 
 - job:

.zuul.d/horizon.yaml

@@ -16,6 +16,7 @@
       project: horizon
     required-projects:
       - openstack/loci
+      - openstack/requirements
       - openstack/horizon
 
 - job:

.zuul.d/ironic.yaml

@@ -16,6 +16,7 @@
       project: ironic
     required-projects:
       - openstack/loci
+      - openstack/requirements
       - openstack/ironic
 
 - job:

.zuul.d/keystone.yaml

@@ -16,6 +16,7 @@
       project: keystone
     required-projects:
       - openstack/loci
+      - openstack/requirements
       - openstack/keystone
 
 - job:

.zuul.d/neutron.yaml

@@ -16,6 +16,7 @@
       project: neutron
     required-projects:
       - openstack/loci
+      - openstack/requirements
       - openstack/neutron
 
 - job:

.zuul.d/nova.yaml

@@ -16,6 +16,7 @@
       project: nova
     required-projects:
       - openstack/loci
+      - openstack/requirements
       - openstack/nova
 
 - job:

.zuul.d/octavia.yaml

@@ -16,6 +16,7 @@
       project: octavia
     required-projects:
       - openstack/loci
+      - openstack/requirements
       - openstack/octavia
 
 - job:

playbooks/loci-builder.yaml

@@ -9,7 +9,13 @@
           - "pydep.txt"
       environment:
           LC_ALL: C
-
+    # NOTE(evrardjp): While reuse_requirements is very nice and optimises
+    # checks and gating, there is a race condition here, because
+    # we are consuming prebuild wheels (see vars.yaml) by default:
+    # The jobs in zuul can be building a new "requirements" image, working
+    # And the new "nova" image would still build on previous "requirements"
+    # image that was last published. This could cause an issue. Instead
+    # in gating we should build directly what will be consumed.
     # NOTE(SamYaple): This process is so we can take advantage of the infra
     # DockerHub mirroring as configured through the Docker daemon. We do this
     # instead of calling fetch_wheels initially. All-in-all this saves
@@ -25,6 +31,7 @@
           async: 1000
           poll: 0
           register: pull
+
         - async_status:
             jid: "{{ item.ansible_job_id }}"
           with_items: "{{ pull.results }}"
@@ -34,11 +41,14 @@
             - pull_result.finished
           retries: 1000
           delay: 5
-      when: project != 'requirements'
+      when:
+       - reuse_requirements | bool
+       - project != 'requirements'
 
     - name: Build base images
       block:
-        - docker_image:
+        - name: "Build base image for {{ item.name }} asynchronously"
+          docker_image:
             path: "{{ loci_src_dir }}/dockerfiles/{{ item.name }}"
             name: base
             tag: "{{ item.name }}"
@@ -47,6 +57,7 @@
           async: 1000
           poll: 0
           register: base
+
         - async_status:
             jid: "{{ item.ansible_job_id }}"
           with_items: "{{ base.results }}"
@@ -57,18 +68,49 @@
           retries: 1000
           delay: 5
 
-    - name: Build project images
+    - name: Build requirements image
       block:
-        - docker_image:
+        - name: "Build requirements image for {{ item.name }}"
+          docker_image:
             path: "{{ loci_src_dir }}"
-            name: loci/{{ project }}
+            name: loci/requirements
             tag: "{{ branch }}-{{ item.name }}"
+            repository: 172.17.0.1:5000/loci/requirements
+            push: yes
             pull: False
-            buildargs: "{{ item.buildargs.project }}"
+            buildargs: "{{ item.buildargs.requirements }}"
           with_items: "{{ distros }}"
           async: 1000
           poll: 0
           register: build
+
+        - async_status:
+            jid: "{{ item.ansible_job_id }}"
+          with_items: "{{ build.results }}"
+          register: build_result
+          until:
+            - build_result.finished is defined
+            - build_result.finished
+          retries: 1000
+          delay: 5
+      when:
+       - (not reuse_requirements) | bool
+       - project != 'requirements'
+
+    - name: Build project images
+      block:
+        - name: "Build {{ project }} image for {{ item.name }}"
+          docker_image:
+            path: "{{ loci_src_dir }}"
+            name: loci/{{ project }}
+            tag: "{{ branch }}-{{ item.name }}"
+            pull: False
+            buildargs: "{{ item.buildargs.project }}"
+          with_items: "{{ distros }}"
+          async: 1000
+          poll: 0
+          register: build
+
         - async_status:
             jid: "{{ item.ansible_job_id }}"
           with_items: "{{ build.results }}"

playbooks/vars.yaml

@@ -7,6 +7,8 @@ docker_daemon:
   insecure-registries:
     - 172.17.0.1:5000
 
+reuse_requirements: False
+
 # NOTE(SamYaple): When running in the loci repo, the project is "loci", but
 # when running as a post job for cinder, the project is "cinder". We statically
 # declare the path rather than using zuul variables so thats not an issue
@@ -26,6 +28,10 @@ distros:
         PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }}
         WHEELS: 172.17.0.1:5000/loci/requirements:{{ branch }}-centos
         FROM: base:centos
+      requirements:
+        PROJECT: requirements
+        PROJECT_REPO: http://172.17.0.1/git/openstack/requirements
+        FROM: base:centos
   - name: ubuntu
     image: ubuntu:xenial
     buildargs:
@@ -41,6 +47,10 @@ distros:
         PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }}
         WHEELS: 172.17.0.1:5000/loci/requirements:{{ branch }}-ubuntu
         FROM: base:ubuntu
+      requirements:
+        PROJECT: requirements
+        PROJECT_REPO: http://172.17.0.1/git/openstack/requirements
+        FROM: base:ubuntu
 #  - name: debian
 #    image: debian:stretch
 #    buildargs:
@@ -57,3 +67,7 @@ distros:
 #        PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }}
 #        WHEELS: 172.17.0.1:5000/loci/requirements:master-debian
 #        FROM: base:debian
+#      requirements:
+#        PROJECT: requirements
+#        PROJECT_REPO: http://172.17.0.1/git/openstack/requirements
+#        FROM: base:debian