[WIP] Refactor Openstack image build

Change-Id: I7c63ee98125bf0988b3926e40dd916087cbb9476

zuul.d/base.yaml

@@ -56,9 +56,9 @@
         - openstack-helm-images-compute-kit-2023-1-ubuntu_jammy
         - openstack-helm-images-cinder-2023-1-ubuntu_jammy
         - openstack-helm-images-horizon-2023-1-ubuntu_jammy
-    templates:
+    # templates:
-      - publish-openstack-docs-pti
+    #   - publish-openstack-docs-pti
-      - release-notes-jobs-python3
+    #   - release-notes-jobs-python3
 
 # This takes the same structure as other infra jobs.
 # This dict keyname matter. See also job definition for more explanations.
@@ -92,7 +92,7 @@
     name: openstack-helm-images-buildset-registry
     parent: opendev-buildset-registry
     timeout: 10800
-    voting: false
+    # voting: false
 
 # In docker_images:
 # `context` is the folder in osh-images

zuul.d/openstack-2023.2.yaml

@@ -11,29 +11,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- project:
+# - project:
-    check:
+#     check:
-      jobs:
+#       jobs:
-        - openstack-helm-images-build-openstack-2023-2-ubuntu_jammy
+#         - openstack-helm-images-build-openstack-2023-2-ubuntu_jammy
-        - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+#         - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
-        - openstack-helm-images-cinder-2023-2-ubuntu_jammy
+#         - openstack-helm-images-cinder-2023-2-ubuntu_jammy
-    gate:
+#     gate:
-      jobs:
+#       jobs:
-        - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
+#         - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
-        - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+#         - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
-        - openstack-helm-images-cinder-2023-2-ubuntu_jammy
+#         - openstack-helm-images-cinder-2023-2-ubuntu_jammy
-    promote:
+#     promote:
-      jobs:
+#       jobs:
-        - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy
+#         - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy
-    periodic-weekly:
+#     periodic-weekly:
-      jobs:
+#       jobs:
-        - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
+#         - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
-        - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy:
+#         - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy:
-            dependencies:
+#             dependencies:
-              - name: openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+#               - name: openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
-              - name: openstack-helm-images-cinder-2023-2-ubuntu_jammy
+#               - name: openstack-helm-images-cinder-2023-2-ubuntu_jammy
-        - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+#         - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
-        - openstack-helm-images-cinder-2023-2-ubuntu_jammy
+#         - openstack-helm-images-cinder-2023-2-ubuntu_jammy
 
 - job:
     name: openstack-helm-images-build-openstack-2023-2-ubuntu_jammy

zuul.d/openstack-2024.1.yaml

@@ -16,6 +16,7 @@
     check:
       jobs:
         - openstack-helm-images-build-openstack-2024-1-ubuntu_jammy
+        - openstack-helm-images-build-openstack-base-2024-1-ubuntu_jammy
         # - openstack-helm-images-compute-kit-2024-1-ubuntu_jammy
         # - openstack-helm-images-cinder-2024-1-ubuntu_jammy
     gate:
@@ -37,32 +38,43 @@
         # - openstack-helm-images-compute-kit-2024-1-ubuntu_jammy
         # - openstack-helm-images-cinder-2024-1-ubuntu_jammy
 
+- job:
+    name: openstack-helm-images-build-openstack-base-2024-1-ubuntu_jammy
+    description: Build base 2024.1 Jammy images
+    parent: openstack-helm-images-build-openstack-ubuntu_jammy
+    vars:
+      branchname: "stable/2024.1"
+      shortbranchname: "2024.1"
+    run:
+      - zuul.d/playbooks/build-openstack-base.yml
+
 - job:
     name: openstack-helm-images-build-openstack-2024-1-ubuntu_jammy
     description: Build 2024.1 Jammy images
     parent: openstack-helm-images-build-openstack-ubuntu_jammy
+    dependencies:
+      - name: openstack-helm-images-build-openstack-base-2024-1-ubuntu_jammy
     vars: &build_vars
       branchname: "stable/2024.1"
       shortbranchname: "2024.1"
       openstack_docker_images:
-        - requirements
         - keystone
-        - glance
+        # - glance
-        - cinder
+        # - cinder
-        - designate
+        # - designate
-        - nova
+        # - nova
-        - ironic
+        # - ironic
-        - neutron
+        # - neutron
-        - horizon
+        # - horizon
-        - heat
+        # - heat
-        - barbican
+        # - barbican
-        - placement
+        # - placement
-        - cyborg
+        # - cyborg
-        - magnum
+        # - magnum
-        - masakari
+        # - masakari
-        - masakari_monitors
+        # - masakari_monitors
-        - manila
+        # - manila
-        - tacker
+        # - tacker
     files: &openstack_files
       - ^openstack/loci/.*
       - zuul.d/openstack-2024.1.yaml

zuul.d/playbooks/Dockerfile

@@ -1,5 +1,5 @@
 FROM nginx:alpine
 
 # Previously built requirements image
-COPY --from=openstackhelm/requirements:latest / /usr/share/nginx/html
+COPY --from=requirements:latest / /usr/share/nginx/html
 RUN cd /usr/share/nginx/html; tar c -f wheels.tar.gz -z  *.whl *.txt; rm -f *.whl

zuul.d/playbooks/build-openstack-base.yml

@@ -0,0 +1,35 @@
+---
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: all[0]
+  gather_facts: true
+  vars_files:
+    - vars-openstack.yml
+  pre_tasks:
+    - name: Use buildset registry
+      include_role:
+        name: use-buildset-registry
+  tasks:
+    - name: Build base image
+      include_role:
+        name: build-docker-image
+      vars:
+        zuul_work_dir: "{{ zuul.project.src_dir }}"
+        docker_images: "{{ base_docker_images }}"
+
+    - name: Build base requirements image
+      include_role:
+        name: build-docker-image
+      vars:
+        zuul_work_dir: "src/opendev.org/openstack/loci"
+        docker_images: "{{ requirements_docker_images }}"

zuul.d/playbooks/build-openstack.yml

@@ -22,35 +22,10 @@
       include_role:
         name: use-buildset-registry
   tasks:
-    - name: Export the right vars
+    - name: Pull requirements from buildset registry
       shell: |
-        source /etc/ci/mirror_info.sh
+        docker pull "{{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ requirements_repository }}:{{ shortbranchname }}-{{ distroname }}"
-        echo ${NODEPOOL_OPENSUSE_MIRROR}/
+        docker tag "{{ buildset_registry_alias }}:{{ buildset_registry.port }}/{{ requirements_repository }}:{{ shortbranchname }}-{{ distroname }}" requirements:latest
-        echo ${NODEPOOL_PYPI_MIRROR}
-        echo ${NODEPOOL_MIRROR_HOST}
-      args:
-        executable: /bin/bash
-      changed_when: False
-      register: _zuul_mirrors
-    - name: Set fact with the exported variables
-      set_fact:
-        opensuse_package_mirror: "{{ _zuul_mirrors.stdout_lines[0] }}"
-        pip_index_url: "{{ _zuul_mirrors.stdout_lines[1] | default('https://pypi.python.org/simple/') }}"
-        pip_trusted_host: "{{ _zuul_mirrors.stdout_lines[2] | default('pypi.python.org') }}"
-
-    - name: Build base image
-      include_role:
-        name: build-docker-image
-      vars:
-        zuul_work_dir: "{{ zuul.project.src_dir }}"
-        docker_images: "{{ base_docker_images }}"
-
-    - name: Build base requirements image
-      include_role:
-        name: build-docker-image
-      vars:
-        zuul_work_dir: "src/opendev.org/openstack/loci"
-        docker_images: "{{ requirements_docker_images }}"
 
     - name: Build webserver serving wheels
       include_role:

zuul.d/playbooks/vars-openstack.yml

@@ -16,6 +16,7 @@ python3: "yes"
 #172.17.0.1 is default docker0 ip.
 wheels_location: "http://172.17.0.1:8080/wheels.tar.gz"
 base_repository: "base"
+requirements_repository: "requirements"
 
 base_docker_images:
   - context: "zuul.d/playbooks/"
@@ -25,33 +26,26 @@ base_docker_images:
       - "FROM='{{ from }}'"
       - "CEPH_REPO='{{ ceph_repo }}'"
       - "CEPH_KEY='{{ ceph_key }}'"
+    tags: &imagetag
+      - "{{ shortbranchname }}-{{ distroname }}"
+      - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}"
 
 requirements_docker_images:
   - context: "."
-    repository: openstackhelm/requirements
+    repository: "{{ requirements_repository }}"
     build_args:
-      - "FROM='{{ base_repository }}'"
+      - "FROM='{{ base_repository }}:{{ shortbranchname }}-{{ distroname }}'"
       - "PROJECT='requirements'"
       - "PROJECT_REF={{ branchname }}"
       - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
 
 openstack_docker_images_dict:
-  requirements:
-    context: "."
-    repository: openstackhelm/requirements
-    build_args:
-      - "FROM='{{ base_repository }}'"
-      - "PROJECT='requirements'"
-      - "PROJECT_REF={{ branchname }}"
-      - "KEEP_ALL_WHEELS=yes"
-    tags: &imagetag
-      - "{{ shortbranchname }}-{{ distroname }}"
-      - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}" # Frozen tag
   keystone:
     context: "."
     repository: openstackhelm/keystone
     build_args:
-      - "FROM='{{ base_repository }}'"
+      - "FROM='{{ base_repository }}:{{ shortbranchname }}-{{ distroname }}'"
       - "PROJECT='keystone'"
       - "PROJECT_REF={{ branchname }}"
       - "PROFILES='fluent apache ldap {{ shortbranchname }}'"