Add 2023.2 (Bobcat) images

Also this PR changes the way how openstack build jobs are organized. This is necessary to avoid rebuilding/retesting images for other releases if we change smth for a particular release job. Older jobs will also be reorganized later. Change-Id: If150a4bd7e72c2989a318b69b05b265f76dac84e
2023-10-02 19:39:38 -05:00 · 2023-10-02 19:39:38 -05:00 · 35e7e67da1
parent ebfef8ee4a
commit 35e7e67da1
6 changed files with 599 additions and 0 deletions
--- a/zuul.d/openstack-2023.2.yaml
+++ b/zuul.d/openstack-2023.2.yaml
@ -0,0 +1,124 @@
+---
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- project:
+    check:
+      jobs:
+        - openstack-helm-images-build-openstack-2023-2-ubuntu_jammy
+        # - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+        # - openstack-helm-images-cinder-2023-1-ubuntu_jammy
+    gate:
+      jobs:
+        - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
+        # - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+        # - openstack-helm-images-cinder-2023-2-ubuntu_jammy
+    promote:
+      jobs:
+        - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy
+    periodic-weekly:
+      jobs:
+        - openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
+        - openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy:
+            dependencies:
+              - name: openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
+              # - name: openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+              # - name: openstack-helm-images-cinder-2023-2-ubuntu_jammy
+        # - openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+        # - openstack-helm-images-cinder-2023-2-ubuntu_jammy
+
+- job:
+    name: openstack-helm-images-build-openstack-2023-2-ubuntu_jammy
+    description: Build 2023.2 Jammy images
+    parent: openstack-helm-images-build-openstack-ubuntu_jammy
+    vars: &build_vars
+      branchname: "stable/2023.2"
+      shortbranchname: "2023.2"
+    files: &openstack_files
+      - ^openstack/loci/.*
+      - zuul.d/openstack-2023.2.yaml
+      - zuul.d/openstack-base.yaml
+      - zuul.d/playbooks/build-openstack.yml
+      - zuul.d/playbooks/vars-openstack.yml
+      - zuul.d/playbooks/Dockerfile_base
+
+- job:
+    name: openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
+    parent: openstack-helm-images-upload-openstack-ubuntu_jammy
+    description: Build and upload 2023.2 Jammy images
+    vars: *build_vars
+    files: *openstack_files
+
+- job:
+    name: openstack-helm-images-promote-openstack-2023-2-ubuntu_jammy
+    parent: openstack-helm-images-promote-openstack-ubuntu_jammy
+    description: Promote previously published 2023.2 Jammy images
+    vars: *build_vars
+    files: *openstack_files
+
+# - job:
+#     name: openstack-helm-images-compute-kit-2023-2-ubuntu_jammy
+#     parent: openstack-helm-compute-kit-2023-2-ubuntu_jammy
+#     match-on-config-updates: false
+#     dependencies:
+#       - name: openstack-helm-images-build-openvswitch
+#         soft: true
+#       - name: openstack-helm-images-build-openstack-2023-2-ubuntu_jammy
+#         soft: true
+#       - name: openstack-helm-images-build-mariadb
+#         soft: true
+#       - name: openstack-helm-images-build-libvirt
+#         soft: true
+#       - name: openstack-helm-images-upload-openvswitch
+#         soft: true
+#       - name: openstack-helm-images-upload-openstack-2023-2-ubuntu_jammy
+#         soft: true
+#       - name: openstack-helm-images-upload-mariadb
+#         soft: true
+#       - name: openstack-helm-images-upload-libvirt
+#         soft: true
+#     files:
+#       - openvswitch/.*
+#       - zuul.d/openvswitch.yaml
+#       - mariadb/.*
+#       - zuul.d/mariadb.yaml
+#       - libvirt/.*
+#       - zuul.d/libvirt.yaml
+#       - ^openstack/loci/.*
+#       - zuul.d/openstack-2023.2.yaml
+#       - zuul.d/openstack-base.yaml
+#       - zuul.d/playbooks/build-openstack.yml
+#       - zuul.d/playbooks/vars-openstack.yml
+#       - zuul.d/playbooks/Dockerfile_base
+
+# - job:
+#     name: openstack-helm-images-cinder-2023-2-ubuntu_jammy
+#     parent: openstack-helm-cinder-2023-2-ubuntu_jammy
+#     match-on-config-updates: false
+#     dependencies:
+#       - name: openstack-helm-images-build-openstack-2023-2-ubuntu_focal
+#         soft: true
+#       - name: openstack-helm-images-build-mariadb
+#         soft: true
+#       - name: openstack-helm-images-upload-openstack-2023-2-ubuntu_focal
+#         soft: true
+#       - name: openstack-helm-images-upload-mariadb
+#         soft: true
+#     files:
+#       - mariadb/.*
+#       - zuul.d/mariadb.yaml
+#       - ^openstack/loci/.*
+#       - zuul.d/openstack-2023.2.yaml
+#       - zuul.d/openstack-base.yaml
+#       - zuul.d/playbooks/build-openstack.yml
+#       - zuul.d/playbooks/vars-openstack.yml
+#       - zuul.d/playbooks/Dockerfile_base
--- a/zuul.d/openstack-base.yaml
+++ b/zuul.d/openstack-base.yaml
@ -0,0 +1,92 @@
+---
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- job:
+    name: openstack-helm-images-build-openstack
+    parent: openstack-helm-images-build
+    abstract: true
+    timeout: 5400
+    required-projects:
+      - openstack/loci
+    run:
+      - zuul.d/playbooks/build-openstack.yml
+    vars: &base_vars
+      branchname: "master"
+      shortbranchname: "master"
+      distroname: ""
+      from: ""
+      ceph_repo: ""
+      ceph_key: ""
+      openstack_docker_images:
+        - requirements
+        - keystone
+        - glance
+        - cinder
+        - nova
+        - ironic
+        - neutron
+        - horizon
+        - heat
+        - barbican
+        - placement
+        - monasca_api
+        - cyborg
+        - masakari
+        - masakari_monitors
+        - manila
+        - tacker
+
+- job:
+    name: openstack-helm-images-upload-openstack
+    parent: openstack-helm-images-upload
+    abstract: true
+    timeout: 5400
+    required-projects:
+      - openstack/loci
+    run:
+      - zuul.d/playbooks/build-openstack.yml
+      - zuul.d/playbooks/upload-openstack.yml
+    secrets:
+      name: docker_credentials
+      secret: openstack-helm-dockerhub
+    vars: *base_vars
+
+- job:
+    name: openstack-helm-images-promote-openstack
+    parent: openstack-helm-images-promote
+    abstract: true
+    run:
+      - zuul.d/playbooks/promote-openstack.yml
+    vars: *base_vars
+
+- job:
+    name: openstack-helm-images-build-openstack-ubuntu_jammy
+    parent: openstack-helm-images-build-openstack
+    abstract: true
+    vars: &jammy_vars
+      distroname: "ubuntu_jammy"
+      from: "docker.io/ubuntu:22.04"
+      ceph_repo: "deb https://download.ceph.com/debian-reef/ jammy main"
+      ceph_key: "http://download.ceph.com/keys/release.asc"
+
+- job:
+    name: openstack-helm-images-upload-openstack-ubuntu_jammy
+    parent: openstack-helm-images-upload-openstack
+    abstract: true
+    vars: *jammy_vars
+
+- job:
+    name: openstack-helm-images-promote-openstack-ubuntu_jammy
+    parent: openstack-helm-images-promote-openstack
+    abstract: true
+    vars: *jammy_vars
--- a/zuul.d/playbooks/build-openstack.yml
+++ b/zuul.d/playbooks/build-openstack.yml
@ -0,0 +1,85 @@
+---
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: all[0]
+  gather_facts: true
+  vars_files:
+    - vars-openstack.yml
+  vars:
+    docker_images: "{{ openstack_docker_images | map('extract', openstack_docker_images_dict) | list }}"
+  pre_tasks:
+    - name: Use buildset registry
+      include_role:
+        name: use-buildset-registry
+  tasks:
+    - name: Export the right vars
+      shell: |
+        source /etc/ci/mirror_info.sh
+        echo ${NODEPOOL_OPENSUSE_MIRROR}/
+        echo ${NODEPOOL_PYPI_MIRROR}
+        echo ${NODEPOOL_MIRROR_HOST}
+      args:
+        executable: /bin/bash
+      changed_when: False
+      register: _zuul_mirrors
+    - name: Set fact with the exported variables
+      set_fact:
+        opensuse_package_mirror: "{{ _zuul_mirrors.stdout_lines[0] }}"
+        pip_index_url: "{{ _zuul_mirrors.stdout_lines[1] | default('https://pypi.python.org/simple/') }}"
+        pip_trusted_host: "{{ _zuul_mirrors.stdout_lines[2] | default('pypi.python.org') }}"
+
+    - name: Build base image
+      include_role:
+        name: build-docker-image
+      vars:
+        zuul_work_dir: "{{ zuul.project.src_dir }}"
+        docker_images: "{{ base_docker_images }}"
+
+    - name: Build base requirements image
+      include_role:
+        name: build-docker-image
+      vars:
+        zuul_work_dir: "src/opendev.org/openstack/loci"
+        docker_images: "{{ requirements_docker_images }}"
+
+    - name: Build webserver serving wheels
+      include_role:
+        name: build-docker-image
+      vars:
+        zuul_work_dir: "{{ zuul.project.src_dir }}"
+        docker_images:
+          - context: zuul.d/playbooks/
+            repository: openstackhelm/loci_wheels
+
+    - name: Allow access to wheels server
+      iptables:
+        action: insert
+        chain: INPUT
+        in_interface: docker0
+        jump: ACCEPT
+      become: yes
+
+    - name: Run http server containing wheels
+      command: docker run -d -p 0.0.0.0:8080:80 openstackhelm/loci_wheels:latest
+
+    - name: Build project images
+      include_role:
+        name: build-docker-image
+      vars:
+        zuul_work_dir: "src/opendev.org/openstack/loci"
+
+    - name: Return images to zuul
+      include_tasks: _return-image.yml
+      loop: "{{ docker_images }}"
+      loop_control:
+        loop_var: zj_image
--- a/zuul.d/playbooks/promote-openstack.yml
+++ b/zuul.d/playbooks/promote-openstack.yml
@ -0,0 +1,7 @@
+- hosts: localhost
+  vars_files:
+    - vars-openstack.yml
+  vars:
+    docker_images: "{{ openstack_docker_images | map('extract', openstack_docker_images_dict) | list }}"
+  roles:
+    - promote-docker-image
--- a/zuul.d/playbooks/upload-openstack.yml
+++ b/zuul.d/playbooks/upload-openstack.yml
@ -0,0 +1,7 @@
+- hosts: all
+  vars_files:
+    - vars-openstack.yml
+  vars:
+    docker_images: "{{ openstack_docker_images | map('extract', openstack_docker_images_dict) | list }}"
+  roles:
+    - role: upload-docker-image
--- a/zuul.d/playbooks/vars-openstack.yml
+++ b/zuul.d/playbooks/vars-openstack.yml
@ -0,0 +1,284 @@
+---
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+currentdate: "{{ now(utc=True,fmt='%Y%m%d') }}"
+python3: "yes"
+#172.17.0.1 is default docker0 ip.
+wheels_location: "http://172.17.0.1:8080/wheels.tar.gz"
+base_repository: "base"
+
+base_docker_images:
+  - context: "zuul.d/playbooks/"
+    dockerfile: "Dockerfile_base"
+    repository: '{{ base_repository }}'
+    build_args:
+      - "FROM='{{ from }}'"
+      - "CEPH_REPO='{{ ceph_repo }}'"
+      - "CEPH_KEY='{{ ceph_key }}'"
+
+requirements_docker_images:
+  - context: "."
+    repository: openstackhelm/requirements
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='requirements'"
+      - "PROJECT_REF={{ branchname }}"
+      - "KEEP_ALL_WHEELS=yes"
+
+openstack_docker_images_dict:
+  requirements:
+    context: "."
+    repository: openstackhelm/requirements
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='requirements'"
+      - "PROJECT_REF={{ branchname }}"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: &imagetag
+      - "{{ shortbranchname }}-{{ distroname }}"
+      - "{{ shortbranchname }}-{{ distroname }}-{{ currentdate }}" # Frozen tag
+  keystone:
+    context: "."
+    repository: openstackhelm/keystone
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='keystone'"
+      - "PROJECT_REF={{ branchname }}"
+      - "PROFILES='fluent apache ldap {{ shortbranchname }}'"
+      - "PIP_PACKAGES='python-openstackclient'"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
+  heat:
+    context: "."
+    repository: openstackhelm/heat
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='heat'"
+      - "PROJECT_REF={{ branchname }}"
+      - "PROFILES='fluent apache'"
+      - "WHEELS='{{ wheels_location }}'"
+      - "DIST_PACKAGES='curl'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
+  barbican:
+    context: "."
+    repository: openstackhelm/barbican
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='barbican'"
+      - "PROJECT_REF={{ branchname }}"
+      - "PROFILES='fluent'"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+      - "DIST_PACKAGES='python3-dev gcc'"
+      - "PIP_ARGS='--only-binary :none:'"
+    tags: *imagetag
+  glance:
+    context: "."
+    repository: openstackhelm/glance
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='glance'"
+      - "PROJECT_REF={{ branchname }}"
+      - "PROFILES='fluent ceph'"
+      - "PIP_PACKAGES='python-swiftclient'"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
+  cinder:
+    context: "."
+    repository: openstackhelm/cinder
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='cinder'"
+      - "PROJECT_REF={{ branchname }}"
+      - "PROFILES='fluent lvm ceph qemu apache'"
+      - "PIP_PACKAGES='python-swiftclient'"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
+  neutron:
+    context: "."
+    repository: openstackhelm/neutron
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='neutron'"
+      - "PROJECT_REF={{ branchname }}"
+      - "PROFILES='fluent linuxbridge openvswitch apache vpn'"
+      - "PIP_PACKAGES='tap-as-a-service'"
+      - "DIST_PACKAGES='jq ethtool lshw'"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
+  neutron_sriov:
+    context: "."
+    repository: openstackhelm/neutron
+    build_args:
+      - "FROM=docker.io/ubuntu:18.04"
+      - "PROJECT='neutron'"
+      - "PROJECT_REF={{ branchname }}"
+      - "PROFILES='fluent neutron linuxbridge openvswitch'"
+      - "PIP_PACKAGES='tap-as-a-service'"
+      - "DIST_PACKAGES='ethtool lshw'"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags:
+      - "{{ shortbranchname }}-18.04-sriov"
+      - "{{ shortbranchname }}-18.04-sriov-{{ currentdate }}"
+  nova:
+    context: "."
+    repository: openstackhelm/nova
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='nova'"
+      - "PROJECT_REF={{ branchname }}"
+      - "PROFILES='fluent ceph linuxbridge openvswitch configdrive qemu apache migration'"
+      - "DIST_PACKAGES='net-tools openssh-server'"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
+  horizon:
+    context: "."
+    repository: openstackhelm/horizon
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='horizon'"
+      - "PROJECT_REF={{ branchname }}"
+      - "PROFILES='fluent apache'"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
+  senlin:
+    context: "."
+    repository: openstackhelm/senlin
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='senlin'"
+      - "PROJECT_REF={{ branchname }}"
+      - "PROFILES='fluent'"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
+  magnum:
+    context: "."
+    repository: openstackhelm/magnum
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='magnum'"
+      - "PROJECT_REF={{ branchname }}"
+      - "PROFILES='fluent'"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
+  ironic:
+    context: "."
+    repository: openstackhelm/ironic
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='ironic'"
+      - "PROJECT_REF={{ branchname }}"
+      - "PROFILES='fluent ipxe ipmi qemu tftp'"
+      - "DIST_PACKAGES='ethtool lshw iproute2'"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
+  placement:
+    context: "."
+    repository: openstackhelm/placement
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='placement'"
+      - "PROJECT_REF={{ branchname }}"
+      - "PROFILES='apache'"
+      - "PIP_PACKAGES='httplib2'"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
+  watcher:
+    context: "."
+    repository: openstackhelm/watcher
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='watcher'"
+      - "PROJECT_REF={{ branchname }}"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
+  monasca_api:
+    context: "."
+    repository: openstackhelm/monasca-api
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='monasca-api'"
+      - "PROJECT_REF={{ branchname }}"
+      - "PROFILES='monasca api'"
+      - "PIP_PACKAGES='influxdb cassandra-driver sqlalchemy'"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
+  cyborg:
+    context: "."
+    repository: openstackhelm/cyborg
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='cyborg'"
+      - "PROJECT_REF={{ branchname }}"
+      - "DIST_PACKAGES='pciutils'"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
+  masakari:
+    context: "."
+    repository: openstackhelm/masakari
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PYTHON3={{ python3 }}"
+      - "PROJECT='masakari'"
+      - "PROJECT_REF={{ branchname }}"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
+  masakari_monitors:
+    context: "."
+    repository: openstackhelm/masakari-monitors
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PYTHON3={{ python3 }}"
+      - "PROJECT='masakari-monitors'"
+      - "PROJECT_REF={{ branchname }}"
+      - "PIP_PACKAGES='libvirt-python'"
+      - "DIST_PACKAGES='libvirt-daemon-system libvirt-clients tcpdump pacemaker-cli-utils'"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
+  manila:
+    context: "."
+    repository: openstackhelm/manila
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='manila'"
+      - "PROJECT_REF={{ branchname }}"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag
+  tacker:
+    context: "."
+    repository: openstackhelm/tacker
+    build_args:
+      - "FROM='{{ base_repository }}'"
+      - "PROJECT='tacker'"
+      - "PROJECT_REF={{ branchname }}"
+      - "WHEELS='{{ wheels_location }}'"
+      - "KEEP_ALL_WHEELS=yes"
+    tags: *imagetag