Full site deploy with custom images

Change-Id: I7f85af1f4f0f9a8972dfd54cd0128d4e9caafc2b

.zuul.yaml

@@ -129,7 +129,10 @@
       loopback_device: /dev/loop100
       loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img"
       ceph_osd_data_device: /dev/loop100
-      kube_version: "1.28.4-1.1"
+      kube_version_repo: "v1.29"
+      # the list of k8s package versions are available here
+      # https://pkgs.k8s.io/core:/stable:/{{ kube_version_repo }}/deb/Packages
+      kube_version: "1.29.2-1.1"
       calico_version: "v3.27.0"
       coredns_version: "v1.11.1"
       helm_version: "v3.13.2"
@@ -160,6 +163,7 @@
         - ./tools/deployment/airskiff/developer/000-clone-dependencies.sh
         - ./tools/deployment/airskiff/developer/020-setup-client.sh
         - ./tools/deployment/airskiff/developer/015-make-all-charts.sh
+        - ./tools/deployment/airskiff/developer/017-make-all-images.sh
         - ./tools/deployment/airskiff/developer/025-start-artifactory.sh
         - ./tools/deployment/airskiff/developer/026-reduce-site.sh
         - ./tools/deployment/airskiff/developer/030-armada-bootstrap.sh
@@ -182,6 +186,13 @@
       HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
       OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
       OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
+      DISTRO: ubuntu_focal
+      DOCKER_REGISTRY: localhost:5000
+      MAKE_ARMADA_IMAGES: true
+      MAKE_DECKHAND_IMAGES: true
+      MAKE_SHIPYARD_IMAGES: true
+      MAKE_PORTHOLE_IMAGES: true
+      MAKE_PROMENADE_IMAGES: true
       gate_scripts_relative_path: .
       gate_scripts:
         - ./tools/deployment/airskiff/developer/000-prepare-k8s.sh
@@ -189,6 +200,7 @@
         - ./tools/deployment/airskiff/developer/000-clone-dependencies.sh
         - ./tools/deployment/airskiff/developer/020-setup-client.sh
         - ./tools/deployment/airskiff/developer/015-make-all-charts.sh
+        - ./tools/deployment/airskiff/developer/017-make-all-images.sh
         - ./tools/deployment/airskiff/developer/025-start-artifactory.sh
         - ./tools/deployment/airskiff/developer/026-reduce-site.sh
         - ./tools/deployment/airskiff/developer/030-armada-bootstrap.sh

tools/deployment/airskiff/developer/015-make-all-charts.sh

@@ -30,6 +30,24 @@ CURRENT_DIR="$(pwd)"
 : "${MAKE_CHARTS_PORTHOLE:=true}"
 : "${MAKE_CHARTS_PROMENADE:=true}"
 
+
+MAKE_CHARTS_OPENSTACK_HELM=$(echo "$MAKE_CHARTS_OPENSTACK_HELM" | tr '[:upper:]' '[:lower:]')
+MAKE_CHARTS_OSH_INFRA=$(echo "$MAKE_CHARTS_OSH_INFRA" | tr '[:upper:]' '[:lower:]')
+MAKE_CHARTS_ARMADA=$(echo "$MAKE_CHARTS_ARMADA" | tr '[:upper:]' '[:lower:]')
+MAKE_CHARTS_DECKHAND=$(echo "$MAKE_CHARTS_DECKHAND" | tr '[:upper:]' '[:lower:]')
+MAKE_CHARTS_SHIPYARD=$(echo "$MAKE_CHARTS_SHIPYARD" | tr '[:upper:]' '[:lower:]')
+MAKE_CHARTS_MAAS=$(echo "$MAKE_CHARTS_MAAS" | tr '[:upper:]' '[:lower:]')
+MAKE_CHARTS_PORTHOLE=$(echo "$MAKE_CHARTS_PORTHOLE" | tr '[:upper:]' '[:lower:]')
+MAKE_CHARTS_PROMENADE=$(echo "$MAKE_CHARTS_PROMENADE" | tr '[:upper:]' '[:lower:]')
+export MAKE_CHARTS_OPENSTACK_HELM
+export MAKE_CHARTS_OSH_INFRA
+export MAKE_CHARTS_ARMADA
+export MAKE_CHARTS_DECKHAND
+export MAKE_CHARTS_SHIPYARD
+export MAKE_CHARTS_MAAS
+export MAKE_CHARTS_PORTHOLE
+export MAKE_CHARTS_PROMENADE
+
 mkdir -p "${ARTIFACTS_PATH}"
 
 cd "${INSTALL_PATH}"

tools/deployment/airskiff/developer/017-make-all-images.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+CURRENT_DIR="$(pwd)"
+: "${INSTALL_PATH:="../"}"
+: "${DISTRO:=ubuntu_focal}"
+: "${DOCKER_REGISTRY:=localhost}"
+: "${MAKE_ARMADA_IMAGES:=false}"
+: "${MAKE_DECKHAND_IMAGES:=false}"
+: "${MAKE_SHIPYARD_IMAGES:=false}"
+: "${MAKE_PORTHOLE_IMAGES:=false}"
+: "${MAKE_PROMENADE_IMAGES:=false}"
+
+# Convert both values to lowercase (or uppercase)
+MAKE_ARMADA_IMAGES=$(echo "$MAKE_ARMADA_IMAGES" | tr '[:upper:]' '[:lower:]')
+MAKE_DECKHAND_IMAGES=$(echo "$MAKE_DECKHAND_IMAGES" | tr '[:upper:]' '[:lower:]')
+MAKE_SHIPYARD_IMAGES=$(echo "$MAKE_SHIPYARD_IMAGES" | tr '[:upper:]' '[:lower:]')
+MAKE_PORTHOLE_IMAGES=$(echo "$MAKE_PORTHOLE_IMAGES" | tr '[:upper:]' '[:lower:]')
+MAKE_PROMENADE_IMAGES=$(echo "$MAKE_PROMENADE_IMAGES" | tr '[:upper:]' '[:lower:]')
+
+export MAKE_ARMADA_IMAGES
+export MAKE_DECKHAND_IMAGES
+export MAKE_SHIPYARD_IMAGES
+export MAKE_PORTHOLE_IMAGES
+export MAKE_PROMENADE_IMAGES
+
+cd "${INSTALL_PATH}"
+
+# Start docker registry
+docker rm registry --force || true
+docker run -d -p 5000:5000 --restart=always --name registry registry:2
+curl -Ik "http://${DOCKER_REGISTRY}"
+
+# Make charts in Airship and OSH-INFRA projects
+if [[ ${MAKE_ARMADA_IMAGES} = true ]] ; then
+    pushd armada
+    make images
+    docker push "${DOCKER_REGISTRY}/airshipit/armada:latest-${DISTRO}"
+    popd
+    pushd treasuremap
+    sed -i "s#quay.io/airshipit/armada:latest-${DISTRO}#${DOCKER_REGISTRY}/airshipit/armada:latest-${DISTRO}#g" ./site/airskiff/software/config/versions.yaml
+    sed -i "s#quay.io/airshipit/armada:latest-${DISTRO}#${DOCKER_REGISTRY}/airshipit/armada:latest-${DISTRO}#g" ./tools/deployment/airskiff/developer/030-armada-bootstrap.sh
+    sed -i "s#quay.io/airshipit/armada:latest-${DISTRO}#${DOCKER_REGISTRY}/airshipit/armada:latest-${DISTRO}#g" ./global/software/config/versions.yaml
+    popd
+fi
+if [[ ${MAKE_DECKHAND_IMAGES} = true ]] ; then
+    pushd deckhand
+    make images
+    docker push "${DOCKER_REGISTRY}/airshipit/deckhand:latest-${DISTRO}"
+    popd
+    pushd treasuremap
+    sed -i "s#quay.io/airshipit/deckhand:latest-${DISTRO}#${DOCKER_REGISTRY}/airshipit/deckhand:latest-${DISTRO}#g" ./site/airskiff/software/config/versions.yaml
+    sed -i "s#quay.io/airshipit/deckhand:latest-${DISTRO}#${DOCKER_REGISTRY}/airshipit/deckhand:latest-${DISTRO}#g" ./global/software/config/versions.yaml
+    popd
+fi
+if [[ ${MAKE_SHIPYARD_IMAGES} = true ]] ; then
+    pushd shipyard
+    make images
+    docker push "${DOCKER_REGISTRY}/airshipit/airflow:latest-${DISTRO}"
+    docker push "${DOCKER_REGISTRY}/airshipit/shipyard:latest-${DISTRO}"
+    popd
+    pushd treasuremap
+    sed -i "s#quay.io/airshipit/airflow:latest-${DISTRO}#${DOCKER_REGISTRY}/airshipit/airflow:latest-${DISTRO}#g" ./site/airskiff/software/config/versions.yaml
+    sed -i "s#quay.io/airshipit/shipyard:latest-${DISTRO}#${DOCKER_REGISTRY}/airshipit/shipyard:latest-${DISTRO}#g" ./site/airskiff/software/config/versions.yaml
+    popd
+fi
+if [[ ${MAKE_PORTHOLE_IMAGES} = true ]] ; then
+    pushd porthole
+    make images
+    # Define a list of images
+    IMAGE_LIST=("calicoctl-utility" "ceph-utility" "compute-utility" "etcdctl-utility" "mysqlclient-utility" "openstack-utility" "postgresql-utility")
+    for IMAGE in "${IMAGE_LIST}"
+    do
+        docker push "${DOCKER_REGISTRY}/airshipit/porthole-${IMAGE}:latest-${DISTRO}"
+    done
+    popd
+fi
+if [[ ${MAKE_PROMENADE_IMAGES} = true ]] ; then
+    pushd promenade
+    make images
+    docker push "${DOCKER_REGISTRY}/airshipit/promenade:latest-${DISTRO}"
+    popd
+fi
+
+docker images
+cd "${CURRENT_DIR}"

tools/gate/playbooks/roles/airship-run-script/defaults/main.yaml

@@ -24,4 +24,11 @@ HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
 OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
 OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
 COREDNS_VERSION: v1.11.1
+DISTRO: ubuntu_focal
+DOCKER_REGISTRY: localhost:5000
+MAKE_ARMADA_IMAGES: false
+MAKE_DECKHAND_IMAGES: false
+MAKE_SHIPYARD_IMAGES: false
+MAKE_PORTHOLE_IMAGES: false
+MAKE_PROMENADE_IMAGES: false
 ...

tools/gate/playbooks/roles/airship-run-script/tasks/main.yaml

@@ -37,4 +37,11 @@
     OSH_INFRA_COMMIT: "{{ OSH_INFRA_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
     OSH_COMMIT: "{{ OSH_COMMIT | default('2d9457e34ca4200ed631466bd87569b0214c92e7') }}"
     COREDNS_VERSION: "{{ coredns_version | default('v1.11.1') }}"
+    DISTRO: "{{ DISTRO | default('ubuntu_focal') }}"
+    DOCKER_REGISTRY: "{{ DOCKER_REGISTRY | default('localhost:5000') }}"
+    MAKE_ARMADA_IMAGES: "{{ MAKE_ARMADA_IMAGES | default('false') }}"
+    MAKE_DECKHAND_IMAGES: "{{ MAKE_DECKHAND_IMAGES | default('false') }}"
+    MAKE_SHIPYARD_IMAGES: "{{ MAKE_SHIPYARD_IMAGES | default('false') }}"
+    MAKE_PORTHOLE_IMAGES: "{{ MAKE_PORTHOLE_IMAGES | default('false') }}"
+    MAKE_PROMENADE_IMAGES: "{{ MAKE_PROMENADE_IMAGES | default('false') }}"
 ...