Airskiff based maas deployment job

Change-Id: Ic7285d6b5a9d25641ff32cee50e4dcabaa6d5a03
Signed-off-by: Sergiy Markin <smarkin@mirantis.com>

.zuul.yaml

@@ -20,7 +20,8 @@
         - airship-maas-docker-build-gate-ubuntu-focal
         - airship-maas-docker-build-gate-ubuntu-jammy
         - airship-maas-lint-yaml
-        - airship-maas-deploy-kubeadm
+        - maas-airskiff-deployment-jammy-kubeadm
+        - maas-airskiff-deployment-focal-kubeadm
     gate:
       jobs:
         - airship-maas-lint-ws
@@ -28,7 +29,7 @@
         - airship-maas-docker-build-gate-ubuntu-focal
         - airship-maas-docker-build-gate-ubuntu-jammy
         - airship-maas-lint-yaml
-        - airship-maas-deploy-kubeadm
+
     post:
       jobs:
         - airship-maas-docker-publish
@@ -48,6 +49,8 @@
 
 - job:
     name: airship-maas-lint-ws
+    # Pinned for now due to Ansible 11 stopped verbosive output.
+    ansible-version: 9
     description: |
       Lints all files for trailing whitespace
     run: tools/gate/playbooks/zuul-linter.yaml
@@ -56,6 +59,8 @@
 
 - job:
     name: airship-maas-chart-build-gate
+    # Pinned for now due to Ansible 11 stopped verbosive output.
+    ansible-version: 9
     description: Build charts using pinned Helm toolkit.
     run: tools/gate/playbooks/helm-linter.yaml
     timeout: 600
@@ -65,6 +70,8 @@
 
 - job:
     name: airship-maas-chart-build-latest-htk
+    # Pinned for now due to Ansible 11 stopped verbosive output.
+    ansible-version: 9
     description: Build charts using latest Helm toolkit.
     voting: false
     run: tools/gate/playbooks/helm-linter.yaml
@@ -75,6 +82,8 @@
 
 - job:
     name: airship-maas-docker-build-gate-ubuntu-jammy
+    # Pinned for now due to Ansible 11 stopped verbosive output.
+    ansible-version: 9
     timeout: 3600
     run: tools/gate/playbooks/docker-image-build.yaml
     nodeset: airship-maas-single-node-jammy
@@ -89,6 +98,8 @@
 
 - job:
     name: airship-maas-docker-build-gate-ubuntu-focal
+    # Pinned for now due to Ansible 11 stopped verbosive output.
+    ansible-version: 9
     timeout: 3600
     run: tools/gate/playbooks/docker-image-build.yaml
     nodeset: airship-maas-single-node-jammy
@@ -104,6 +115,8 @@
 
 - job:
     name: airship-maas-lint-yaml
+    # Pinned for now due to Ansible 11 stopped verbosive output.
+    ansible-version: 9
     voting: true
     timeout: 900
     run: tools/gate/playbooks/lint-yaml.yaml
@@ -113,6 +126,8 @@
 
 - job:
     name: airship-maas-docker-publish
+    # Pinned for now due to Ansible 11 stopped verbosive output.
+    ansible-version: 9
     timeout: 3600
     run: tools/gate/playbooks/docker-image-build.yaml
     nodeset: airship-maas-single-node-jammy
@@ -130,51 +145,49 @@
         static:
           - latest
 
-
 - job:
-    name: airship-maas-deploy-base
+    name: maas-airskiff-deployment-jammy-kubeadm
+    # Pinned for now due to Ansible 11 stopped verbosive output.
+    ansible-version: 9
     description: |
-      Run integration tests for the Airship Deckhand project
-    abstract: true
-    parent: treasuremap-airskiff-infra-deploy-base
+      Deploy maas using Airskiff and submitted Shipyard changes.
+    parent: treasuremap-airskiff-deploy-maas-drydock-base
+    nodeset: treasuremap-airskiff-1node-32GB-ubuntu_jammy
     required-projects:
       - name: airship/treasuremap
         override-checkout: v1.9
     roles:
       - zuul: airship/treasuremap
     vars:
-      gate_scripts_relative_path: ../../openstack/openstack-helm
-      gate_scripts:
-        - ./tools/deployment/common/prepare-k8s.sh
-        - ./tools/deployment/common/setup-client.sh
-        - ./tools/deployment/common/prepare-charts.sh
-        - ./tools/deployment/ceph/ceph.sh
-        - ./tools/deployment/ceph/ceph-ns-activate.sh
-    irrelevant-files:
-      - ^.*\.rst$
-      - ^doc/.*$
-      - ^releasenotes/.*$
-      - ^site/seaworthy/.*$
-      - ^site/airsloop/.*$
-      - ^site/aiab/.*$
-
+      treasuremap_ref: v1.9
+      CLONE_MAAS: false
+      MAKE_MAAS_IMAGES: true
+      DISTRO: ubuntu_jammy
+      DOCKER_REGISTRY: localhost:5000
+      gate_scripts_relative_path: ../../airship/treasuremap
+      zuul_treasuremap_relative_path: ../../airship/treasuremap
 
 - job:
-    name: airship-maas-deploy-kubeadm
-    description: Deploys all UC's and Executes functional tests
-    parent: airship-maas-deploy-base
-    nodeset: treasuremap-airskiff-1node-ubuntu_jammy
-    irrelevant-files:
-      - ^.*\.rst$
-      - ^doc/.*$
-      - ^releasenotes/.*$
-      - ^site/seaworthy/.*$
-      - ^site/airsloop/.*$
-      - ^site/aiab/.*$
-    run:
-      - tools/gate/playbooks/airship-run-scripts.yaml
-      - tools/gate/playbooks/run-integration-tests.yaml
-
+    name: maas-airskiff-deployment-focal-kubeadm
+    # Pinned for now due to Ansible 11 stopped verbosive output.
+    ansible-version: 9
+    description: |
+      Deploy maas using Airskiff and submitted Shipyard changes.
+    parent: treasuremap-airskiff-deploy-maas-drydock-base
+    nodeset: treasuremap-airskiff-1node-32GB-ubuntu_jammy
+    required-projects:
+      - name: airship/treasuremap
+        override-checkout: v1.9
+    roles:
+      - zuul: airship/treasuremap
+    vars:
+      treasuremap_ref: v1.9
+      CLONE_MAAS: false
+      MAKE_MAAS_IMAGES: true
+      DISTRO: ubuntu_focal
+      DOCKER_REGISTRY: localhost:5000
+      gate_scripts_relative_path: ../../airship/treasuremap
+      zuul_treasuremap_relative_path: ../../airship/treasuremap
 
 - secret:
     name: airship_maas_quay_creds
@@ -204,6 +217,8 @@
 
 - job:
     name: maas-upload-git-mirror
+    # Pinned for now due to Ansible 11 stopped verbosive output.
+    ansible-version: 9
     parent: upload-git-mirror
     nodeset: treasuremap-airskiff-1node-ubuntu_jammy
     description: Mirrors airship/maas to airshipit/maas

Makefile

@@ -27,8 +27,10 @@ PUSH_IMAGE        ?= false
 LABEL             ?= org.airshipit.build=community
 COMMIT            ?= $(shell git rev-parse HEAD)
 DISTRO            ?= ubuntu_jammy
+DISTRO_ALIAS	  ?= ubuntu_jammy
 STRIPPED_DISTRO   := $(shell echo $(DISTRO) | sed 's/^ubuntu_//')
-IMAGE_NAME        := maas-rack-controller-$(STRIPPED_DISTRO) maas-region-controller-$(STRIPPED_DISTRO) sstream-cache-$(STRIPPED_DISTRO)
+STRIPPED_DISTRO_ALIAS   := $(shell echo $(DISTRO_ALIAS) | sed 's/^ubuntu_//')
+IMAGE_NAME        := maas-rack-controller maas-region-controller sstream-cache
 BUILD_DIR         := $(shell mktemp -d)
 HELM              := $(BUILD_DIR)/helm
 SSTREAM_IMAGE     := "https://images.maas.io/ephemeral-v3/stable/"
@@ -37,6 +39,10 @@ UBUNTU_BASE_IMAGE ?= quay.io/airshipit/ubuntu:$(STRIPPED_DISTRO)
 USE_CACHED_IMG    ?= false
 DOCKER_EXTRA_ARGS ?=
 
+IMAGE			  := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$(IMAGE_NAME):${IMAGE_TAG}
+IMAGE_ALIAS       := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME_ALIAS}:${IMAGE_TAG}
+
+
 ifeq ($(USE_CACHED_IMG), true)
     DOCKER_EXTRA_ARGS += --build-arg BUILDKIT_INLINE_CACHE=1
 else
@@ -55,8 +61,12 @@ images: $(IMAGE_NAME)
 
 $(IMAGE_NAME):
 	@echo
-	@echo "===== Processing [$@] image ====="
-	@make build IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$@:${IMAGE_TAG} IMAGE_DIR=images/$@
+	@echo "===== Processing [$@-$(STRIPPED_DISTRO)] image ====="
+	@make build \
+		IMAGE=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$@-$(STRIPPED_DISTRO):${IMAGE_TAG} \
+		IMAGE_ALIAS=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$@-$(STRIPPED_DISTRO_ALIAS):${IMAGE_TAG} \
+		IMAGE_NAME_FULL=$@-$(STRIPPED_DISTRO) \
+		IMAGE_DIR=images/$@-$(STRIPPED_DISTRO)
 
 # Create tgz of the chart
 .PHONY: charts
@@ -85,13 +95,22 @@ build:
 	docker build -t $(IMAGE) --label $(LABEL) --network=host \
         --label "org.opencontainers.image.revision=$(COMMIT)" \
         --label "org.opencontainers.image.created=$(shell date --rfc-3339=seconds --utc)" \
-        --label "org.opencontainers.image.title=$(IMAGE_NAME)" \
+        --label "org.opencontainers.image.title=$(IMAGE_NAME_FULL)" \
         -f $(IMAGE_DIR)/Dockerfile \
         $(DOCKER_EXTRA_ARGS) \
         --build-arg FROM=$(UBUNTU_BASE_IMAGE) \
         --build-arg SSTREAM_IMAGE=$(SSTREAM_IMAGE) \
         --build-arg SSTREAM_RELEASE=$(SSTREAM_RELEASE) \
         $(IMAGE_DIR)
+ifneq ($(DISTRO), $(DISTRO_ALIAS))
+	docker tag $(IMAGE) $(IMAGE_ALIAS)
+ifeq ($(DOCKER_REGISTRY), localhost:5000)
+	docker push $(IMAGE_ALIAS)
+endif
+endif
+ifeq ($(DOCKER_REGISTRY), localhost:5000)
+	docker push $(IMAGE)
+endif
 ifeq ($(PUSH_IMAGE), true)
 	docker push $(IMAGE)
 endif

images/maas-rack-controller-focal/Dockerfile

@@ -15,10 +15,10 @@ ARG http_proxy
 ARG https_proxy
 ARG no_proxy
 
-ENV DEBIAN_FRONTEND noninteractive
-ENV container docker
+ENV DEBIAN_FRONTEND=noninteractive
+ENV container=docker
 
-ENV MAAS_VERSION 1:3.0.2-10056-g.089ec66c2-0ubuntu1~20.04.1
+ENV MAAS_VERSION=1:3.0.2-10056-g.089ec66c2-0ubuntu1~20.04.1
 
 RUN apt-get -qq update \
  && apt-get install -y \

images/maas-rack-controller-jammy/Dockerfile

@@ -15,10 +15,10 @@ ARG http_proxy
 ARG https_proxy
 ARG no_proxy
 
-ENV DEBIAN_FRONTEND noninteractive
-ENV container docker
+ENV DEBIAN_FRONTEND=noninteractive
+ENV container=docker
 
-ENV MAAS_VERSION 3.5
+ENV MAAS_VERSION=3.5
 
 RUN apt-get -qq update \
  && apt upgrade -y \

images/maas-region-controller-focal/Dockerfile

@@ -15,10 +15,10 @@ ARG http_proxy
 ARG https_proxy
 ARG no_proxy
 
-ENV DEBIAN_FRONTEND noninteractive
-ENV container docker
+ENV DEBIAN_FRONTEND=noninteractive
+ENV container=docker
 
-ENV MAAS_VERSION 1:3.0.2-10056-g.089ec66c2-0ubuntu1~20.04.1
+ENV MAAS_VERSION=1:3.0.2-10056-g.089ec66c2-0ubuntu1~20.04.1
 
 RUN apt-get -qq update \
  && apt-get install -y \

images/maas-region-controller-jammy/Dockerfile

@@ -15,10 +15,10 @@ ARG http_proxy
 ARG https_proxy
 ARG no_proxy
 
-ENV DEBIAN_FRONTEND noninteractive
-ENV container docker
+ENV DEBIAN_FRONTEND=noninteractive
+ENV container=docker
 
-ENV MAAS_VERSION 3.5
+ENV MAAS_VERSION=3.5
 
 RUN apt-get -qq update \
  && apt upgrade -y \

images/sstream-cache-focal/Dockerfile

@@ -16,7 +16,7 @@ ARG https_proxy
 ARG no_proxy
 
 ARG SSTREAM_IMAGE=https://images.maas.io/ephemeral-v3/stable/
-ARG SSTREAM_RELEASE=bionic
+ARG SSTREAM_RELEASE=focal
 
 ENV DEBIAN_FRONTEND=noninteractive
 

tools/gate/playbooks/airship-run-scripts.yaml

@@ -1,75 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
----
-- hosts: primary
-  tasks:
-    - name: Override images
-      when: buildset_registry is defined
-      vars:
-        work_dir: "{{ zuul.project.src_dir }}"
-      block:
-        - name: Buildset registry alias
-          include_role:
-            name: deploy-env
-            tasks_from: buildset_registry_alias
-
-        - name: Print zuul
-          debug:
-            var: zuul
-
-        - name: Override proposed images from artifacts
-          shell: >
-            find {{ override_paths | join(" ") }} -type f -exec sed -Ei
-            "s#['\"]?docker\.io/({{ repo }}):({{ tag }})['\"]?\$#{{ buildset_registry_alias }}:{{ buildset_registry.port }}/\1:\2#g" {} +
-          loop: "{{ zuul.artifacts | default([]) }}"
-          args:
-            chdir: "{{ work_dir }}"
-          loop_control:
-            loop_var: zj_zuul_artifact
-          when: "'metadata' in zj_zuul_artifact and zj_zuul_artifact.metadata.type | default('') == 'container_image'"
-          vars:
-            tag: "{{ zj_zuul_artifact.metadata.tag }}"
-            repo: "{{ zj_zuul_artifact.metadata.repository }}"
-            override_paths:
-              - ../openstack-helm*/*/values*
-              - ../openstack-helm-infra/tools/deployment/
-
-        - name: Diff
-          shell: |
-              set -ex;
-              for dir in openstack-helm openstack-helm-infra; do
-                path="{{ work_dir }}/../${dir}/"
-                if [ ! -d "${path}" ]; then continue; fi
-                echo "${dir} diff"
-                cd "${path}"; git diff; cd -;
-              done
-
-    - name: "creating directory for run artifacts"
-      file:
-        path: "/tmp/artifacts"
-        state: directory
-
-    - name: Run gate scripts
-      include_role:
-        name: "{{ ([item] | flatten | length == 1) | ternary('airship-run-script', 'airship-run-script-set') }}"
-      vars:
-        workload: "{{ [item] | flatten }}"
-      loop: "{{ gate_scripts }}"
-
-    - name: "Downloads artifacts to executor"
-      synchronize:
-        src: "/tmp/artifacts"
-        dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
-        mode: pull
-      ignore_errors: True
-...

tools/gate/playbooks/docker-image-build.yaml

@@ -62,6 +62,15 @@
             executable: pip3
       become: True
 
+    - name: Make images - verbosive
+      when: not publish
+      shell: |
+        make images
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+        executable: /bin/bash
+      become: True
+
     - name: Make images
       when: not publish
       block:
@@ -97,6 +106,7 @@
               IMAGE_PREFIX: "airshipit"
               IMAGE_TAG: "{{ item }}"
               DISTRO: "{{ distro }}"
+              DISTRO_ALIAS: "{{ distro }}"
               COMMIT: "{{ zuul.newrev | default('') }}"
               PUSH_IMAGE: "true"
           with_items: "{{ image_tags.stdout_lines }}"

tools/gate/playbooks/run-integration-tests.yaml

@@ -1,32 +0,0 @@
-# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
----
-- name: Run integration tests
-  hosts: primary
-  tasks:
-
-  - name: Execute deploy scripts
-    shell: |
-      set -ex
-
-      ./tools/deployment/002-build-charts.sh
-      ./tools/deployment/003-prepare-k8s.sh
-      ./tools/deployment/004-make-images.sh
-      ./tools/maas/100-cert-manager.sh
-      ./tools/maas/120-postgresql.sh
-      ./tools/maas/121-maas.sh
-
-    args:
-      chdir: "{{ zuul.project.src_dir }}"