Full-site deploy

Change-Id: I9a986aa1fb64e527f1e3b1d5077ccd60f563588b
2024-02-06 23:44:03 +00:00 · 2024-02-06 23:44:03 +00:00 · cbf713ff52
commit cbf713ff52
parent 01d80e18d8
37 changed files with 1613 additions and 15 deletions
--- a/.zuul.yaml
+++ b/.zuul.yaml
@ -25,7 +25,8 @@
        - treasuremap-airsloop-site-lint
        - treasuremap-aiab-site-lint
        - treasuremap-airskiff-deployment-ubuntu
-        # - treasuremap-airskiff-deployment-suse
+        - treasuremap-airskiff-multinode-full-site
+        - treasuremap-airskiff-1node-reduced-site
    gate:
      jobs:
        - treasuremap-seaworthy-site-lint
@ -56,6 +57,147 @@
        label: ubuntu-focal


+- nodeset:
+    name: treasuremap-airskiff-3nodes-ubuntu_focal
+    nodes:
+      - name: primary
+        label: ubuntu-focal
+      - name: node-1
+        label: ubuntu-focal
+      - name: node-2
+        label: ubuntu-focal
+    groups:
+      - name: primary
+        nodes:
+          - primary
+      - name: nodes
+        nodes:
+          - node-1
+          - node-2
+
+
+- nodeset:
+    name: treasuremap-airskiff-1node-ubuntu_focal
+    nodes:
+      - name: primary
+        label: ubuntu-focal
+    groups:
+      - name: primary
+        nodes:
+          - primary
+
+
+- job:
+    name: treasuremap-airskiff-infra-deploy
+    abstract: true
+    roles:
+      - zuul: openstack/openstack-helm-infra
+      - zuul: zuul/zuul-jobs
+    required-projects:
+      - openstack/openstack-helm
+      - openstack/openstack-helm-infra
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*$
+      - ^releasenotes/.*$
+      - ^site/seaworthy/.*$
+      - ^site/airsloop/.*$
+      - ^site/aiab/.*$
+    timeout: 10800
+    pre-run:
+      - tools/gate/playbooks/prepare-hosts.yaml
+      - tools/gate/playbooks/mount-volumes.yaml
+      - tools/gate/playbooks/deploy-env.yaml
+      - tools/gate/playbooks/git-config.yaml
+    run:
+      - tools/gate/playbooks/airship-run-scripts.yaml
+    post-run:
+      - tools/gate/playbooks/osh-infra-collect-logs.yaml
+    vars:
+      extra_volume:
+        size: 80G
+        type: Linux
+        mount_point: /opt/ext_vol
+      docker:
+        root_path: "/opt/ext_vol/docker"
+      containerd:
+        root_path: "/opt/ext_vol/containerd"
+      kubeadm:
+        pod_network_cidr: "10.244.0.0/24"
+        service_cidr: "10.96.0.0/16"
+      loopback_setup: true
+      loopback_device: /dev/loop100
+      loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img"
+      ceph_osd_data_device: /dev/loop100
+      kube_version: "1.28.4-1.1"
+      calico_version: "v3.27.0"
+      coredns_version: "v1.11.1"
+      helm_version: "v3.13.2"
+      yq_version: "v4.6.0"
+      crictl_version: "v1.26.1"
+      zuul_osh_infra_relative_path: ../openstack-helm-infra
+      gate_scripts_relative_path: ../openstack-helm-infra
+      run_helm_tests: "no"
+
+- job:
+    name: treasuremap-airskiff-multinode-full-site
+    parent: treasuremap-airskiff-infra-deploy
+    nodeset: treasuremap-airskiff-3nodes-ubuntu_focal
+    description: |
+      Deploy Openstack using Airskiff and latest Treasuremap changes.
+      Airskiff is using latest Airship v1.x based on Airflow-2.6.2
+    voting: true
+    vars:
+      site: airskiff
+      HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
+      HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+      OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+      OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
+      gate_scripts_relative_path: .
+      gate_scripts:
+        - ./tools/deployment/airskiff/developer/000-prepare-k8s.sh
+        - ./tools/deployment/airskiff/developer/009-setup-apparmor.sh
+        - ./tools/deployment/airskiff/developer/000-clone-dependencies.sh
+        - ./tools/deployment/airskiff/developer/020-setup-client.sh
+        - ./tools/deployment/airskiff/developer/015-make-all-charts.sh
+        - ./tools/deployment/airskiff/developer/025-start-artifactory.sh
+        - ./tools/deployment/airskiff/developer/026-reduce-site.sh
+        - ./tools/deployment/airskiff/developer/030-armada-bootstrap.sh
+        - ./tools/deployment/airskiff/developer/100-deploy-osh.sh
+        - ./tools/deployment/airskiff/common/os-env.sh
+        - ./tools/gate/wait-for-shipyard.sh
+        # - ./tools/deployment/airskiff/common/sleep.sh
+
+- job:
+    name: treasuremap-airskiff-1node-reduced-site
+    parent: treasuremap-airskiff-infra-deploy
+    nodeset: treasuremap-airskiff-1node-ubuntu_focal
+    description: |
+      Deploy Openstack using Airskiff and latest Treasuremap changes.
+      Airskiff is using latest Airship v1.x based on Airflow-2.6.2
+    voting: true
+    vars:
+      site: airskiff
+      HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
+      HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+      OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+      OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
+      gate_scripts_relative_path: .
+      gate_scripts:
+        - ./tools/deployment/airskiff/developer/000-prepare-k8s.sh
+        - ./tools/deployment/airskiff/developer/009-setup-apparmor.sh
+        - ./tools/deployment/airskiff/developer/000-clone-dependencies.sh
+        - ./tools/deployment/airskiff/developer/020-setup-client.sh
+        - ./tools/deployment/airskiff/developer/015-make-all-charts.sh
+        - ./tools/deployment/airskiff/developer/025-start-artifactory.sh
+        - ./tools/deployment/airskiff/developer/026-reduce-site.sh
+        - ./tools/deployment/airskiff/developer/030-armada-bootstrap.sh
+        - ./tools/deployment/airskiff/developer/100-deploy-osh.sh
+        - ./tools/deployment/airskiff/common/os-env.sh
+        - ./tools/gate/wait-for-shipyard.sh
+        # - ./tools/deployment/airskiff/common/sleep.sh
+
+
 - job:
    name: treasuremap-site-lint
    description:
@ -181,7 +323,7 @@
    post-run: tools/gate/playbooks/debug-report.yaml
    vars:
      site: airskiff
-      HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz
+      HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
      HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
      OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
      OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
--- a/tools/deployment/airskiff/common/sleep.sh
+++ b/tools/deployment/airskiff/common/sleep.sh
@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -ex
+
+while true; do sleep 10; done
--- a/tools/deployment/airskiff/developer/000-prepare-k8s.sh
+++ b/tools/deployment/airskiff/developer/000-prepare-k8s.sh
@ -0,0 +1,70 @@
+#!/bin/bash
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# Add labels to the core namespaces & nodes
+kubectl label --overwrite namespace default name=default
+kubectl label --overwrite namespace kube-system name=kube-system
+kubectl label --overwrite namespace kube-public name=kube-public
+kubectl label --overwrite nodes --all openstack-control-plane=enabled
+kubectl label --overwrite nodes --all openstack-compute-node=enabled
+kubectl label --overwrite nodes --all openvswitch=enabled
+kubectl label --overwrite nodes --all linuxbridge=enabled
+kubectl label --overwrite nodes --all ceph-mon=enabled
+kubectl label --overwrite nodes --all ceph-osd=enabled
+kubectl label --overwrite nodes --all ceph-mds=enabled
+kubectl label --overwrite nodes --all ceph-rgw=enabled
+kubectl label --overwrite nodes --all ceph-mgr=enabled
+# We deploy l3 agent only on the node where we run test scripts.
+# In this case virtual router will be created only on this node
+# and we don't need L2 overlay (will be implemented later).
+kubectl label --overwrite nodes -l "node-role.kubernetes.io/control-plane" l3-agent=enabled
+
+for NAMESPACE in ceph mariadb-operator openstack osh-infra; do
+tee /tmp/${NAMESPACE}-ns.yaml << EOF
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    kubernetes.io/metadata.name: ${NAMESPACE}
+    name: ${NAMESPACE}
+  name: ${NAMESPACE}
+EOF
+
+kubectl apply -f /tmp/${NAMESPACE}-ns.yaml
+done
+
+
+PATCH=$(mktemp)
+HOSTIP=$(hostname -I| awk '{print $1}')
+kubectl get configmap coredns -n kube-system -o json | jq -r "{data: .data}"  | sed 's/ready\\n/header \{\\n        response set ra\\n    \}\\n    ready\\n/g' > "${PATCH}"
+sed -i "s;forward . /etc/resolv.conf {\\\n       max_concurrent 1000\\\n    }\\\n;forward . /etc/resolv.conf {\\\n       max_concurrent 1000\\\n    }\\\n    hosts {\\\n       $HOSTIP control-plane.minikube.internal\\\n       fallthrough\\\n    }\\\n;" "${PATCH}"
+kubectl patch configmap coredns -n kube-system --patch-file "${PATCH}"
+kubectl set image deployment coredns -n kube-system "coredns=registry.k8s.io/coredns/coredns:${COREDNS_VERSION}"
+rm -f "${PATCH}"
+kubectl rollout restart -n kube-system deployment/coredns
+kubectl rollout status --watch --timeout=300s -n kube-system deployment/coredns
+sleep 10
+host -v control-plane.minikube.internal
+
+kubectl label nodes --all --overwrite ucp-control-plane=enabled
+
+
+kubectl run multitool --image=praqma/network-multitool
+kubectl wait --for=condition=ready pod multitool --timeout=300s
+kubectl exec -it multitool -- nslookup control-plane.minikube.internal
+kubectl exec -it multitool -- ping -c 4 8.8.8.8
+kubectl exec -it multitool -- nslookup google.com
+
+# make all
--- a/tools/deployment/airskiff/developer/015-make-all-charts.sh
+++ b/tools/deployment/airskiff/developer/015-make-all-charts.sh
@ -20,7 +20,7 @@ set -xe
 CURRENT_DIR="$(pwd)"
 : "${INSTALL_PATH:="../"}"
 : "${ARTIFACTS_PATH:="../artifacts"}"
-: "${HTK_COMMIT:="ae91cf3fc3f288b6d92ace4a3a405606a653638f"}"
+: "${HTK_COMMIT:="cfff60ec10a6c386f38db79bb9f59a552c2b032f"}"
 : "${MAKE_CHARTS_OPENSTACK_HELM:=true}"
 : "${MAKE_CHARTS_OSH_INFRA:=true}"
 : "${MAKE_CHARTS_ARMADA:=true}"
--- a/tools/deployment/airskiff/developer/025-start-artifactory.sh
+++ b/tools/deployment/airskiff/developer/025-start-artifactory.sh
@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -ex
+
+# start http server with artifacts
+docker rm artifacts --force || true
+docker run --name artifacts -p 8282:80 -v $(pwd)/../artifacts:/usr/share/nginx/html -d nginx
+sleep 10
+curl --verbose -I http://control-plane.minikube.internal:8282/memcached.tgz
--- a/tools/deployment/airskiff/developer/026-reduce-site.sh
+++ b/tools/deployment/airskiff/developer/026-reduce-site.sh
@ -0,0 +1,4 @@
+#!/bin/bash
+
+set -ex
+mv tools/gate/manifests/full-site.yaml  type/skiff/manifests/full-site.yaml
--- a/tools/gate/playbooks/airship-run-scripts.yaml
+++ b/tools/gate/playbooks/airship-run-scripts.yaml
@ -0,0 +1,75 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: primary
+  tasks:
+    - name: Override images
+      when: buildset_registry is defined
+      vars:
+        work_dir: "{{ zuul.project.src_dir }}"
+      block:
+        - name: Buildset registry alias
+          include_role:
+            name: deploy-env
+            tasks_from: buildset_registry_alias
+
+        - name: Print zuul
+          debug:
+            var: zuul
+
+        - name: Override proposed images from artifacts
+          shell: >
+            find {{ override_paths | join(" ") }} -type f -exec sed -Ei
+            "s#['\"]?docker\.io/({{ repo }}):({{ tag }})['\"]?\$#{{ buildset_registry_alias }}:{{ buildset_registry.port }}/\1:\2#g" {} +
+          loop: "{{ zuul.artifacts | default([]) }}"
+          args:
+            chdir: "{{ work_dir }}"
+          loop_control:
+            loop_var: zj_zuul_artifact
+          when: "'metadata' in zj_zuul_artifact and zj_zuul_artifact.metadata.type | default('') == 'container_image'"
+          vars:
+            tag: "{{ zj_zuul_artifact.metadata.tag }}"
+            repo: "{{ zj_zuul_artifact.metadata.repository }}"
+            override_paths:
+              - ../openstack-helm*/*/values*
+              - ../openstack-helm-infra/tools/deployment/
+
+        - name: Diff
+          shell: |
+              set -ex;
+              for dir in openstack-helm openstack-helm-infra; do
+                path="{{ work_dir }}/../${dir}/"
+                if [ ! -d "${path}" ]; then continue; fi
+                echo "${dir} diff"
+                cd "${path}"; git diff; cd -;
+              done
+
+    - name: "creating directory for run artifacts"
+      file:
+        path: "/tmp/artifacts"
+        state: directory
+
+    - name: Run gate scripts
+      include_role:
+        name: "{{ ([item] | flatten | length == 1) | ternary('airship-run-script', 'airship-run-script-set') }}"
+      vars:
+        workload: "{{ [item] | flatten }}"
+      loop: "{{ gate_scripts }}"
+
+    - name: "Downloads artifacts to executor"
+      synchronize:
+        src: "/tmp/artifacts"
+        dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
+        mode: pull
+      ignore_errors: True
+...
--- a/tools/gate/playbooks/airskiff-deploy-gate.yaml
+++ b/tools/gate/playbooks/airskiff-deploy-gate.yaml
@ -55,7 +55,7 @@
        ./tools/deployment/airskiff/developer/020-setup-client.sh
      args:
        chdir: "{{ zuul.project.src_dir }}"
-      become: yes
+      become: true

    - name: Build all charts locally
      shell: |
@ -67,7 +67,7 @@
        ./tools/deployment/airskiff/developer/015-make-all-charts.sh
      args:
        chdir: "{{ zuul.project.src_dir }}"
-      become: yes
+      become: true

    - name: Start artifactory
      shell: |
@ -79,7 +79,7 @@
        curl --verbose -I http://control-plane.minikube.internal:8282/memcached.tgz
      args:
        chdir: "{{ zuul.project.src_dir }}"
-      become: yes
+      become: true

    - name: Deploy Airship components using Armada
      shell: |
@ -90,7 +90,7 @@
        ./tools/deployment/airskiff/developer/030-armada-bootstrap.sh
      args:
        chdir: "{{ zuul.project.src_dir }}"
-      become: yes
+      become: true

    - name: Deploy Software using Airship
      shell: |
@ -99,7 +99,7 @@
        ./tools/deployment/airskiff/developer/100-deploy-osh.sh
      args:
        chdir: "{{ zuul.project.src_dir }}"
-      become: yes
+      become: true

    - name: Wait for deployment completion
      shell: |
@ -108,7 +108,7 @@
        ./tools/gate/wait-for-shipyard.sh
      args:
        chdir: "{{ zuul.project.src_dir }}"
-      become: yes
+      become: true

    - name: Stop artifactory
      shell: |
@ -117,4 +117,4 @@
        docker rm artifacts --force || true
      args:
        chdir: "{{ zuul.project.src_dir }}"
-      become: yes
+      become: true
--- a/tools/gate/playbooks/deploy-env.yaml
+++ b/tools/gate/playbooks/deploy-env.yaml
@ -0,0 +1,38 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: all
+  become: true
+  gather_facts: true
+
+  roles:
+    - role: add-authorized-keys
+      public_keys:
+        - public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDA7eM8WFJrqQmki8rR0O3QBHyl8xq42jb1RduwuRwjWoGYJI5cX7Fx+7VR4A9ITCoiqxKS8DMfgKbt5jKC6SmvMALULZsnYlthB34KywurgxsW6fgp68DHWQ7J4CCBhoIpl0W3JW7s6b0vHLhab59r0E+AYemBVuWUqbFEy8nDAHcQv1S/2o1udhmljIN7c2ogO4KAJ7Lge0BoIP9ps4u6AVwyQZixp4anU9DHGNA/UQj4M5UyuALj5buEAuATBe9Vqj4sOvZjObPJAGPUrNRrGEWAFk+lSZHRzKXo0eeWtPqoh5UN9UDb5Pocg1krncMIZwjHKovlD1z/O1y91aY5LM1wxm/7aaIiX8eCihyVZaOuDCLF7WDT2SMs7ABcotX2MDtVQTrNNV3MmMAScFNDflzPKszd7cdjLl6PBq8bvPxmCkLmnitPTGOoh9d8i+JlbINvgx1pguYrpeciIyreCO1rjTW3MgB0tyoMEa31V+7HrauBMeNnE68YTqLTIB0= smarkin@mirantis.com
+    - ensure-python
+    - ensure-pip
+    - ensure-tox
+    - clear-firewall
+    - deploy-apparmor
+    - deploy-selenium
+    - airship-deploy-env
+  tasks:
+    - name: Install Packaging python module for tools/airship
+      block:
+        - pip:
+            name: packaging
+            version: 23.1
+            executable: pip3
+      become: True
+
+...
--- a/tools/gate/playbooks/generate-certs.yaml
+++ b/tools/gate/playbooks/generate-certs.yaml
@ -28,4 +28,4 @@
          /target/collected/treasuremap.yaml
      args:
        chdir: "{{ zuul.project.src_dir }}"
-      become: yes
+      become: true
--- a/tools/gate/playbooks/mount-volumes.yaml
+++ b/tools/gate/playbooks/mount-volumes.yaml
@ -0,0 +1,17 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: all
+  roles:
+    - mount-extra-volume
+...
--- a/tools/gate/playbooks/osh-infra-collect-logs.yaml
+++ b/tools/gate/playbooks/osh-infra-collect-logs.yaml
@ -0,0 +1,43 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: all
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+    logs_dir: "/tmp/logs"
+  roles:
+    - gather-host-logs
+  tags:
+    - gather-host-logs
+
+- hosts: primary
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+    logs_dir: "/tmp/logs"
+  roles:
+    - helm-release-status
+    - describe-kubernetes-objects
+    - gather-pod-logs
+    - gather-prom-metrics
+    - gather-selenium-data
+  tags:
+    - helm-release-status
+    - describe-kubernetes-objects
+    - gather-pod-logs
+    - gather-prom-metrics
+    - gather-selenium-data
+...
--- a/tools/gate/playbooks/prepare-hosts.yaml
+++ b/tools/gate/playbooks/prepare-hosts.yaml
@ -0,0 +1,17 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- hosts: all
+  roles:
+    - start-zuul-console
+...
--- a/tools/gate/playbooks/roles/airship-deploy-env/README.md
+++ b/tools/gate/playbooks/roles/airship-deploy-env/README.md
@ -0,0 +1,37 @@
+This role is used to deploy test environment which includes
+- install necessary prerequisites including Helm
+- deploy Containerd and a container runtime for Kubernetes
+- deploy Kubernetes using Kubeadm with a single control plain node
+- install Calico as a Kubernetes networking
+
+The role works both for singlenode and multinode inventories and
+assumes the inventory has the node called `primary` and the group called `nodes`.
+
+See for example:
+
+```yaml
+all:
+  children:
+    ungrouped:
+      hosts:
+        primary:
+          ansible_port: 22
+          ansible_host: 10.10.10.10
+          ansible_user: ubuntu
+          ansible_ssh_private_key_file: /home/ubuntu/.ssh/id_rsa
+          ansible_ssh_extra_args: -o StrictHostKeyChecking=no
+    nodes:
+      hosts:
+        node-1:
+          ansible_port: 22
+          ansible_host: 10.10.10.11
+          ansible_user: ubuntu
+          ansible_ssh_private_key_file: /home/ubuntu/.ssh/id_rsa
+          ansible_ssh_extra_args: -o StrictHostKeyChecking=no
+        node-2:
+          ansible_port: 22
+          ansible_host: 10.10.10.12
+          ansible_user: ubuntu
+          ansible_ssh_private_key_file: /home/ubuntu/.ssh/id_rsa
+          ansible_ssh_extra_args: -o StrictHostKeyChecking=no
+```
--- a/tools/gate/playbooks/roles/airship-deploy-env/defaults/main.yaml
+++ b/tools/gate/playbooks/roles/airship-deploy-env/defaults/main.yaml
@ -0,0 +1,38 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+kube_version_repo: "v1.28"
+# the list of k8s package versions are available here
+# https://pkgs.k8s.io/core:/stable:/{{ kube_version_repo }}/deb/Packages
+kube_version: "1.28.4-1.1"
+calico_version: "v3.27.0"
+calico_manifest_url: "https://raw.githubusercontent.com/projectcalico/calico/{{ calico_version }}/manifests/calico.yaml"
+helm_version: "v3.6.3"
+crictl_version: "v1.26.1"
+
+kubectl:
+  user: zuul
+  group: zuul
+kubeadm:
+  pod_network_cidr: "10.244.0.0/24"
+  service_cidr: "10.96.0.0/16"
+docker:
+  root_path: /var/lib/docker
+docker_users:
+  - zuul
+containerd:
+  root_path: /var/lib/containerd
+loopback_setup: false
+loopback_device: /dev/loop100
+loopback_image: /var/lib/openstack-helm/ceph-loop.img
+loopback_image_size: 12G
+...
--- a/tools/gate/playbooks/roles/airship-deploy-env/files/calico_patch.yaml
+++ b/tools/gate/playbooks/roles/airship-deploy-env/files/calico_patch.yaml
@ -0,0 +1,23 @@
+---
+spec:
+  template:
+    metadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9091"
+    spec:
+      containers:
+        - name: calico-node
+          env:
+            - name: FELIX_PROMETHEUSMETRICSENABLED
+              value: "true"
+            - name: FELIX_PROMETHEUSMETRICSPORT
+              value: "9091"
+            - name: FELIX_IGNORELOOSERPF
+              value: "true"
+            # We assign IP on br-ex interface while testing the deployed Openstack cluster and
+            # we need Calico to skip this interface while discovering the
+            # network changes on the host to prevent announcing unnecessary networks.
+            - name: IP_AUTODETECTION_METHOD
+              value: "skip-interface=br-ex"
+...
--- a/tools/gate/playbooks/roles/airship-deploy-env/files/containerd_config.toml
+++ b/tools/gate/playbooks/roles/airship-deploy-env/files/containerd_config.toml
@ -0,0 +1,256 @@
+disabled_plugins = []
+imports = []
+oom_score = 0
+plugin_dir = ""
+required_plugins = []
+root = "{{ containerd.root_path }}"
+state = "/run/containerd"
+temp = ""
+version = 2
+
+[cgroup]
+  path = ""
+
+[debug]
+  address = ""
+  format = ""
+  gid = 0
+  level = ""
+  uid = 0
+
+[grpc]
+  address = "/run/containerd/containerd.sock"
+  gid = 0
+  max_recv_message_size = 16777216
+  max_send_message_size = 16777216
+  tcp_address = ""
+  tcp_tls_ca = ""
+  tcp_tls_cert = ""
+  tcp_tls_key = ""
+  uid = 0
+
+[metrics]
+  address = ""
+  grpc_histogram = false
+
+[plugins]
+
+  [plugins."io.containerd.gc.v1.scheduler"]
+    deletion_threshold = 0
+    mutation_threshold = 100
+    pause_threshold = 0.02
+    schedule_delay = "0s"
+    startup_delay = "100ms"
+
+  [plugins."io.containerd.grpc.v1.cri"]
+    device_ownership_from_security_context = false
+    disable_apparmor = false
+    disable_cgroup = false
+    disable_hugetlb_controller = true
+    disable_proc_mount = false
+    disable_tcp_service = true
+    enable_selinux = false
+    enable_tls_streaming = false
+    enable_unprivileged_icmp = false
+    enable_unprivileged_ports = false
+    ignore_image_defined_volumes = false
+    max_concurrent_downloads = 3
+    max_container_log_line_size = 16384
+    netns_mounts_under_state_dir = false
+    restrict_oom_score_adj = false
+    sandbox_image = "registry.k8s.io/pause:3.6"
+    selinux_category_range = 1024
+    stats_collect_period = 10
+    stream_idle_timeout = "4h0m0s"
+    stream_server_address = "127.0.0.1"
+    stream_server_port = "0"
+    systemd_cgroup = false
+    tolerate_missing_hugetlb_controller = true
+    unset_seccomp_profile = ""
+
+    [plugins."io.containerd.grpc.v1.cri".cni]
+      bin_dir = "/opt/cni/bin"
+      conf_dir = "/etc/cni/net.d"
+      conf_template = ""
+      ip_pref = ""
+      max_conf_num = 1
+
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "runc"
+      disable_snapshot_annotations = true
+      discard_unpacked_layers = false
+      ignore_rdt_not_enabled_errors = false
+      no_pivot = false
+      snapshotter = "overlayfs"
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
+        base_runtime_spec = ""
+        cni_conf_dir = ""
+        cni_max_conf_num = 0
+        container_annotations = []
+        pod_annotations = []
+        privileged_without_host_devices = false
+        runtime_engine = ""
+        runtime_path = ""
+        runtime_root = ""
+        runtime_type = ""
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options]
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          base_runtime_spec = ""
+          cni_conf_dir = ""
+          cni_max_conf_num = 0
+          container_annotations = []
+          pod_annotations = []
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_path = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+            BinaryName = ""
+            CriuImagePath = ""
+            CriuPath = ""
+            CriuWorkPath = ""
+            IoGid = 0
+            IoUid = 0
+            NoNewKeyring = false
+            NoPivotRoot = false
+            Root = ""
+            ShimCgroup = ""
+            SystemdCgroup = true
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
+        base_runtime_spec = ""
+        cni_conf_dir = ""
+        cni_max_conf_num = 0
+        container_annotations = []
+        pod_annotations = []
+        privileged_without_host_devices = false
+        runtime_engine = ""
+        runtime_path = ""
+        runtime_root = ""
+        runtime_type = ""
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options]
+
+    [plugins."io.containerd.grpc.v1.cri".image_decryption]
+      key_model = "node"
+
+    [plugins."io.containerd.grpc.v1.cri".registry]
+      config_path = "/etc/containerd/certs.d"
+
+      [plugins."io.containerd.grpc.v1.cri".registry.auths]
+
+      [plugins."io.containerd.grpc.v1.cri".registry.configs]
+{% for item in registry_namespaces %}
+{% if item.auth is defined %}
+        [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ item.namespace }}".auth]
+          auth = "{{ item.auth }}"
+{% endif %}
+{% endfor %}
+
+      [plugins."io.containerd.grpc.v1.cri".registry.headers]
+
+      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+
+    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
+      tls_cert_file = ""
+      tls_key_file = ""
+
+  [plugins."io.containerd.internal.v1.opt"]
+    path = "/opt/containerd"
+
+  [plugins."io.containerd.internal.v1.restart"]
+    interval = "10s"
+
+  [plugins."io.containerd.internal.v1.tracing"]
+    sampling_ratio = 1.0
+    service_name = "containerd"
+
+  [plugins."io.containerd.metadata.v1.bolt"]
+    content_sharing_policy = "shared"
+
+  [plugins."io.containerd.monitor.v1.cgroups"]
+    no_prometheus = false
+
+  [plugins."io.containerd.runtime.v1.linux"]
+    no_shim = false
+    runtime = "runc"
+    runtime_root = ""
+    shim = "containerd-shim"
+    shim_debug = false
+
+  [plugins."io.containerd.runtime.v2.task"]
+    platforms = ["linux/amd64"]
+    sched_core = false
+
+  [plugins."io.containerd.service.v1.diff-service"]
+    default = ["walking"]
+
+  [plugins."io.containerd.service.v1.tasks-service"]
+    rdt_config_file = ""
+
+  [plugins."io.containerd.snapshotter.v1.aufs"]
+    root_path = ""
+
+  [plugins."io.containerd.snapshotter.v1.btrfs"]
+    root_path = ""
+
+  [plugins."io.containerd.snapshotter.v1.devmapper"]
+    async_remove = false
+    base_image_size = ""
+    discard_blocks = false
+    fs_options = ""
+    fs_type = ""
+    pool_name = ""
+    root_path = ""
+
+  [plugins."io.containerd.snapshotter.v1.native"]
+    root_path = ""
+
+  [plugins."io.containerd.snapshotter.v1.overlayfs"]
+    root_path = ""
+    upperdir_label = false
+
+  [plugins."io.containerd.snapshotter.v1.zfs"]
+    root_path = ""
+
+  [plugins."io.containerd.tracing.processor.v1.otlp"]
+    endpoint = ""
+    insecure = false
+    protocol = ""
+
+[proxy_plugins]
+
+[stream_processors]
+
+  [stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]
+    accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]
+    args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
+    env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
+    path = "ctd-decoder"
+    returns = "application/vnd.oci.image.layer.v1.tar"
+
+  [stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]
+    accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"]
+    args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
+    env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
+    path = "ctd-decoder"
+    returns = "application/vnd.oci.image.layer.v1.tar+gzip"
+
+[timeouts]
+  "io.containerd.timeout.bolt.open" = "0s"
+  "io.containerd.timeout.shim.cleanup" = "5s"
+  "io.containerd.timeout.shim.load" = "5s"
+  "io.containerd.timeout.shim.shutdown" = "3s"
+  "io.containerd.timeout.task.state" = "2s"
+
+[ttrpc]
+  address = ""
+  gid = 0
+  uid = 0
--- a/tools/gate/playbooks/roles/airship-deploy-env/files/daemon.json
+++ b/tools/gate/playbooks/roles/airship-deploy-env/files/daemon.json
@ -0,0 +1,10 @@
+{
+    "data-root": "{{ docker.root_path }}",
+    "exec-opts": ["native.cgroupdriver=systemd"],
+    "log-driver": "json-file",
+    "log-opts": {
+        "max-size": "100m"
+    },
+    "storage-driver": "overlay2",
+    "live-restore": true
+}
--- a/tools/gate/playbooks/roles/airship-deploy-env/files/hosts
+++ b/tools/gate/playbooks/roles/airship-deploy-env/files/hosts
@ -0,0 +1,5 @@
+127.0.0.1 localhost
+{{ ansible_default_ipv4['address'] }} {{ ansible_hostname }}
+{% if buildset_registry is defined and (buildset_registry.host | ipaddr) %}
+{{ buildset_registry.host }} zuul-jobs.buildset-registry
+{% endif %}
--- a/tools/gate/playbooks/roles/airship-deploy-env/files/hosts.toml
+++ b/tools/gate/playbooks/roles/airship-deploy-env/files/hosts.toml
@ -0,0 +1,12 @@
+{% if item.skip_server is not defined or not item.skip_server %}
+server = "{{ item.server | default('https://' + item.namespace) }}"
+{% endif %}
+
+[host."{{ item.mirror }}"]
+capabilities = ["pull", "resolve", "push"]
+{% if item.ca is defined %}
+ca = "{{ item.ca }}"
+{% endif %}
+{% if item.skip_verify is defined and item.skip_verify %}
+skip_verify = true
+{% endif %}
--- a/tools/gate/playbooks/roles/airship-deploy-env/files/kubeadm_config.yaml
+++ b/tools/gate/playbooks/roles/airship-deploy-env/files/kubeadm_config.yaml
@ -0,0 +1,13 @@
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+mode: ipvs
+...
+---
+apiVersion: kubeadm.k8s.io/v1beta3
+kind: ClusterConfiguration
+networking:
+  serviceSubnet: "{{ kubeadm.service_cidr }}"  # --service-cidr
+  podSubnet: "{{ kubeadm.pod_network_cidr }}"  # --pod-network-cidr
+  dnsDomain: "cluster.local"
+...
--- a/tools/gate/playbooks/roles/airship-deploy-env/files/loop-setup.service
+++ b/tools/gate/playbooks/roles/airship-deploy-env/files/loop-setup.service
@ -0,0 +1,18 @@
+[Unit]
+Description=Setup loop devices
+DefaultDependencies=no
+Conflicts=umount.target
+Before=local-fs.target
+After=systemd-udevd.service
+Requires=systemd-udevd.service
+
+[Service]
+Type=oneshot
+ExecStart=/sbin/losetup {{ loopback_device }} '{{ loopback_image }}'
+ExecStop=/sbin/losetup -d {{ loopback_device }}
+TimeoutSec=60
+RemainAfterExit=yes
+
+[Install]
+WantedBy=local-fs.target
+Also=systemd-udevd.service
--- a/tools/gate/playbooks/roles/airship-deploy-env/files/resolv.conf
+++ b/tools/gate/playbooks/roles/airship-deploy-env/files/resolv.conf
@ -0,0 +1,4 @@
+nameserver 8.8.8.8
+nameserver 8.8.4.4
+search svc.cluster.local cluster.local
+options ndots:5 timeout:1 attempts:1
--- a/tools/gate/playbooks/roles/airship-deploy-env/handlers/main.yaml
+++ b/tools/gate/playbooks/roles/airship-deploy-env/handlers/main.yaml
@ -0,0 +1,21 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: Systemd reload
+  shell: systemctl daemon-reload
+
+- name: Restart loop-setup
+  service:
+    name: loop-setup
+    state: restarted
+...
--- a/tools/gate/playbooks/roles/airship-deploy-env/tasks/buildset_registry_alias.yaml
+++ b/tools/gate/playbooks/roles/airship-deploy-env/tasks/buildset_registry_alias.yaml
@ -0,0 +1,25 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: Set buildset_registry alias variable when using ip
+  set_fact:
+    buildset_registry_alias: zuul-jobs.buildset-registry
+  when:
+    - buildset_registry.host | ipaddr
+
+- name: Set buildset_registry alias variable when using name
+  set_fact:
+    buildset_registry_alias: "{{ buildset_registry.host }}"
+  when:
+    - not ( buildset_registry.host | ipaddr )
+...
--- a/tools/gate/playbooks/roles/airship-deploy-env/tasks/common_k8s.yaml
+++ b/tools/gate/playbooks/roles/airship-deploy-env/tasks/common_k8s.yaml
@ -0,0 +1,128 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: Load necessary modules
+  modprobe:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - overlay
+    - br_netfilter
+
+- name: Configure sysctl
+  sysctl:
+    name: "{{ item }}"
+    value: "1"
+    state: present
+  loop:
+    - net.ipv6.conf.default.disable_ipv6
+    - net.ipv6.conf.all.disable_ipv6
+    - net.ipv6.conf.lo.disable_ipv6
+    - net.bridge.bridge-nf-call-iptables
+    - net.bridge.bridge-nf-call-ip6tables
+    - net.ipv4.ip_forward
+  ignore_errors: true
+
+# This is necessary when we run dnsmasq.
+# Otherwise, we get the error:
+# failed to create inotify: Too many open files
+- name: Configure number of inotify instances
+  sysctl:
+    name: "fs.inotify.max_user_instances"
+    value: "256"
+    state: present
+  ignore_errors: true
+
+- name: Remove swapfile from /etc/fstab
+  mount:
+    name: "{{ item }}"
+    fstype: swap
+    state: absent
+  with_items:
+    - swap
+    - none
+
+- name: Disable swap
+  command: swapoff -a
+  when: ansible_swaptotal_mb > 0
+
+- name: Ensure dependencies are installed
+  apt:
+    name:
+      - apt-transport-https
+      - ca-certificates
+      - gnupg2
+      - ipvsadm
+      - jq
+    state: present
+
+- name: Add Kubernetes apt repository key
+  apt_key:
+    url: "https://pkgs.k8s.io/core:/stable:/{{ kube_version_repo }}/deb/Release.key"
+    state: present
+
+- name: Add Kubernetes apt repository
+  apt_repository:
+    repo: "deb https://pkgs.k8s.io/core:/stable:/{{ kube_version_repo }}/deb/ /"
+    state: present
+    filename: kubernetes.list
+
+- name: Install Kubernetes binaries
+  apt:
+    state: present
+    update_cache: true
+    allow_downgrade: true
+    pkg:
+      - "kubelet={{ kube_version }}"
+      - "kubeadm={{ kube_version }}"
+      - "kubectl={{ kube_version }}"
+
+- name: Restart kubelet
+  service:
+    name: kubelet
+    daemon_reload: yes
+    state: restarted
+
+- name: Disable systemd-resolved
+  service:
+    name: systemd-resolved
+    enabled: false
+    state: stopped
+
+- name: Configure resolv.conf
+  copy:
+    src: files/resolv.conf
+    dest: "{{ item }}"
+  loop:
+    - /etc/resolv.conf
+    - /run/systemd/resolve/resolv.conf
+
+# We download Calico manifest on all nodes because we then want to download
+# Calico images BEFORE deploying it
+- name: Download Calico manifest
+  shell: |
+    curl -LSs {{ calico_manifest_url }} -o /tmp/calico.yaml
+    sed -i -e 's#docker.io/calico/#quay.io/calico/#g' /tmp/calico.yaml
+  args:
+    executable: /bin/bash
+
+# Download images needed for calico before applying manifests, so that `kubectl wait` timeout
+# for `k8s-app=kube-dns` isn't reached by slow download speeds
+- name: Download Calico images
+  shell: |
+    export CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock
+    export IMAGE_SERVICE_ENDPOINT=unix:///run/containerd/containerd.sock
+    awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} crictl pull {}
+  args:
+    executable: /bin/bash
+...
--- a/tools/gate/playbooks/roles/airship-deploy-env/tasks/containerd.yaml
+++ b/tools/gate/playbooks/roles/airship-deploy-env/tasks/containerd.yaml
@ -0,0 +1,167 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: Remove old docker packages
+  apt:
+    pkg:
+      - docker.io
+      - docker-doc
+      - docker-compose
+      - podman-docker
+      - containerd
+      - runc
+    state: absent
+
+- name: Ensure dependencies are installed
+  apt:
+    name:
+      - apt-transport-https
+      - ca-certificates
+      - gnupg2
+    state: present
+
+- name: Add Docker apt repository key
+  apt_key:
+    url: https://download.docker.com/linux/ubuntu/gpg
+    keyring: /etc/apt/trusted.gpg.d/docker.gpg
+    state: present
+
+- name: Get dpkg arch
+  command: dpkg --print-architecture
+  register: dpkg_architecture
+
+- name: Add Docker apt repository
+  apt_repository:
+    repo: deb [arch="{{ dpkg_architecture.stdout }}" signed-by=/etc/apt/trusted.gpg.d/docker.gpg] https://download.docker.com/linux/ubuntu "{{ ansible_distribution_release }}" stable
+    state: present
+    filename: docker.list
+
+- name: Install docker packages
+  apt:
+    pkg:
+      - docker-ce
+      - docker-ce-cli
+      - containerd.io
+      - docker-buildx-plugin
+      - docker-compose-plugin
+    state: present
+    update_cache: true
+
+- name: Install Crictl
+  shell: |
+    wget https://github.com/kubernetes-sigs/cri-tools/releases/download/{{crictl_version}}/crictl-{{crictl_version}}-linux-amd64.tar.gz
+    sudo tar zxvf crictl-{{crictl_version}}-linux-amd64.tar.gz -C /usr/local/bin
+    rm -f crictl-{{crictl_version}}-linux-amd64.tar.gz
+  args:
+    executable: /bin/bash
+
+- name: Configure Docker daemon
+  template:
+    src: files/daemon.json
+    dest: /etc/docker/daemon.json
+
+- name: Add users to docker group
+  command: "adduser {{ item }} docker"
+  loop: "{{ docker_users }}"
+
+- name: Restart docker
+  service:
+    name: docker
+    daemon_reload: yes
+    state: restarted
+
+- name: Reset ssh connection to apply user changes.
+  meta: reset_connection
+
+- name: Set mirror_fqdn fact
+  when:
+    - registry_mirror is not defined
+    - zuul_site_mirror_fqdn is defined
+  set_fact:
+    registry_mirror: "http://{{ zuul_site_mirror_fqdn }}:8082"
+
+- name: Set regitstry namespaces
+  set_fact:
+    registry_namespaces:
+      - namespace: "_default"
+        mirror: "{{ registry_mirror }}"
+        skip_server: true
+        skip_verify: true
+  when: registry_mirror is defined
+
+- name: Init registry_namespaces if not defined
+  set_fact:
+    registry_namespaces: "[]"
+  when: not registry_namespaces is defined
+
+- name: Buildset registry namespace
+  when: buildset_registry is defined
+  block:
+    - name: Buildset registry alias
+      include_tasks:
+        file: buildset_registry_alias.yaml
+
+    - name: Write buildset registry TLS certificate
+      copy:
+        content: "{{ buildset_registry.cert }}"
+        dest: "/usr/local/share/ca-certificates/{{ buildset_registry_alias }}.crt"
+        mode: 0644
+      register: buildset_registry_tls_ca
+
+    - name: Update CA certs
+      command: "update-ca-certificates"
+      when: buildset_registry_tls_ca is changed
+
+    - name: Set buildset registry namespace
+      set_fact:
+        buildset_registry_namespace:
+          namespace: '{{ buildset_registry_alias }}:{{ buildset_registry.port }}'
+          mirror: 'https://{{ buildset_registry_alias }}:{{ buildset_registry.port }}'
+          ca: "/usr/local/share/ca-certificates/{{ buildset_registry_alias }}.crt"
+          auth: "{{ (buildset_registry.username + ':' + buildset_registry.password) | b64encode }}"
+
+    - name: Append buildset_registry to registry namespaces
+      when:
+        - buildset_registry_namespace is defined
+        - registry_namespaces is defined
+      set_fact:
+        registry_namespaces: "{{ registry_namespaces + [ buildset_registry_namespace ] }}"
+
+- name: Configure containerd
+  template:
+    src: files/containerd_config.toml
+    dest: /etc/containerd/config.toml
+
+- name: Create containerd config directory hierarchy
+  file:
+    state: directory
+    path: /etc/containerd/certs.d
+
+- name: Create host namespace directory
+  file:
+    state: directory
+    path: "/etc/containerd/certs.d/{{ item.namespace }}"
+  loop: "{{ registry_namespaces }}"
+
+- name: Create hosts.toml file
+  template:
+    src: files/hosts.toml
+    dest: "/etc/containerd/certs.d/{{ item.namespace }}/hosts.toml"
+  loop: "{{ registry_namespaces }}"
+
+- name: Restart containerd
+  service:
+    name: containerd
+    daemon_reload: yes
+    state: restarted
+...
--- a/tools/gate/playbooks/roles/airship-deploy-env/tasks/control-plane.yaml
+++ b/tools/gate/playbooks/roles/airship-deploy-env/tasks/control-plane.yaml
@ -0,0 +1,84 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: Mount tmpfs to /var/lib/etcd
+  mount:
+    path: /var/lib/etcd
+    src: tmpfs
+    fstype: tmpfs
+    opts: size=1g
+    state: mounted
+
+- name: Prepare kubeadm config
+  template:
+    src: files/kubeadm_config.yaml
+    dest: /tmp/kubeadm_config.yaml
+
+- name: Initialize the Kubernetes cluster using kubeadm
+  command: kubeadm init --config /tmp/kubeadm_config.yaml
+
+- name: "Setup kubeconfig for {{ kubectl.user }} user"
+  shell: |
+    mkdir -p /home/{{ kubectl.user }}/.kube
+    cp -i /etc/kubernetes/admin.conf /home/{{ kubectl.user }}/.kube/config
+    chown -R {{ kubectl.user }}:{{ kubectl.group }} /home/{{ kubectl.user }}/.kube
+  args:
+    executable: /bin/bash
+
+- name: Deploy Calico
+  become: false
+  command: kubectl apply -f /tmp/calico.yaml
+
+- name: Sleep before trying to check Calico pods
+  pause:
+    seconds: 20
+
+- name: Wait for Calico pods ready
+  become: false
+  command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=calico-node
+
+- name: Prepare Calico patch
+  copy:
+    src: files/calico_patch.yaml
+    dest: /tmp/calico_patch.yaml
+
+- name: Patch Calico
+  become: false
+  command: kubectl -n kube-system patch daemonset calico-node --patch-file /tmp/calico_patch.yaml
+
+- name: Wait for Calico pods ready
+  become: false
+  command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=calico-node
+
+- name: Generate join command
+  command: kubeadm token create --print-join-command
+  register: join_command
+
+- name: Untaint Kubernetes control plane node
+  become: false
+  command: kubectl taint nodes -l 'node-role.kubernetes.io/control-plane' node-role.kubernetes.io/control-plane-
+
+- name: Enable recursive queries for coredns
+  become: false
+  shell: |
+    PATCH=$(mktemp)
+    kubectl get configmap coredns -n kube-system -o json | jq -r "{data: .data}"  | sed 's/ready\\n/header \{\\n        response set ra\\n    \}\\n    ready\\n/g' > "${PATCH}"
+    kubectl patch configmap coredns -n kube-system --patch-file "${PATCH}"
+    kubectl set image deployment coredns -n kube-system "coredns=registry.k8s.io/coredns/coredns:v1.9.4"
+    kubectl rollout restart -n kube-system deployment/coredns
+    sleep 10
+    kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=kube-dns
+    rm -f "${PATCH}"
+  args:
+    executable: /bin/bash
+...
--- a/tools/gate/playbooks/roles/airship-deploy-env/tasks/loopback_devices.yaml
+++ b/tools/gate/playbooks/roles/airship-deploy-env/tasks/loopback_devices.yaml
@ -0,0 +1,45 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: Create loop device image
+  shell: |
+    mkdir -p {{ loopback_image | dirname }}
+    truncate -s {{ loopback_image_size }} {{ loopback_image }}
+
+- name: Create loop device
+  shell: |
+    mknod {{ loopback_device }} b $(grep loop /proc/devices | cut -c3) {{ loopback_device | regex_search('[0-9]+') }}
+
+- name: Create loop-setup systemd unit
+  template:
+    src: files/loop-setup.service
+    dest: /etc/systemd/system/loop-setup.service
+  notify:
+    - Systemd reload
+
+- name: Systemd reload
+  shell: systemctl daemon-reload
+
+- name: Configure loop-setup systemd unit
+  service:
+    name: loop-setup
+    enabled: yes
+    state: started
+  notify:
+    - Systemd reload
+    - Restart loop-setup
+
+- name: Check {{ loopback_device }} is attached
+  shell: |
+    losetup | grep -i {{ loopback_device }}
+...
--- a/tools/gate/playbooks/roles/airship-deploy-env/tasks/main.yaml
+++ b/tools/gate/playbooks/roles/airship-deploy-env/tasks/main.yaml
@ -0,0 +1,61 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: Include prerequisites tasks
+  include_tasks:
+    file: prerequisites.yaml
+
+- name: Configure /etc/hosts
+  template:
+    src: files/hosts
+    dest: /etc/hosts
+
+- name: Deploy Containerd
+  include_tasks:
+    file: containerd.yaml
+
+- name: Common K8s tasks
+  include_tasks:
+    file: common_k8s.yaml
+
+- name: Include control-plane tasks
+  include_tasks:
+    file: control-plane.yaml
+  when: inventory_hostname == 'primary'
+
+- name: Join workload nodes to cluster
+  command: "{{ hostvars['primary']['join_command'].stdout_lines[0] }}"
+  when: inventory_hostname in (groups['nodes'] | default([]))
+
+- name: Wait for cluster is ready
+  become: false
+  block:
+    - name: Sleep 10 before checking calico nodes
+      pause:
+        seconds: 10
+    - name: Wait for Calico pods ready
+      command: kubectl -n kube-system wait --timeout=240s --for=condition=Ready pods -l k8s-app=calico-node
+  when: inventory_hostname == 'primary'
+
+- name: Add coredns to /etc/resolv.conf
+  lineinfile:
+    line: nameserver 10.96.0.10
+    path: /etc/resolv.conf
+    state: present
+    insertbefore: "BOF"
+
+- name: Loop devices
+  include_tasks:
+    file: loopback_devices.yaml
+  when: loopback_setup
+...
--- a/tools/gate/playbooks/roles/airship-deploy-env/tasks/prerequisites.yaml
+++ b/tools/gate/playbooks/roles/airship-deploy-env/tasks/prerequisites.yaml
@ -0,0 +1,73 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: Add Ceph apt repository key
+  apt_key:
+    url: https://download.ceph.com/keys/release.asc
+    state: present
+
+- name: Add Ceph apt repository
+  apt_repository:
+    repo: deb https://download.ceph.com/debian-reef/ "{{ ansible_distribution_release }}" main
+    state: present
+    filename: ceph.list
+
+- name: Install necessary packages
+  apt:
+    pkg:
+      - socat
+      - jq
+      - util-linux
+      - bridge-utils
+      - iptables
+      - conntrack
+      - libffi-dev
+      - ipvsadm
+      - make
+      - bc
+      - git-review
+      - notary
+      - ceph-common
+      - rbd-nbd
+      - nfs-common
+      - ethtool
+      - python3-dev
+      - ca-certificates
+      - git
+      - nmap
+      - curl
+      - uuid-runtime
+      - net-tools
+      - less
+      - telnet
+      - tcpdump
+      - vim
+      - lvm2
+
+- name: Deploy Helm
+  when: inventory_hostname == 'primary'
+  block:
+    - name: Install Helm
+      shell: |
+        TMP_DIR=$(mktemp -d)
+        curl -sSL https://get.helm.sh/helm-{{ helm_version }}-linux-amd64.tar.gz | tar -zxv --strip-components=1 -C ${TMP_DIR}
+        mv "${TMP_DIR}"/helm /usr/local/bin/helm
+        rm -rf "${TMP_DIR}"
+      args:
+        executable: /bin/bash
+
+    # This is to improve build time
+    - name: Remove stable Helm repo
+      command: helm repo remove stable
+      ignore_errors: true
+...
--- a/tools/gate/playbooks/roles/airship-run-script-set/defaults/main.yaml
+++ b/tools/gate/playbooks/roles/airship-run-script-set/defaults/main.yaml
@ -0,0 +1,27 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+ceph_osd_data_device: "/dev/loop0"
+kubeadm:
+  pod_network_cidr: "10.244.0.0/24"
+osh_params:
+  container_distro_name: ubuntu
+  container_distro_version: focal
+  # feature_gates:
+site: airskiff
+HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
+HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
+COREDNS_VERSION: v1.11.1
+...
--- a/tools/gate/playbooks/roles/airship-run-script-set/tasks/main.yaml
+++ b/tools/gate/playbooks/roles/airship-run-script-set/tasks/main.yaml
@ -0,0 +1,64 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- block:
+    - name: "Run script set {{ workload }}"
+      shell: |
+        set -xe;
+        {{ gate_script_path }}
+      loop: "{{ workload }}"
+      loop_control:
+        loop_var: gate_script_path
+        pause: 5
+      args:
+        chdir: "{{ zuul.project.src_dir }}/{{ gate_scripts_relative_path }}"
+      environment:
+        CEPH_OSD_DATA_DEVICE: "{{ ceph_osd_data_device }}"
+        POD_NETWORK_CIDR: "{{ kubeadm.pod_network_cidr }}"
+        zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
+        OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"
+        OSH_PATH: "{{ zuul_osh_relative_path | default('../openstack-helm/') }}"
+        OSH_INFRA_PATH: "{{ zuul_osh_infra_relative_path | default('../openstack-helm-infra/') }}"
+        OPENSTACK_RELEASE: "{{ osh_params.openstack_release | default('') }}"
+        CONTAINER_DISTRO_NAME: "{{ osh_params.container_distro_name | default('') }}"
+        CONTAINER_DISTRO_VERSION: "{{ osh_params.container_distro_version | default('') }}"
+        FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}"
+        RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}"
+        PL_SITE: "{{ site | default('airskiff') }}"
+        HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz') }}"
+        HTK_COMMIT: "{{ HTK_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
+        OSH_INFRA_COMMIT: "{{ OSH_INFRA_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
+        OSH_COMMIT: "{{ OSH_COMMIT | default('2d9457e34ca4200ed631466bd87569b0214c92e7') }}"
+        COREDNS_VERSION: "{{ coredns_version | default('v1.11.1') }}"
+      # NOTE(aostapenko) using bigger than async_status timeout due to async_status issue with
+      # not recognizing timed out jobs: https://github.com/ansible/ansible/issues/25637
+      async: 3600
+      poll: 0
+      register: async_results
+
+    - name: Wait for script set to finish
+      async_status:
+        jid: '{{ item.ansible_job_id }}'
+      register: jobs
+      until: jobs.finished
+      delay: 5
+      retries: 360
+      loop: "{{ async_results.results }}"
+
+  always:
+    - name: Print script set output
+      shell: |
+          # NOTE(aostapenko) safely retrieving items for the unlikely case if jobs timed out in async_status
+          echo 'STDOUT:\n{{ item.get("stdout") | regex_replace("\'", "") }}\nSTDERR:\n{{ item.get("stderr") | regex_replace("\'", "") }}'
+      loop: "{{ jobs.results }}"
+...
--- a/tools/gate/playbooks/roles/airship-run-script/defaults/main.yaml
+++ b/tools/gate/playbooks/roles/airship-run-script/defaults/main.yaml
@ -0,0 +1,27 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+ceph_osd_data_device: "/dev/loop0"
+kubeadm:
+  pod_network_cidr: "10.244.0.0/24"
+osh_params:
+  container_distro_name: ubuntu
+  container_distro_version: focal
+  # feature_gates:
+site: airskiff
+HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
+HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
+OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
+COREDNS_VERSION: v1.11.1
+...
--- a/tools/gate/playbooks/roles/airship-run-script/tasks/main.yaml
+++ b/tools/gate/playbooks/roles/airship-run-script/tasks/main.yaml
@ -0,0 +1,40 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: "Run script {{ workload[0] }}"
+  shell: |
+    set -xe;
+    {{ gate_script_path }}
+  vars:
+    gate_script_path: "{{ workload[0] }}"
+  args:
+    chdir: "{{ zuul.project.src_dir }}/{{ gate_scripts_relative_path }}"
+  environment:
+    CEPH_OSD_DATA_DEVICE: "{{ ceph_osd_data_device }}"
+    POD_NETWORK_CIDR: "{{ kubeadm.pod_network_cidr }}"
+    zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
+    OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}"
+    OSH_PATH: "{{ zuul_osh_relative_path | default('../openstack-helm/') }}"
+    OSH_INFRA_PATH: "{{ zuul_osh_infra_relative_path | default('../openstack-helm-infra/') }}"
+    OPENSTACK_RELEASE: "{{ osh_params.openstack_release | default('') }}"
+    CONTAINER_DISTRO_NAME: "{{ osh_params.container_distro_name | default('') }}"
+    CONTAINER_DISTRO_VERSION: "{{ osh_params.container_distro_version | default('') }}"
+    FEATURE_GATES: "{{ osh_params.feature_gates | default('') }}"
+    RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}"
+    PL_SITE: "{{ site | default('airskiff') }}"
+    HELM_ARTIFACT_URL: "{{ HELM_ARTIFACT_URL | default('https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz') }}"
+    HTK_COMMIT: "{{ HTK_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
+    OSH_INFRA_COMMIT: "{{ OSH_INFRA_COMMIT | default('cfff60ec10a6c386f38db79bb9f59a552c2b032f') }}"
+    OSH_COMMIT: "{{ OSH_COMMIT | default('2d9457e34ca4200ed631466bd87569b0214c92e7') }}"
+    COREDNS_VERSION: "{{ coredns_version | default('v1.11.1') }}"
+...
--- a/tools/gate/playbooks/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml
+++ b/tools/gate/playbooks/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml
@ -19,14 +19,14 @@
    masked: yes
    daemon_reload: yes
    name: systemd-resolved
-  become: yes
+  become: true

 - name: Remove local stub dns from resolv.conf, if it exists
  lineinfile:
    path: /etc/resolv.conf
    state: absent
    regexp: '^nameserver.*127.0.0.1'
-  become: yes
+  become: true

 - name: Add upstream nameservers in resolv.conf
  blockinfile:
@ -34,4 +34,4 @@
    block: |
      nameserver 8.8.8.8
      nameserver 8.8.4.4
-  become: yes
+  become: true
--- a/tools/gate/playbooks/site-lint.yaml
+++ b/tools/gate/playbooks/site-lint.yaml
@ -20,4 +20,4 @@
        ./tools/gate/lint-site.sh "{{ site }}"
      args:
        chdir: "{{ zuul.project.src_dir }}"
-      become: yes
+      become: true