Deckhand updates

This PS makes the following changes:

- uses deploy-k8s.sh from treasuremap
- makes sure the airskiff-deploy playbook is using 80Gb partition if
  available
- adds available security updates to docker images

Change-Id: I0f330cb15ec32b12703f0bc6620b3f3c797a25bb

.zuul.yaml

@@ -60,9 +60,9 @@
       jobs:
         - deckhand-upload-git-mirror
         - deckhand-docker-publish-ubuntu_focal
-        - deckhand-docker-tag-ubuntu_focal
+        # - deckhand-docker-tag-ubuntu_focal
         - deckhand-docker-publish-ubuntu_bionic
-        - deckhand-docker-tag-ubuntu_bionic
+        # - deckhand-docker-tag-ubuntu_bionic
 
 
 - nodeset:
@@ -300,7 +300,7 @@
       - airship/treasuremap
     vars:
       CLONE_DECKHAND: false
-      OSH_INFRA_COMMIT: 8e96a91ffae745b952c053923aa177e615b49b74
+      OSH_INFRA_COMMIT: 443ff3e3e340c94c5cbb214d1e2a8b2a3937541d
       DECKHAND_IMAGE_DISTRO: ubuntu_focal
       distro: ubuntu_focal
     irrelevant-files:
@@ -324,7 +324,7 @@
       - airship/treasuremap
     vars:
       CLONE_DECKHAND: false
-      OSH_INFRA_COMMIT: 8e96a91ffae745b952c053923aa177e615b49b74
+      OSH_INFRA_COMMIT: 443ff3e3e340c94c5cbb214d1e2a8b2a3937541d
       DECKHAND_IMAGE_DISTRO: ubuntu_bionic
       distro: ubuntu_focal
     irrelevant-files:
@@ -335,7 +335,7 @@
 
 - job:
     name: deckhand-docker-build-gate-ubuntu_focal
-    timeout: 1800
+    timeout: 3600
     run: tools/gate/playbooks/docker-image-build.yaml
     nodeset: deckhand-single-node-focal
     irrelevant-files: &non-code-files-template
@@ -354,7 +354,7 @@
 
 - job:
     name: deckhand-docker-build-gate-ubuntu_bionic
-    timeout: 1800
+    timeout: 3600
     run: tools/gate/playbooks/docker-image-build.yaml
     nodeset: deckhand-single-node
     irrelevant-files: *non-code-files-template
@@ -371,7 +371,7 @@
       Runs on every merge, unless files in a dictionary below are changed.
       Builds and publishes container ubuntu images on quay.io with a set of tags
       listed in vars section. Waits in Zuul queue for a node (VM) assignment.
-    timeout: 1800
+    timeout: 3600
     run: tools/gate/playbooks/docker-image-build.yaml
     nodeset: deckhand-single-node-focal
     secrets:
@@ -393,7 +393,7 @@
       Runs on every merge, unless files in a dictionary below are changed.
       Builds and publishes container ubuntu images on quay.io with a set of tags
       listed in vars section. Waits in Zuul queue for a node (VM) assignment.
-    timeout: 1800
+    timeout: 3600
     run: tools/gate/playbooks/docker-image-build.yaml
     nodeset: deckhand-single-node
     secrets:
@@ -409,37 +409,37 @@
         static:
           - latest
 
-- job:
+# - job:
-    name: deckhand-docker-tag-ubuntu_focal
+#     name: deckhand-docker-tag-ubuntu_focal
-    description: |
+#     description: |
-      Runs on every merge when files in a dictionalry below are changed, and
+#       Runs on every merge when files in a dictionalry below are changed, and
-      adds git commit id tag onto the ubuntu container image published on quay.io,
+#       adds git commit id tag onto the ubuntu container image published on quay.io,
-      which has `latest` tag set. Does not wait in queue for a node (VM)
+#       which has `latest` tag set. Does not wait in queue for a node (VM)
-      assignment, runs almost immediately.
+#       assignment, runs almost immediately.
-    timeout: 1800
+#     timeout: 3600
-    run: tools/gate/playbooks/docker-image-tag.yaml
+#     run: tools/gate/playbooks/docker-image-tag.yaml
-    nodeset:
+#     nodeset:
-      nodes: []
+#       nodes: []
-    secrets:
+#     secrets:
-      - airship_deckhand_quay_creds
+#       - airship_deckhand_quay_creds
-    vars:
+#     vars:
-      distro: ubuntu_focal
+#       distro: ubuntu_focal
 
-- job:
+# - job:
-    name: deckhand-docker-tag-ubuntu_bionic
+#     name: deckhand-docker-tag-ubuntu_bionic
-    description: |
+#     description: |
-      Runs on every merge when files in a dictionalry below are changed, and
+#       Runs on every merge when files in a dictionalry below are changed, and
-      adds git commit id tag onto the ubuntu container image published on quay.io,
+#       adds git commit id tag onto the ubuntu container image published on quay.io,
-      which has `latest` tag set. Does not wait in queue for a node (VM)
+#       which has `latest` tag set. Does not wait in queue for a node (VM)
-      assignment, runs almost immediately.
+#       assignment, runs almost immediately.
-    timeout: 1800
+#     timeout: 3600
-    run: tools/gate/playbooks/docker-image-tag.yaml
+#     run: tools/gate/playbooks/docker-image-tag.yaml
-    nodeset:
+#     nodeset:
-      nodes: []
+#       nodes: []
-    secrets:
+#     secrets:
-      - airship_deckhand_quay_creds
+#       - airship_deckhand_quay_creds
-    vars:
+#     vars:
-      distro: ubuntu_bionic
+#       distro: ubuntu_bionic
 
 
 

charts/deckhand/values.yaml

@@ -29,7 +29,7 @@ images:
   tags:
     deckhand: quay.io/airshipit/deckhand:latestlatest-ubuntu_focal
     dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0
-    db_init: docker.io/postgres:14.6
+    db_init: docker.io/postgres:14.8
     db_sync: quay.io/airshipit/deckhand:latest-ubuntu_focal
     image_repo_sync: docker.io/docker:23.0.3
     ks_endpoints: docker.io/openstackhelm/heat:wallaby-ubuntu_focal

images/deckhand/Dockerfile.ubuntu_bionic

@@ -30,7 +30,7 @@ ENV PORT 9000
 EXPOSE $PORT
 
 RUN set -x && \
-    apt-get -qq update && \
+    apt-get update && apt-get upgrade -y && \
     apt-get -y install \
     automake \
     ca-certificates \

images/deckhand/Dockerfile.ubuntu_focal

@@ -30,7 +30,7 @@ ENV PORT 9000
 EXPOSE $PORT
 
 RUN set -x && \
-    apt-get -qq update && \
+    apt-get update && apt-get upgrade -y && \
     apt-get -y install \
     automake \
     ca-certificates \

tools/gate/playbooks/airskiff-deploy.yaml

@@ -16,11 +16,19 @@
   roles:
     - clear-firewall
     - bindep
+    - ensure-docker
     - disable-systemd-resolved
     - install-test-requirements
 
   tasks:
 
+    - name: Install Packaging python module for airship
+      block:
+        - pip:
+            name: packaging
+            executable: pip3
+      become: True
+
     - name: Clone Required Repositories
       shell: |
         export CLONE_DECKHAND={{ CLONE_DECKHAND }}
@@ -37,6 +45,17 @@
 
     - name: Deploy Kubernetes with Minikube
       shell: |
+        set -ex
+        sudo fdisk --list
+        df -h
+        sudo mkdir -p /opt/ext_vol
+        BIG_VOLUME=$(sudo fdisk -l 2>&1 | grep  -E 80G | grep Linux | awk '{print $1}')
+        if ! mount | grep "${BIG_VOLUME}"
+        then
+          sudo mkfs.ext4 "${BIG_VOLUME}"
+          sudo mount "${BIG_VOLUME}" /opt/ext_vol
+          df -h
+        fi
         ./tools/deployment/airskiff/developer/010-deploy-k8s.sh
       args:
         chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
@@ -46,15 +65,33 @@
         set -ex
         export DISTRO={{ DECKHAND_IMAGE_DISTRO }}
         make images
-        if test "${DISTRO}" = 'ubuntu_bionic'
+        docker system prune --force
-        then
-          # this trick is needed to use bionic image instead of focal in airskiff deployment test
-          docker tag quay.io/airshipit/deckhand:latest-ubuntu_bionic quay.io/airshipit/deckhand:latest-ubuntu_focal
-        fi
       args:
         chdir: "{{ zuul.project.src_dir }}"
       become: yes
 
+    - name: Use locally built images in manifests
+      shell: |
+        set -ex
+        export DISTRO={{ DECKHAND_IMAGE_DISTRO }}
+        docker rm registry --force || true
+        docker run -d -p 5000:5000 --restart=always --name registry registry:2
+        if test "${DISTRO}" = 'ubuntu_bionic'
+        then
+          docker tag quay.io/airshipit/deckhand:latest-ubuntu_bionic localhost:5000/deckhand:latest-ubuntu_bionic
+          docker push localhost:5000/deckhand:latest-ubuntu_bionic
+          sed -i "s#quay.io/airshipit/deckhand:latest-ubuntu_focal#localhost:5000/deckhand:latest-ubuntu_bionic#g" ./site/airskiff/software/config/versions.yaml
+          sed -i "s#quay.io/airshipit/deckhand:latest-ubuntu_focal#localhost:5000/deckhand:latest-ubuntu_bionic#g" ./global/software/config/versions.yaml
+        else
+          docker tag quay.io/airshipit/deckhand:latest-ubuntu_focal localhost:5000/deckhand:latest-ubuntu_focal
+          docker push localhost:5000/deckhand:latest-ubuntu_focal
+          sed -i "s#quay.io/airshipit/deckhand:latest-ubuntu_focal#localhost:5000/deckhand:latest-ubuntu_focal#g" ./site/airskiff/software/config/versions.yaml
+          sed -i "s#quay.io/airshipit/deckhand:latest-ubuntu_focal#localhost:5000/deckhand:latest-ubuntu_focal#g" ./global/software/config/versions.yaml
+        fi
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
     - name: Build all charts locally
       shell: |
         set -ex
@@ -63,6 +100,18 @@
         chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
       become: yes
 
+    - name: Start artifactory
+      shell: |
+        set -ex
+        # start http server with artifacts
+        docker rm artifacts --force || true
+        docker run --name artifacts -p 8282:80 -v $(pwd)/../artifacts:/usr/share/nginx/html -d nginx
+        sleep 10
+        curl --verbose -I http://control-plane.minikube.internal:8282/memcached.tgz
+      args:
+        chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
+      become: yes
+
     - name: Deploy Airship components using Armada
       shell: |
         mkdir -p ~/.kube

tools/gate/playbooks/docker-image-build.yaml

@@ -54,6 +54,13 @@
             executable: pip3
       become: True
 
+    - name: Install tox python module for ansible docker login
+      block:
+        - pip:
+            name: tox
+            version: 3.28.0
+            executable: pip3
+      become: True
 
     - name: Make images
       when: not publish

tools/gate/playbooks/git-config.yaml

@@ -16,7 +16,7 @@
   tasks:
     - name: Git config
       shell: |
-        set -xe;
+        set -xe
         tee .git/config << EOF
         [remote "origin"]
         url = https://opendev.org/airship/treasuremap.git

tools/gate/playbooks/run-integration-tests-docker.yaml

@@ -16,10 +16,12 @@
   vars_files:
     - vars.yaml
   roles:
+    - clear-firewall
     - bindep
     - ensure-docker
     - ensure-python
     - ensure-pip
+    - disable-systemd-resolved
     - install-test-requirements
     - build-images
     - deploy-keystone-dependencies

tools/gate/roles/deploy-keystone-dependencies/tasks/deploy-keystone-dependencies.yaml

@@ -82,4 +82,4 @@
     set -xe;
     ./tools/deployment/component/keystone/keystone.sh
   args:
-    chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}"
+    chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('') }}"

tools/gate/roles/install-test-requirements/tasks/install-test-requirements.yaml

@@ -12,13 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Set modprobe br_netfilter
-  shell: |
-    set -xe;
-    sudo modprobe br_netfilter
-    sudo sysctl net.bridge.bridge-nf-call-iptables=1
-    sudo sysctl net.bridge.bridge-nf-call-ip6tables=1
-
 - name: Install pip3 and gabbi
   shell: |
     set -xe;

tools/gate/roles/run-integration-tests/tasks/integration-tests.yaml

@@ -62,6 +62,16 @@
     else
       sudo -E -H pip3 install -r requirements-frozen.txt
     fi
+    sudo fdisk --list
+    df -h
+    sudo mkdir -p /opt/ext_vol
+    BIG_VOLUME=$(sudo fdisk -l 2>&1 | grep  -E 80G | grep Linux | awk '{print $1}')
+    if ! mount | grep "${BIG_VOLUME}"
+    then
+      sudo mkfs.ext4 "${BIG_VOLUME}"
+      sudo mount "${BIG_VOLUME}" /opt/ext_vol
+      df -h
+    fi
     pifpaf run postgresql -- ./tools/integration-tests.sh
   args:
     chdir: "{{ zuul.project.src_dir }}"

tools/gate/scripts/020-deploy-postgresql.sh

@@ -23,7 +23,7 @@ POSTGRES_ID=$(
         -e POSTGRES_DB=deckhand \
         -e POSTGRES_USER=deckhand \
         -e POSTGRES_PASSWORD=password \
-            postgres:14.6
+            postgres:14.8
 )
 
 POSTGRES_IP=$(

tools/helm_install.sh

@@ -17,7 +17,7 @@
 set -x
 
 HELM=$1
-HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.11.1-linux-amd64.tar.gz"}
+HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz"}
 
 
 function install_helm_binary {

tools/helm_tk.sh

@@ -17,7 +17,7 @@
 set -eux
 
 HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"}
-HTK_STABLE_COMMIT=${HTK_COMMIT:-"f4972121bcb41c8d74748917804d2b239ab757f9"}
+HTK_STABLE_COMMIT=${HTK_COMMIT:-"443ff3e3e340c94c5cbb214d1e2a8b2a3937541d"}
 
 TMP_DIR=$(mktemp -d)
 

tools/integration-tests.sh

@@ -13,8 +13,18 @@
 set -xe
 
 CURRENT_DIR="$(pwd)"
-: ${OSH_INFRA_PATH:="../openstack-helm-infra"}
+: "${OSH_INFRA_PATH:="../openstack-helm-infra"}"
-: ${OSH_PATH:="../openstack-helm"}
+: "${OSH_PATH:="../openstack-helm"}"
+: "${TM_PATH:="../treasuremap"}"
+
+export MAKE_CHARTS_OPENSTACK_HELM="${MAKE_CHARTS_OPENSTACK_HELM:-true}"
+export MAKE_CHARTS_OSH_INFRA="${MAKE_CHARTS_OSH_INFRA:-true}"
+export MAKE_CHARTS_ARMADA="${MAKE_CHARTS_ARMADA:-false}"
+export MAKE_CHARTS_DECKHAND="${MAKE_CHARTS_DECKHAND:-false}"
+export MAKE_CHARTS_SHIPYARD="${MAKE_CHARTS_SHIPYARD:-false}"
+export MAKE_CHARTS_MAAS="${MAKE_CHARTS_MAAS:-false}"
+export MAKE_CHARTS_PORTHOLE="${MAKE_CHARTS_PORTHOLE:-false}"
+export MAKE_CHARTS_PROMENADE="${MAKE_CHARTS_PROMENADE:-false}"
 
 
 function deploy_barbican {
@@ -46,34 +56,28 @@ function deploy_osh_keystone_barbican {
         git clone https://git.openstack.org/openstack/openstack-helm.git ../openstack-helm
     fi
 
-    cd ${OSH_INFRA_PATH}
+    if [ ! -d "$TM_PATH" ]; then
-    # git reset --hard ${BARBICAN_STABLE_COMMIT}
+        git clone https://git.openstack.org/airship/treasuremap.git ../treasuremap
+        pushd ../treasuremap
+        git checkout v1.9
+        popd
+    fi
+
+    cd "${TM_PATH}"
     # Deploy required packages
-    ./tools/deployment/common/000-install-packages.sh
+    ./tools/deployment/airskiff/developer/009-setup-apparmor.sh
-    ./tools/deployment/common/001-setup-apparmor-profiles.sh
-    #
-    cd ${OSH_PATH}
-    # git reset --hard ${BARBICAN_STABLE_COMMIT}
-    # Deploy required packages
-    ./tools/deployment/common/install-packages.sh
     #
     # Deploy Kubernetes
-    sudo modprobe br_netfilter
+    ./tools/deployment/airskiff/developer/010-deploy-k8s.sh
-    ./tools/deployment/common/deploy-k8s.sh
+    #
+    # Make charts
+    ./tools/deployment/airskiff/developer/015-make-all-charts.sh
+    #
+    # Deploy docker-based openstack client
+    ./tools/deployment/airskiff/developer/020-setup-client.sh
 
-    cd ${CURRENT_DIR}
 
-    # remove systemd-resolved local stub dns from resolv.conf
+    cd "${OSH_PATH}"
-    sudo sed -i.bkp '/^nameserver.*127.0.0.1/d
-                     w /dev/stdout' /etc/resolv.conf
-    # add external nameservers
-    echo "nameserver 8.8.8.8" | sudo tee -a /etc/resolv.conf
-    echo "nameserver 8.8.4.4" | sudo tee -a /etc/resolv.conf
-    cat /etc/resolv.conf
-
-    cd ${OSH_PATH}
-    # Setup clients on the host and assemble the charts
-    ./tools/deployment/common/setup-client.sh
     # Deploy the ingress controller
     ./tools/deployment/component/common/ingress.sh
     # Deploy NFS Provisioner
@@ -102,7 +106,7 @@ function deploy_deckhand {
     interfaces=("admin" "public" "internal")
     deckhand_endpoint="http://127.0.0.1:9000"
 
-    if [ -z "$( openstack service list --format value 2>/dev/null | grep deckhand )" ]; then
+    if [ -z "$( openstack_client openstack service list --format value 2>/dev/null | grep deckhand )" ]; then
         openstack service create --enable --name deckhand deckhand 2>/dev/null
     fi