airship/deckhand
Code Issues Proposed changes

Use deploy-env role

Browse Source 
Change-Id: I50530eabc8506319478d6ed02b7abb5eb1d2dd05
This commit is contained in:
Sergiy Markin 2024-02-16 01:04:08 +00:00
parent 5b7da9ebd6
commit 496c525c27
9 changed files with 253 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
.zuul.yaml
View File

@ -22,6 +22,7 @@
- deckhand-tox-py38-postgresql
- deckhand-functional-uwsgi-py38
- deckhand-functional-docker-py38
- deckhand-functional-docker-py38-kubeadm
- deckhand-integration-uwsgi-py38
- deckhand-integration-docker-py38
- deckhand-chart-build-gate
@ -78,18 +79,18 @@
name: deckhand-openstack-tox-pep8-focal
parent: openstack-tox-pep8
description: Runs pep8 job on focal
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
- job:
name: deckhand-openstack-tox-cover-focal
parent: openstack-tox-cover
description: Runs cover job on focal
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
- job:
name: deckhand-tox-py38-postgresql
parent: openstack-tox-py38
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
pre-run:
- tools/gate/playbooks/install-postgresql.yaml
vars:
@ -104,7 +105,7 @@
minimalistic deployment consisting of uwsgi for Deckhand API and pifpaf
for ephemeral PostgreSQL DB, under cPython version 3.8.
run: tools/gate/playbooks/run-functional-tests-uwsgi.yaml
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
vars:
tox_envlist: py38-functional-dev
distro: ubuntu_focal
@ -116,17 +117,86 @@
- ^setup.cfg$
- ^deckhand/tests/unit/.*$
- job:
name: deckhand-base
abstract: true
roles:
- zuul: openstack/openstack-helm-infra
- zuul: aprship/porthole
- zuul: zuul/zuul-jobs
required-projects:
- openstack/openstack-helm
- openstack/openstack-helm-infra
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- ^releasenotes/.*$
- ^site/seaworthy/.*$
- ^site/airsloop/.*$
- ^site/aiab/.*$
timeout: 10800
pre-run:
- tools/gate/playbooks/prepare-hosts.yaml
- tools/gate/playbooks/mount-volumes.yaml
- tools/gate/playbooks/deploy-env.yaml
run:
- tools/gate/playbooks/airship-run-scripts.yaml
post-run:
- tools/gate/playbooks/osh-infra-collect-logs.yaml
vars:
extra_volume:
size: 80G
type: Linux
mount_point: /opt/ext_vol
docker:
root_path: "/opt/ext_vol/docker"
containerd:
root_path: "/opt/ext_vol/containerd"
kubeadm:
pod_network_cidr: "10.244.0.0/24"
service_cidr: "10.96.0.0/16"
loopback_setup: true
loopback_device: /dev/loop100
loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img"
ceph_osd_data_device: /dev/loop100
kube_version: "1.28.4-1.1"
calico_version: "v3.27.0"
coredns_version: "v1.11.1"
helm_version: "v3.13.2"
yq_version: "v4.6.0"
crictl_version: "v1.26.1"
zuul_osh_infra_relative_path: ../openstack-helm-infra
gate_scripts_relative_path: ../openstack-helm-infra
run_helm_tests: "no"
- job:
name: deckhand-functional-docker-py38-kubeadm
description: |
Run tox-based functional tests for the Airship Deckhand project under
cPython version 3.8. Uses tox with the ``functional-py38`` environment.
Ubuntu (focal) image is built and used.
parent: deckhand-base
run: tools/gate/playbooks/run-functional-tests-docker.yaml
nodeset: treasuremap-airskiff-1node-ubuntu_focal
vars:
tox_envlist: py38-functional
disable_keystone: true
distro: ubuntu_focal
irrelevant-files: *irrelevant-files
- job:
name: deckhand-functional-docker-base
description: |
Base job for running deckhand functional tests. Runs tests
against Docker image generated from source code.
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
roles:
- zuul: openstack/openstack-helm-infra
timeout: 3600
pre-run:
- tools/gate/playbooks/osh-infra-upgrade-host.yaml
- tools/gate/playbooks/
- tools/gate/playbooks/osh-infra-deploy-docker.yaml
run: tools/gate/playbooks/run-functional-tests-docker.yaml
required-projects:
@ -139,7 +209,7 @@
cPython version 3.8. Uses tox with the ``functional-py38`` environment.
Ubuntu (focal) image is built and used.
parent: deckhand-functional-docker-base
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
vars:
tox_envlist: py38-functional
disable_keystone: true
@ -156,7 +226,7 @@
for ephemeral PostgreSQL DB, under cPython version 3.8.
timeout: 9600
run: tools/gate/playbooks/run-integration-tests-uwsgi.yaml
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
irrelevant-files: *irrelevant-files
vars:
disable_keystone: true
@ -168,9 +238,9 @@
Build charts using pinned Helm toolkit.
timeout: 900
run: tools/gate/playbooks/build-charts.yaml
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
vars:
HTK_COMMIT: ae91cf3fc3f288b6d92ace4a3a405606a653638f
HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
- job:
name: deckhand-chart-build-latest-htk
@ -178,7 +248,7 @@
Build charts using latest Helm toolkit.
timeout: 900
run: tools/gate/playbooks/build-charts.yaml
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
vars:
HTK_COMMIT: master
@ -187,7 +257,7 @@
description: |
Base job for running deckhand integration tests. Runs tests
against Docker image generated from source code.
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
timeout: 3600
roles:
- zuul: openstack/openstack-helm-infra
@ -224,7 +294,7 @@
- job:
name: deckhand-airskiff-deployment-focal
voting: true
nodeset: deckhand-single-node-airskiff-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
description: |
Deploy Memcached using Airskiff and submitted Deckhand changes.
timeout: 9600
@ -253,7 +323,7 @@
name: deckhand-docker-build-gate
timeout: 3600
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
irrelevant-files: &non-code-files-template
- ^.*\.rst$
- ^doc/.*$
@ -276,7 +346,7 @@
listed in vars section. Waits in Zuul queue for a node (VM) assignment.
timeout: 3600
run: tools/gate/playbooks/docker-image-build.yaml
nodeset: deckhand-single-node-focal
nodeset: treasuremap-airskiff-1node-ubuntu_focal
secrets:
- airship_deckhand_quay_creds
irrelevant-files: *non-code-files-template

75
tools/gate/playbooks/airship-run-scripts.yaml Normal file
View File

@ -0,0 +1,75 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
- hosts: primary
tasks:
- name: Override images
when: buildset_registry is defined
vars:
work_dir: "{{ zuul.project.src_dir }}"
block:
- name: Buildset registry alias
include_role:
name: deploy-env
tasks_from: buildset_registry_alias
- name: Print zuul
debug:
var: zuul
- name: Override proposed images from artifacts
shell: >
find {{ override_paths | join(" ") }} -type f -exec sed -Ei
"s#['\"]?docker\.io/({{ repo }}):({{ tag }})['\"]?\$#{{ buildset_registry_alias }}:{{ buildset_registry.port }}/\1:\2#g" {} +
loop: "{{ zuul.artifacts | default([]) }}"
args:
chdir: "{{ work_dir }}"
loop_control:
loop_var: zj_zuul_artifact
when: "'metadata' in zj_zuul_artifact and zj_zuul_artifact.metadata.type | default('') == 'container_image'"
vars:
tag: "{{ zj_zuul_artifact.metadata.tag }}"
repo: "{{ zj_zuul_artifact.metadata.repository }}"
override_paths:
- ../openstack-helm*/*/values*
- ../openstack-helm-infra/tools/deployment/
- name: Diff
shell: |
set -ex;
for dir in openstack-helm openstack-helm-infra; do
path="{{ work_dir }}/../${dir}/"
if [ ! -d "${path}" ]; then continue; fi
echo "${dir} diff"
cd "${path}"; git diff; cd -;
done
- name: "creating directory for run artifacts"
file:
path: "/tmp/artifacts"
state: directory
- name: Run gate scripts
include_role:
name: "{{ ([item] | flatten | length == 1) | ternary('airship-run-script', 'airship-run-script-set') }}"
vars:
workload: "{{ [item] | flatten }}"
loop: "{{ gate_scripts }}"
- name: "Downloads artifacts to executor"
synchronize:
src: "/tmp/artifacts"
dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
mode: pull
ignore_errors: True
...

36
tools/gate/playbooks/deploy-env.yaml Normal file
View File

@ -0,0 +1,36 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
- hosts: all
become: true
gather_facts: true
roles:
- ensure-python
- ensure-pip
- ensure-tox
- clear-firewall
- deploy-apparmor
- deploy-selenium
- deploy-env
tasks:
- name: Install Packaging python module for tools/airship
block:
- pip:
name: packaging
version: 23.1
executable: pip3
become: True
...

17
tools/gate/playbooks/mount-volumes.yaml Normal file
View File

@ -0,0 +1,17 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
- hosts: all
roles:
- mount-extra-volume
...

25
tools/gate/playbooks/osh-infra-collect-logs.yaml
View File

@ -1,5 +1,3 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@ -12,17 +10,34 @@
# See the License for the specific language governing permissions and
# limitations under the License.
---
- hosts: all
vars_files:
- vars.yaml
vars:
work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
logs_dir: "/tmp/logs"
roles:
- gather-host-logs
- helm-release-status
- describe-kubernetes-objects
- gather-pod-logs
tags:
- gather-host-logs
- hosts: primary
vars_files:
- vars.yaml
vars:
work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
logs_dir: "/tmp/logs"
roles:
- helm-release-status
- describe-kubernetes-objects
- gather-pod-logs
- gather-prom-metrics
- gather-selenium-data
tags:
- helm-release-status
- describe-kubernetes-objects
- gather-pod-logs
- gather-prom-metrics
- gather-selenium-data
...

17
tools/gate/playbooks/prepare-hosts.yaml Normal file
View File

@ -0,0 +1,17 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
- hosts: all
roles:
- start-zuul-console
...

26
tools/gate/playbooks/run-functional-tests-docker.yaml
View File

@ -16,34 +16,10 @@
gather_facts: False
become: yes
roles:
- install-test-requirements
- build-images
tags:
- build-images
- hosts: all
gather_facts: False
roles:
- deploy-postgresql
tags:
- deploy-postgresql
- hosts: all
gather_facts: False
roles:
- generate-test-config
tags:
- generate-test-config
- hosts: all
gather_facts: False
roles:
- deploy-deckhand
tags:
- deploy-deckhand
- hosts: all
gather_facts: False
roles:
- run-functional-tests
tags:
- run-functional-tests

2
tools/helm_install.sh
View File

@ -17,7 +17,7 @@
set -x
HELM=$1
HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.12.2-linux-amd64.tar.gz"}
HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz"}
function install_helm_binary {

2
tools/helm_tk.sh
View File

@ -17,7 +17,7 @@
set -eux
HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"}
HTK_STABLE_COMMIT=${HTK_COMMIT:-"d0c93b5648704b9d9999412d932f8329c80362c0"}
HTK_STABLE_COMMIT=${HTK_COMMIT:-"cfff60ec10a6c386f38db79bb9f59a552c2b032f"}
TMP_DIR=$(mktemp -d)