airship/treasuremap
Code Issues Proposed changes

Update gates to ubuntu-focal

Browse Source 
- zuul gates were switched to ubuntu-focal
- tox.ini has been fixed to work with ubuntu-jammy
- added initial-setup.yaml playbook to setup focal nodes
- uplifted latest OSH commit Id with kubernetes 1.26.3
- added bindep.txt wiht binary dependences
- updated manifests to latest commits of OSH and OSH-INFRA
- added limit to setuptools version to fix ./tools/sirship
- added debug option to armada command
- fixed ~/.kube/config permission for armada bootstrap run
- uplifted OSH and OSH-INFRA images
- treasuremap-airskiff-deployment-ubuntu is now voting gate again
- k8s.gcr.io replaced with registry.k8s.io

Change-Id: I166924c8cadfc8e21c115ce8abf49cb65093a7eb
This commit is contained in:
Sergiy Markin 2023-03-27 19:29:15 +00:00
parent 342c9eedd5
commit 59ef68a27b
22 changed files with 459 additions and 355 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.gitignore vendored
View File

@ -14,3 +14,17 @@ _build
.vimrc
*.swp
.vscode/
# Packages
*.egg*
*.egg-info
dist
build
eggs
parts
var
sdist
develop-eggs
.installed.cfg
lib
lib64

20
.zuul.yaml
View File

@ -49,14 +49,21 @@
- name: ubuntu-bionic
label: ubuntu-bionic
- nodeset:
name: treasuremap-single-node-ubuntu-focal
nodes:
- name: ubuntu-focal
label: ubuntu-focal
- job:
name: treasuremap-site-lint
description:
Lint a site using Pegleg. Default site is seaworthy.
nodeset: treasuremap-single-node
nodeset: treasuremap-single-node-ubuntu-focal
timeout: 900
pre-run:
- tools/gate/playbooks/install-docker.yaml
- tools/gate/playbooks/initial-setup.yaml
- tools/gate/playbooks/git-config.yaml
run: tools/gate/playbooks/site-lint.yaml
vars:
@ -146,6 +153,7 @@
Lint the aiab site using Pegleg.
parent: treasuremap-site-lint
pre-run:
- tools/gate/playbooks/initial-setup.yaml
- tools/gate/playbooks/generate-certs.yaml
vars:
site: aiab
@ -159,12 +167,13 @@
- job:
name: treasuremap-airskiff-deployment-ubuntu
nodeset: treasuremap-single-node
nodeset: treasuremap-single-node-ubuntu-focal
description: |
Deploy Memcached using Airskiff and latest Treasuremap changes.
voting: false
voting: true
timeout: 9600
pre-run:
- tools/gate/playbooks/initial-setup.yaml
- tools/gate/playbooks/git-config.yaml
- tools/gate/playbooks/airskiff-reduce-site.yaml
run: tools/gate/playbooks/airskiff-deploy-gate.yaml
@ -180,12 +189,13 @@
- job:
name: treasuremap-airskiff-deployment-suse
nodeset: treasuremap-single-node
nodeset: treasuremap-single-node-ubuntu-focal
description: |
Deploy Memcached using Airskiff-suse and latest Treasuremap changes.
voting: false
timeout: 9600
pre-run:
- tools/gate/playbooks/initial-setup.yaml
- tools/gate/playbooks/git-config.yaml
- tools/gate/playbooks/airskiff-reduce-site.yaml
run: tools/gate/playbooks/airskiff-deploy-gate.yaml

10
bindep.txt Normal file
View File

@ -0,0 +1,10 @@
# This file contains runtime (non-python) dependencies
# More info at: https://docs.openstack.org/infra/bindep/readme.html
libffi-dev [test platform:dpkg]
libkrb5-dev [platform:dpkg]
libpq-dev [platform:dpkg]
libsasl2-dev [platform:dpkg]
libssl-dev [platform:dpkg]
libre2-dev [platform:dpkg]
ethtool [platform:dpkg]

2
doc/source/conf.py
View File

@ -68,7 +68,7 @@ author = 'The Airship Authors'
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
language = 'en'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.

3
global/software/charts/kubernetes/dns/coredns.yaml
View File

@ -93,6 +93,9 @@ data:
.:53 {
errors
health
header {
response set ra
}
autopath @kubernetes
kubernetes CLUSTER_DOMAIN SERVICE_CIDR POD_CIDR {
pods insecure

579
global/software/config/versions.yaml
View File

File diff suppressed because it is too large Load Diff

10
site/airskiff/software/config/versions.yaml
View File

@ -21,10 +21,10 @@ data:
armada:
api: "quay.io/airshipit/armada:latest-ubuntu_bionic"
deckhand:
deckhand: "quay.io/airshipit/deckhand:latest-ubuntu_xenial"
deckhand: "quay.io/airshipit/deckhand:latest-ubuntu_bionic"
shipyard:
shipyard: "quay.io/airshipit/shipyard:latest-ubuntu_xenial"
shipyard_db_sync: "quay.io/airshipit/shipyard:latest-ubuntu_xenial"
airflow: "quay.io/airshipit/airflow:latest-ubuntu_xenial"
airflow_db_sync: "quay.io/airshipit/airflow:latest-ubuntu_xenial"
shipyard: "quay.io/airshipit/shipyard:latest-ubuntu_bionic"
shipyard_db_sync: "quay.io/airshipit/shipyard:latest-ubuntu_bionic"
airflow: "quay.io/airshipit/airflow:latest-ubuntu_bionic"
airflow_db_sync: "quay.io/airshipit/airflow:latest-ubuntu_bionic"
...

12
tools/airship
View File

@ -67,7 +67,7 @@ manifests_lookup(){
local allow_fail="$6"
FAIL=false
RESULT=`python3 -c "
RESULT=$(python3 -c "
import yaml,sys
from distutils.version import StrictVersion
@ -94,7 +94,7 @@ for x in y:
print(x$key_path)
break
else:
sys.exit(1)" 2>&1` || FAIL=true
sys.exit(1)" 2>&1) || FAIL=true
if [[ $FAIL = true ]] && [[ $allow_fail != true ]]; then
echo "error: Lookup failed for schema '$schema', \
@ -130,7 +130,7 @@ pegleg() {
USER=pegleg
EOF
docker run --rm --net=host $TERM_OPTS \
docker run --rm --net=host $TERM_OPTS \
-u "${USER}:${GROUP}" \
-w /target \
-v $(pwd):/target \
@ -169,14 +169,14 @@ EOF
# extract Hyperkube binary before running Promenade container
# this is replacing internal extraction step in Promenade
# no need to share Docker socket
docker run --rm $TERM_OPTS \
docker run --rm $TERM_OPTS \
-v "${PROMENADE_TMP}:/tmp/${PROMENADE_TMP_LOCAL}" \
--env-file $ENV_FILE \
--entrypoint /bin/cp \
"${IMAGE_HYPERKUBE}" \
/hyperkube "/tmp/${PROMENADE_TMP_LOCAL}"
docker run --rm --net=host $TERM_OPTS \
docker run --rm --net=host $TERM_OPTS \
-u "${USER}:${GROUP}" \
-w /target \
-v $(pwd):/target \
@ -190,7 +190,7 @@ shipyard() {
versions_lookup "['data']['images']['ucp']['shipyard']['shipyard']"
SHIPYARD_IMAGE=$RESULT
docker run --rm --net=host $TERM_OPTS \
docker run --rm --net=host $TERM_OPTS \
-u "${USER}:${GROUP}" \
-w /target \
-v $(pwd):/target \

12
tools/deployment/airskiff/common/os-env.sh
View File

@ -14,9 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
export OS_PROJECT_DOMAIN_NAME=`grep -m 1 project_domain_name /etc/openstack/clouds.yaml | cut -d "'" -f 2`
export OS_USER_DOMAIN_NAME=`grep -m 1 user_domain_name /etc/openstack/clouds.yaml | cut -d "'" -f 2`
export OS_PROJECT_NAME=`grep -m 1 project_name /etc/openstack/clouds.yaml | cut -d "'" -f 2`
export OS_USERNAME=`grep -m 1 username /etc/openstack/clouds.yaml | cut -d "'" -f 2`
export OS_PASSWORD=`grep -m 1 password /etc/openstack/clouds.yaml | cut -d "'" -f 2`
export OS_AUTH_URL=`grep -m 1 auth_url /etc/openstack/clouds.yaml | cut -d "'" -f 2`
export OS_PROJECT_DOMAIN_NAME=$(grep -m 1 project_domain_name /etc/openstack/clouds.yaml | cut -d "'" -f 2)
export OS_USER_DOMAIN_NAME=$(grep -m 1 user_domain_name /etc/openstack/clouds.yaml | cut -d "'" -f 2)
export OS_PROJECT_NAME=$(grep -m 1 project_name /etc/openstack/clouds.yaml | cut -d "'" -f 2)
export OS_USERNAME=$(grep -m 1 username /etc/openstack/clouds.yaml | cut -d "'" -f 2)
export OS_PASSWORD=$(grep -m 1 password /etc/openstack/clouds.yaml | cut -d "'" -f 2)
export OS_AUTH_URL=$(grep -m 1 auth_url /etc/openstack/clouds.yaml | cut -d "'" -f 2)

2
tools/deployment/airskiff/developer/000-clone-dependencies.sh
View File

@ -19,7 +19,7 @@ set -xe
CURRENT_DIR="$(pwd)"
: "${INSTALL_PATH:="../"}"
: "${OSH_INFRA_COMMIT:="dc58ef9dddd0326cc86229eda4e21e269adb31be"}"
: "${OSH_INFRA_COMMIT:="e6dfa15c269caa9fffb7d2205e614bb2deae43d6"}"
: "${CLONE_ARMADA:=true}"
: "${CLONE_DECKHAND:=true}"
: "${CLONE_SHIPYARD:=true}"

2
tools/deployment/airskiff/developer/020-setup-client.sh
View File

@ -19,7 +19,7 @@ set -xe
# Install OpenStack client and create OpenStack client configuration file.
sudo -H -E pip3 install "cmd2<=0.8.7"
sudo -H -E pip3 install --upgrade setuptools
sudo -H -E pip3 install --upgrade setuptools==50.0.0
sudo -H -E pip3 install python-openstackclient python-heatclient
sudo -H mkdir -p /etc/openstack

13
tools/deployment/airskiff/developer/030-armada-bootstrap.sh
View File

@ -31,20 +31,17 @@ AIRSKIFF_PERMISSIONS=$(stat --format '%a' airskiff.yaml)
KUBE_CONFIG_PERMISSIONS=$(stat --format '%a' ~/.kube/config)
sudo chmod 0644 airskiff.yaml
sudo chmod 0644 ~/.kube/config
# sudo chmod 0644 ~/.kube/config
# In the event that this docker command fails, we want to continue the script
# and reset the file permissions.
set +e
# Download latest Armada image and deploy Airship components
docker run --rm --net host -p 8000:8000 --name armada \
-v ~/.kube/config:/armada/.kube/config \
-v "$(pwd)"/airskiff.yaml:/airskiff.yaml \
-v "${INSTALL_PATH}":/airship-components \
quay.io/airshipit/armada:latest-ubuntu_bionic \
apply /airskiff.yaml --target-manifest $TARGET_MANIFEST
quay.io/airshipit/armada:latest-ubuntu_focal\
apply /airskiff.yaml --debug --target-manifest $TARGET_MANIFEST
# Set back permissions of the files
# # Set back permissions of the files
sudo chmod "${AIRSKIFF_PERMISSIONS}" airskiff.yaml
sudo chmod "${KUBE_CONFIG_PERMISSIONS}" ~/.kube/config
# sudo chmod "${KUBE_CONFIG_PERMISSIONS}" ~/.kube/config

33
tools/gate/playbooks/airskiff-deploy-gate.yaml
View File

@ -12,44 +12,36 @@
# See the License for the specific language governing permissions and
# limitations under the License.
- hosts: ubuntu-bionic
tasks:
# Stop systemd-resolved service before starting docker.
- name: stop systemd-resolved service
systemd:
state: stopped
enabled: no
masked: yes
daemon_reload: yes
name: systemd-resolved
become: yes
- name: ensure pip3 installed
apt:
name: "{{ item }}"
with_items:
- python3-pip
become: yes
- hosts: all
# roles:
# - disable-systemd-resolved
tasks:
- name: Clone dependencies
shell: |
set -ex
./tools/deployment/airskiff/developer/000-clone-dependencies.sh
args:
chdir: "{{ zuul.project.src_dir }}"
- name: Setup AppArmor
shell: |
set -ex
./tools/deployment/airskiff/developer/009-setup-apparmor.sh
args:
chdir: "{{ zuul.project.src_dir }}"
- name: Deploy Kubernetes with Minikube
shell: |
set -ex
./tools/deployment/airskiff/developer/010-deploy-k8s.sh
args:
chdir: "{{ zuul.project.src_dir }}"
- name: Setup OpenStack Client
shell: |
set -ex
./tools/deployment/airskiff/developer/020-setup-client.sh
args:
chdir: "{{ zuul.project.src_dir }}"
@ -57,7 +49,8 @@
- name: Deploy Airship components using Armada
shell: |
mkdir ~/.kube
set -ex
mkdir -p ~/.kube
cp -rp /home/zuul/.kube/config ~/.kube/config
export PL_SITE="{{ site }}"
./tools/deployment/airskiff/developer/030-armada-bootstrap.sh
@ -67,6 +60,7 @@
- name: Deploy Software using Airship
shell: |
set -ex
export PL_SITE="{{ site }}"
./tools/deployment/airskiff/developer/100-deploy-osh.sh
args:
@ -75,7 +69,8 @@
- name: Wait for deployment completion
shell: |
. ./tools/deployment/airskiff/common/os-env.sh
set -ex
./tools/deployment/airskiff/common/os-env.sh
./tools/gate/wait-for-shipyard.sh
args:
chdir: "{{ zuul.project.src_dir }}"

2
tools/gate/playbooks/airskiff-reduce-site.yaml
View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
- hosts: ubuntu-bionic
- hosts: all
tasks:
- name: Replace Armada manifest
shell: |

3
tools/gate/playbooks/debug-report.yaml
View File

@ -12,7 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
- hosts: ubuntu-bionic
- hosts: all
tasks:
- name: Build debug report
shell: |

2
tools/gate/playbooks/generate-certs.yaml
View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
- hosts: ubuntu-bionic
- hosts: all
tasks:
- name: Generate site certificates
shell: |

2
tools/gate/playbooks/git-config.yaml
View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
- hosts: ubuntu-bionic
- hosts: all
tasks:
- name: Git config
shell: |

24
tools/gate/playbooks/install-docker.yaml → tools/gate/playbooks/initial-setup.yaml
View File

@ -12,20 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
- hosts: ubuntu-bionic
tasks:
- name: Install Docker CE
shell: |
set -xe;
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
apt-add-repository \
"deb [arch=amd64] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) \
stable"
apt-get update
apt-get install --no-install-recommends -y docker-ce
args:
chdir: "{{ zuul.project.src_dir }}"
become: yes
- hosts: all
roles:
- bindep
- clear-firewall
- ensure-docker
- ensure-python
- ensure-pip

37
tools/gate/playbooks/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml Normal file
View File

@ -0,0 +1,37 @@
# Copyright 2020 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Disable systemd-resolved service
systemd:
state: stopped
enabled: no
masked: yes
daemon_reload: yes
name: systemd-resolved
become: yes
- name: Remove local stub dns from resolv.conf, if it exists
lineinfile:
path: /etc/resolv.conf
state: absent
regexp: '^nameserver.*127.0.0.1'
become: yes
- name: Add upstream nameservers in resolv.conf
blockinfile:
path: /etc/resolv.conf
block: |
nameserver 8.8.8.8
nameserver 8.8.4.4
become: yes

15
tools/gate/playbooks/roles/disable-systemd-resolved/tasks/main.yaml Normal file
View File

@ -0,0 +1,15 @@
# Copyright 2020 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- include: disable-systemd-resolved.yaml

2
tools/gate/playbooks/site-lint.yaml
View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
- hosts: ubuntu-bionic
- hosts: all
tasks:
- name: Lint site
shell: |

15
tox.ini
View File

@ -2,21 +2,26 @@
# Allows docs to be built without setup.py having to exist. Requires that
# usedevelop be False as well (which it is by default).
envlist = pep8
minversion = 2.3.1
minversion = 3.28
skipsdist = True
[testenv]
setenv = VIRTUAL_ENV={envdir}
LANGUAGE=en_US
LC_ALL=en_US.utf-8
passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
passenv =
http_proxy
HTTP_PROXY
https_proxy
HTTPS_PROXY
no_proxy
NO_PROXY
[testenv:venv]
commands = {posargs}
[testenv:docs]
basepython = python3
whitelist_externals = rm
allowlist_externals = rm
deps =
-r{toxinidir}/doc/requirements.txt
commands =
@ -24,14 +29,12 @@ commands =
sphinx-build -W -b html doc/source doc/build/html
[testenv:fmt]
basepython = python3
deps =
-r{toxinidir}/test-requirements.txt
commands =
yapf -ir {toxinidir}/tools
[testenv:pep8]
basepython = python3
deps =
-r{toxinidir}/test-requirements.txt
commands =