airship/shipyard
Code Issues Proposed changes

Shipyard upgrade for focal

Browse Source 
- upgraded Airflow to 1.10.15  -
  https://airflow.apache.org/docs/apache-airflow/1.10.15/changelog.html
- disabled xenial, bionic and opensuse images build  gates
- added focal image build  gate
- added focal zuul build node
- adjusted Makefile for focal
- added bindep.txt to utilize bindep zuul base role for zuul build node
  pre-setup
- added focal Dockerfile
- implemented freeze requirements.txt approach like in other Airship
  projects
- removed specific requirements.txt for airflow in favor of using
  requirements-frozen.txt from shipyard_airflow project when building
  airflow docker image
- fixed docker image publishing to Quay
- replaces deprecated LOG.warn with new LOG.warning call
- replaced deprecated body attribute in responce wiht responce.text
  attribute
- update of falcon module deprecated .API call - replaced wiht
  falcon.App call
- deprecated routing.create_http_method_map method replaced with
  routing.map_http_methods
- re-formatted code tabulations based on yapf recommendations
- replaced deprecated protocol attribute in Pytest create_environ() with
  http_version attribute
- replaced deprecated app attribute in Pytest create_environ() with
  root_path attribute
- fixed airflow CLI commands to match 1.10.15 version
- updated zuul gates to work on focal nodes and added focal specific
  node setup items by adding appriate ansible tasks and roles
- uplifted Helm to 3.9.4
- uplifted stable HTK commit id
- updated tox.in to with with tox v4
- uplifted dependences references to other Airship projects
- common python dependences were syncronized with other Airship
  projects(Promenade, Deckhand, Armada, Drydock)
- fixed airskiff deployment gate
- fixed genconfig* profiles in shipyard-airflow tox.ini responsible for
  maintanance of policy.yaml.sample and shipyard.conf.sample

Change-Id: I0c85187dc9bacf0849382563dd5ff7e9b2814c59
This commit is contained in:
Sergiy Markin 2023-03-10 00:20:04 +00:00
parent e3e71f7d16
commit 154a099b28
65 changed files with 686 additions and 478 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
.zuul.yaml
View File

@ -13,7 +13,7 @@
- project:
templates:
- docs-on-readthedocs
- openstack-python36-jobs
- openstack-python38-jobs
vars:
rtd_webhook_id: '38576'
rtd_project_name: 'airship-shipyard'
@ -25,22 +25,16 @@
- airship-shipyard-chart-build-latest-htk
- airship-shipyard-whitespace-lint-gate
- airship-shipyard-airskiff-deployment
- airship-shipyard-image-gate-ubuntu_xenial
- airship-shipyard-image-gate-ubuntu_bionic
- airship-shipyard-image-gate-opensuse
- airship-shipyard-image-gate-ubuntu_focal
gate:
jobs:
- openstack-tox-pep8
- airship-shipyard-chart-build-gate
- airship-shipyard-whitespace-lint-gate
- airship-shipyard-image-gate-ubuntu_xenial
- airship-shipyard-image-gate-ubuntu_bionic
- airship-shipyard-image-gate-opensuse
- airship-shipyard-image-gate-ubuntu_focal
post:
jobs:
- airship-shipyard-docker-build-post-ubuntu_xenial
- airship-shipyard-docker-build-post-ubuntu_bionic
- airship-shipyard-docker-build-post-opensuse
- airship-shipyard-docker-build-post-ubuntu_focal
- shipyard-upload-git-mirror
- nodeset:
@ -49,12 +43,18 @@
- name: primary
label: ubuntu-bionic
- nodeset:
name: airship-shipyard-single-node-focal
nodes:
- name: primary
label: ubuntu-focal
- job:
name: airship-shipyard-chart-build-gate
description: Build charts using pinned Helm toolkit.
timeout: 900
run: tools/gate/playbooks/build-charts.yaml
nodeset: airship-shipyard-single-node
nodeset: airship-shipyard-single-node-focal
- job:
name: airship-shipyard-chart-build-latest-htk
@ -62,7 +62,7 @@
timeout: 900
voting: false
run: tools/gate/playbooks/build-charts.yaml
nodeset: airship-shipyard-single-node
nodeset: airship-shipyard-single-node-focal
vars:
HTK_COMMIT: master
@ -71,7 +71,7 @@
description: |
Lints all files by checking them for whitespace.
run: tools/gate/playbooks/zuul-linter.yaml
nodeset: airship-shipyard-single-node
nodeset: airship-shipyard-single-node-focal
- job:
name: airship-shipyard-image-base
@ -79,44 +79,31 @@
Base job for running airship-shipyard image related jobs.
timeout: 3600
run: tools/gate/playbooks/run-image.yaml
nodeset: airship-shipyard-single-node
nodeset: airship-shipyard-single-node-focal
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- ^releasenotes/.*$
- job:
name: airship-shipyard-image-gate-ubuntu_xenial
voting: false
name: airship-shipyard-image-gate-ubuntu_focal
description: |
Run shipyard-image build for ubuntu_xenial.
Run shipyard-image build for ubuntu_focal.
parent: airship-shipyard-image-base
vars:
publish: false
distro: ubuntu_xenial
tags:
dynamic:
patch_set: true
- job:
name: airship-shipyard-image-gate-ubuntu_bionic
description: |
Run shipyard-image build for ubuntu_bionic.
parent: airship-shipyard-image-base
vars:
publish: false
distro: ubuntu_bionic
distro: ubuntu_focal
tags:
dynamic:
patch_set: true
- job:
name: airship-shipyard-airskiff-deployment
nodeset: airship-shipyard-single-node
nodeset: airship-shipyard-single-node-focal
description: |
Deploy Memcached using Airskiff and submitted Armada changes.
timeout: 9600
voting: false
voting: true
pre-run:
- tools/gate/playbooks/airskiff-reduce-site.yaml
- tools/gate/playbooks/git-config.yaml
@ -144,27 +131,9 @@
dynamic:
patch_set: true
- job:
name: airship-shipyard-docker-build-post-ubuntu_xenial
voting: false
timeout: 1800
run: tools/gate/playbooks/run-image.yaml
nodeset: airship-shipyard-single-node
irrelevant-files: *irrelevant-files
secrets:
- airship_shipyard_quay_creds
vars:
publish: true
distro: ubuntu_xenial
tags:
dynamic:
branch: true
commit: true
static:
- latest
- job:
name: airship-shipyard-docker-build-post-ubuntu_bionic
name: airship-shipyard-docker-build-post-ubuntu_focal
timeout: 1800
run: tools/gate/playbooks/run-image.yaml
nodeset: airship-shipyard-single-node
@ -173,7 +142,7 @@
- airship_shipyard_quay_creds
vars:
publish: true
distro: ubuntu_bionic
distro: ubuntu_focal
tags:
dynamic:
branch: true

16
Makefile
View File

@ -34,7 +34,7 @@ USE_PROXY ?= false
AIRFLOW_SRC ?=
AIRFLOW_HOME ?=
DISTRO_BASE_IMAGE ?=
DISTRO ?= ubuntu_bionic
DISTRO ?= ubuntu_focal
IMAGE:=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$(IMAGE_NAME):${IMAGE_TAG}-${DISTRO}
IMAGE_DIR:=images/$(IMAGE_NAME)
@ -61,7 +61,7 @@ charts: clean helm-toolkit
# Perform Linting
.PHONY: lint
lint: pep8 helm_lint build_docs
lint: pep8 helm-lint build_docs
# Dry run templating of chart
.PHONY: dry-run
@ -129,20 +129,26 @@ clean:
rm -rf doc/build
cd $(BUILD_CTX)/shipyard_client; rm -rf build
cd $(BUILD_CTX)/shipyard_airflow; rm -rf build
rm -rf doc/build
rm -f charts/*.tgz
rm -f charts/*/requirements.lock
rm -rf charts/*/charts
rm -rf .tox
.PHONY: pep8
pep8:
cd $(BUILD_CTX)/shipyard_client; tox -e pep8
cd $(BUILD_CTX)/shipyard_airflow; tox -e pep8
.PHONY: helm_lint
helm_lint: clean helm-toolkit
.PHONY: helm-lint
helm-lint: clean helm-toolkit
$(HELM) dep up charts/shipyard
$(HELM) lint charts/shipyard
# Initialize local helm config
.PHONY: helm-toolkit
helm-toolkit: helm-install
tools/helm_tk.sh $(HELM)
./tools/helm_tk.sh $(HELM)
# Install helm binary
.PHONY: helm-install

10
bindep.txt Normal file
View File

@ -0,0 +1,10 @@
# This is a cross-platform list tracking distribution packages needed by tests;
# see http://docs.openstack.org/infra/bindep/ for additional information.
libffi-dev [test platform:dpkg]
libkrb5-dev [platform:dpkg]
libpq-dev [platform:dpkg]
libsasl2-dev [platform:dpkg]
libssl-dev [platform:dpkg]
libre2-dev [platform:dpkg]
libzmq3-dev [platform:dpkg]

5
doc/requirements.txt
View File

@ -1,5 +1,6 @@
#
# Requirements for creating documentation only.
#
sphinx>=1.6.2
sphinx_rtd_theme==0.2.4
Sphinx
sphinx-rtd-theme==0.5.0
Jinja2<=3.0.3

1
doc/source/_static/shipyard.policy.yaml.sample
View File

@ -106,4 +106,3 @@
# targeted release
# POST /api/v1.0/actions
#"workflow_orchestrator:action_test_site": "rule:admin_required"

23
images/airflow/Dockerfile.ubuntu_xenial → images/airflow/Dockerfile.ubuntu_focal
View File

@ -13,7 +13,7 @@
# limitations under the License.
# Docker image to run Airflow on Kubernetes
ARG FROM=ubuntu:16.04
ARG FROM=ubuntu:focal
FROM ${FROM}
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
@ -43,12 +43,12 @@ ARG AIRFLOW_HOME=/usr/local/airflow
# Moved celery to images/airflow/requirements.txt as apache-airflow uses a
# version of celery incompatibile with the version of kombu needed by other
# Airship components
ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs,jdbc]==1.10.5"
ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs]==1.10.15"
ARG DEBIAN_FRONTEND=noninteractive
ARG ctx_base=src/bin
# Kubectl version
ARG KUBECTL_VERSION=1.24.6
ARG KUBECTL_VERSION=1.26.3
# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
ENV AIRFLOW_HOME=${AIRFLOW_HOME}
@ -61,22 +61,23 @@ RUN set -ex && \
curl \
git \
g++ \
libkrb5-dev \
libffi-dev \
libssl-dev \
libpq-dev \
libtool \
libsasl2-dev \
locales \
netcat \
netbase \
python3 \
python3-setuptools \
python3-pip \
python3-dev \
python3-dateutil \
python3-openssl \
make \
--no-install-recommends \
&& python3 -m pip install -U 'pip<21.0' \
&& python3 -m pip install -U 'setuptools<58' \
&& python3 -m pip install -U pip \
&& apt-get clean \
&& rm -rf \
/var/lib/apt/lists/* \
@ -111,9 +112,9 @@ RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
# Note - removing snakebite (python 2 vs. 3). See:
# https://github.com/puckel/docker-airflow/issues/77
# Install Airflow directly to allow overriding source
COPY images/airflow/requirements.txt /tmp/
RUN sed -i "s/pyOpenSSL==.*//" /tmp/requirements.txt \
&& pip3 install -r /tmp/requirements.txt --no-cache-dir \
# COPY images/airflow/requirements.txt /tmp/
COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/requirements.txt
RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \
&& pip3 install $AIRFLOW_SRC --no-cache-dir \
&& (pip3 uninstall -y snakebite || true) \
&& (pip3 uninstall -y psycopg2 || true) \
@ -137,12 +138,12 @@ ENV PBR_VERSION 0.1a1
# Since Shipyard and Airflow are built together as images, this should prevent
# stale or out-of-date code between these parts.
# Shipyard requirements, source and installation
COPY ${ctx_base}/shipyard_airflow/requirements.txt /tmp/api_requirements.txt
COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/api_requirements.txt
RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
RUN cd /tmp/shipyard \
&& python3 setup.py install
&& pip3 install $(pwd)
# Note: The value for the dags and plugins directories that are sourced
# from the values.yaml of the Shipyard Helm chart need to align with these

44
images/airflow/requirements.txt
View File

@ -1,44 +0,0 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
celery==4.3
pytz==2018.5
python-dateutil==2.8.1
pyOpenSSL==22.0.0
ndg-httpsclient==0.5.1
oslo.utils==3.42.1
oslo.config==7.0.0
oslo.serialization==2.29.2
pyasn1<0.5.0,>=0.4.6
psycopg2-binary==2.8.4
docker==3.7.2
# Airflow is now installed in the Dockerfile directory to allow for
# overriding where it is sourced from
python-openstackclient==3.16.1
python-keystoneclient==3.22.0
kubernetes>=6.0.0
# Need to lock marshmellow-sqlalchemy and tabulate for compatibility issues
marshmallow-sqlalchemy==0.18.0
WTForms~=2.2.1
tabulate==0.8.03
pbr==5.4.5
setuptools>=40.4.3,<58
six>=1.15.0
urllib3==1.25.9
vine<5.0.0a1,>=1.1.3
# Dependencies for other UCP components
git+https://opendev.org/airship/deckhand.git@1f0c011a1708c1235a2be65edada7d386cd55d2a#egg=deckhand
git+https://opendev.org/airship/drydock.git@96db568d4b28f5f163efbf606854243614a6c821#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@416eff4e52f72169c3ebf49a8ffca36086692b7a#egg=armada

4
images/airflow/script/airflow_start_service.sh
View File

@ -41,9 +41,9 @@ elif [[ $cmd == *scheduler* ]]; then
elif [[ $cmd == 'quicktest' ]]; then
${python3_path} ${airflow_path} initdb
${python3_path} ${airflow_path} webserver -p 8080 &
airflow run example_bash_operator runme_0 2018-01-01
airflow tasks run example_bash_operator runme_0 2018-01-01
airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
airflow dag_state example_bash_operator 2018-01-01
airflow dags state example_bash_operator 2018-01-01
else
echo "Invalid Command!"
exit 1

33
images/shipyard/Dockerfile.ubuntu_xenial → images/shipyard/Dockerfile.ubuntu_focal
View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
ARG FROM=ubuntu:16.04
ARG FROM=ubuntu:focal
FROM ${FROM}
LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc://#airshipit@freenode' \
@ -41,11 +41,13 @@ RUN set -ex && \
automake \
ca-certificates \
curl \
build-essential \
git \
libtool \
make \
netbase \
python3-dev \
python3-setuptools \
--no-install-recommends \
&& apt-get autoremove -yqq --purge \
&& apt-get clean \
@ -84,33 +86,36 @@ RUN chown -R shipyard: /home/shipyard \
&& chmod +x /home/shipyard/entrypoint.sh
# Requirements and Shipyard source
COPY ${ctx_base}/shipyard_airflow/requirements.txt /home/shipyard/api_requirements.txt
COPY ${ctx_base}/shipyard_client/requirements.txt /home/shipyard/client_requirements.txt
COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /home/shipyard/api_requirements.txt
COPY ${ctx_base}/shipyard_client/requirements-frozen.txt /home/shipyard/client_requirements.txt
COPY ${ctx_base}/shipyard_client /home/shipyard/shipyard_client/
COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/
# Build
RUN set -ex \
&& buildDeps=' \
gcc \
build-essential \
git \
libssl-dev \
libffi-dev \
libkrb5-dev \
libpq-dev \
libsasl2-dev \
libssl-dev \
libre2-dev \
libzmq3-dev \
make \
python3-pip \
' \
&& apt-get -qq update \
&& apt-get -y install -y \
$buildDeps \
python3-openssl \
libffi-dev \
python3-dev \
--no-install-recommends \
&& python3 -m pip install -U 'pip<21.0' \
&& python3 -m pip install -U setuptools \
&& sed -i "s/pyOpenSSL==.*//" /home/shipyard/client_requirements.txt\
&& apt-get -y install -y $buildDeps --no-install-recommends \
&& python3 -m pip install -U pip \
&& pip3 install -r /home/shipyard/client_requirements.txt --no-cache-dir \
&& cd /home/shipyard/shipyard_client \
&& pip3 install $(pwd) --use-pep517 \
&& pip3 install -r /home/shipyard/api_requirements.txt --no-cache-dir \
&& cd /home/shipyard/shipyard \
&& python3 setup.py install \
&& pip3 install $(pwd) --use-pep517 \
&& apt-get purge -y --auto-remove $buildDeps \
&& apt-get autoremove -yqq --purge \
&& apt-get clean \

55
src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample
View File

@ -180,9 +180,13 @@
# and will be removed in the S release.
#auth_uri = <None>
# API version of the admin Identity API endpoint. (string value)
# API version of the Identity API endpoint. (string value)
#auth_version = <None>
# Interface to use for the Identity API endpoint. Valid values are "public",
# "internal" (default) or "admin". (string value)
#interface = internal
# Do not handle authorization requests within the middleware, but delegate the
# authorization decision to downstream WSGI components. (boolean value)
#delay_auth_decision = false
@ -217,14 +221,6 @@
# The region in which the identity server can be found. (string value)
#region_name = <None>
# DEPRECATED: Directory used to cache files related to PKI tokens. This option
# has been deprecated in the Ocata release and will be removed in the P
# release. (string value)
# This option is deprecated for removal since Ocata.
# Its value may be silently ignored in the future.
# Reason: PKI token format is no longer supported.
#signing_dir = <None>
# Optionally specify a list of memcached server(s) to use for caching. If left
# undefined, tokens will instead be cached in-process. (list value)
# Deprecated group/name - [keystone_authtoken]/memcache_servers
@ -235,16 +231,6 @@
# to -1 to disable caching completely. (integer value)
#token_cache_time = 300
# DEPRECATED: Determines the frequency at which the list of revoked tokens is
# retrieved from the Identity service (in seconds). A high number of revocation
# events combined with a low cache duration may significantly reduce
# performance. Only valid for PKI tokens. This option has been deprecated in
# the Ocata release and will be removed in the P release. (integer value)
# This option is deprecated for removal since Ocata.
# Its value may be silently ignored in the future.
# Reason: PKI token format is no longer supported.
#revocation_cache_time = 10
# (Optional) If defined, indicate whether token data should be authenticated or
# authenticated and encrypted. If MAC, token data is authenticated (with HMAC)
# in the cache. If ENCRYPT, token data is encrypted and authenticated in the
@ -280,9 +266,9 @@
# client connection from the pool. (integer value)
#memcache_pool_conn_get_timeout = 10
# (Optional) Use the advanced (eventlet safe) memcached client pool. The
# advanced pool will only work under python 2.x. (boolean value)
#memcache_use_advanced_pool = false
# (Optional) Use the advanced (eventlet safe) memcached client pool. (boolean
# value)
#memcache_use_advanced_pool = true
# (Optional) Indicate whether to set the X-Service-Catalog header. If False,
# middleware will not ask for service catalog on token validation and will not
@ -298,27 +284,6 @@
# (string value)
#enforce_token_bind = permissive
# DEPRECATED: If true, the revocation list will be checked for cached tokens.
# This requires that PKI tokens are configured on the identity server. (boolean
# value)
# This option is deprecated for removal since Ocata.
# Its value may be silently ignored in the future.
# Reason: PKI token format is no longer supported.
#check_revocations_for_cached = false
# DEPRECATED: Hash algorithms to use for hashing PKI tokens. This may be a
# single algorithm or multiple. The algorithms are those supported by Python
# standard hashlib.new(). The hashes will be tried in the order given, so put
# the preferred one first for performance. The result of the first hash will be
# stored in the cache. This will typically be set to multiple values only while
# migrating from a less secure algorithm to a more secure one. Once all the old
# tokens are expired this option should be set to a single value for better
# performance. (list value)
# This option is deprecated for removal since Ocata.
# Its value may be silently ignored in the future.
# Reason: PKI token format is no longer supported.
#hash_algorithms = md5
# A choice of roles that must be present in a service token. Service tokens are
# allowed to request that an expired token can be used and so this check should
# tightly control that only actual services should be sending this token. Roles
@ -333,6 +298,10 @@
# possible. (boolean value)
#service_token_roles_required = false
# The name or type of the service as it appears in the service catalog. This is
# used to validate tokens that have restricted access rules. (string value)
#service_type = <None>
# Authentication type to load (string value)
# Deprecated group/name - [keystone_authtoken]/auth_plugin
#auth_type = <None>

29
src/bin/shipyard_airflow/requirements-direct.txt Normal file
View File

@ -0,0 +1,29 @@
arrow<=0.17.0
# apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.15
apache-airflow[crypto,celery,hive,hdfs]==1.10.15
coverage==5.3
falcon
jsonschema<=3.2.0
keystoneauth1<=5.1.1
kubernetes==26.1.0
networkx
oslo.config<=8.7.1
oslo.policy<=3.10.1
oslo.utils<=4.12.3
pyarrow==6.0.1
pylibyaml==0.1.0
PyYAML<=5.4.1
regex==2020.11.13
requests==2.23.0
responses==0.12.1
setuptools<=45.2.0
SQLAlchemy<=1.3.20
ulid==1.1
uWSGI==2.0.21
Werkzeug
WTForms<=2.3.3
git+https://opendev.org/airship/deckhand.git@13c5199f18664d66ccdd3a1e54bb40ad2c293d1a#egg=deckhand
git+https://opendev.org/airship/drydock.git@d00eaf0303f50478d6503c2c2096e931134189ac#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@a6dcf17d178a89bad626b0e5ef6fafc428dca7e4#egg=armada
git+https://opendev.org/airship/promenade.git@5602e85fb936eeafdc3e4699fd3afbe1576078bc#egg=promenade

255
src/bin/shipyard_airflow/requirements-frozen.txt Normal file
View File

@ -0,0 +1,255 @@
alabaster==0.7.13
alembic==1.4.3
amqp==2.6.1
apache-airflow==1.10.15
apispec==1.3.3
argcomplete==1.12.3
Armada @ git+https://opendev.org/airship/armada.git@a6dcf17d178a89bad626b0e5ef6fafc428dca7e4#egg=armada
arrow==0.17.0
attrs==20.3.0
autopage==0.5.1
Babel==2.12.1
bandit==1.6.0
bcrypt==4.0.1
Beaker==1.12.0
billiard==3.6.4.0
cached-property==1.5.2
cachetools==5.3.0
cattrs==1.10.0
celery==4.4.7
certifi==2022.12.7
cffi==1.15.1
chardet==3.0.4
charset-normalizer==3.1.0
click==7.1.2
cliff==3.10.1
cmd2==2.4.3
colorama==0.4.6
colorlog==4.0.2
configparser==3.5.3
coverage==5.3
croniter==0.3.37
cryptography==3.4.8
debtcollector==2.5.0
Deckhand @ git+https://opendev.org/airship/deckhand.git@13c5199f18664d66ccdd3a1e54bb40ad2c293d1a#egg=deckhand
decorator==5.1.1
deepdiff==5.8.1
defusedxml==0.7.1
dill==0.3.6
distlib==0.3.6
dnspython==2.3.0
docutils==0.17
dogpile.cache==1.2.0
drydock-provisioner @ git+https://opendev.org/airship/drydock.git@d00eaf0303f50478d6503c2c2096e931134189ac#egg=drydock_provisioner&subdirectory=python
dulwich==0.21.3
email-validator==1.3.1
eventlet==0.33.3
extras==1.0.0
falcon==3.1.1
fasteners==0.18
filelock==3.12.0
fixtures==3.0.0
flake8==3.8.4
Flask==1.1.4
Flask-Admin==1.5.4
Flask-AppBuilder==2.3.4
Flask-Babel==1.0.0
Flask-Caching==1.3.3
Flask-JWT-Extended==3.25.1
Flask-Login==0.4.1
Flask-OpenID==1.3.0
Flask-SQLAlchemy==2.5.1
flask-swagger==0.2.14
Flask-WTF==0.14.3
flower==0.9.7
funcsigs==1.0.2
future==0.18.3
futurist==2.4.1
gitdb==4.0.10
GitPython==3.1.31
google-auth==2.17.3
graphviz==0.20.1
greenlet==2.0.2
gunicorn==20.1.0
hacking==4.1.0
hmsclient==0.1.1
html5lib==0.9999999
httpexceptor==1.4.0
humanize==4.6.0
idna==2.10
imagesize==1.4.1
importlib-metadata==2.1.3
importlib-resources==1.5.0
iso8601==1.1.0
itsdangerous==1.1.0
Jinja2==2.11.3
json-merge-patch==0.2
jsonpath-ng==1.5.3
jsonpath-rw==1.4.0
jsonpath-rw-ext==1.2.2
jsonpickle==1.4.1
jsonschema==3.2.0
keystoneauth1==5.1.1
keystonemiddleware==10.2.0
kombu==4.6.11
kubernetes==26.1.0
lazy-object-proxy==1.4.3
lockfile==0.12.2
Mako==1.2.4
Markdown==2.6.11
MarkupSafe==2.0.1
marshmallow==2.21.0
marshmallow-enum==1.5.1
marshmallow-sqlalchemy==0.23.1
mccabe==0.6.1
mock==5.0.2
more-itertools==9.1.0
msgpack==1.0.5
natsort==8.3.1
netaddr==0.8.0
netifaces==0.11.0
networkx==3.1
nose==1.3.7
numpy==1.24.3
oauthlib==3.2.2
ordered-set==4.1.0
os-service-types==1.7.0
oslo.cache==2.10.1
oslo.concurrency==5.1.1
oslo.config==8.7.1
oslo.context==4.1.0
oslo.db==10.0.0
oslo.i18n==6.0.0
oslo.log==4.6.0
oslo.messaging==12.13.0
oslo.metrics==0.6.0
oslo.middleware==4.4.0
oslo.policy==3.10.1
oslo.serialization==4.2.0
oslo.service==3.1.1
oslo.utils==4.12.3
packaging==21.3
pandas==1.5.3
Paste==3.5.0
PasteDeploy==3.0.1
PasteScript==3.3.0
pbr==5.5.1
pendulum==1.4.4
pip==23.0.1
platformdirs==3.4.0
pluggy==0.6.0
ply==3.11
prettytable==3.7.0
prison==0.2.1
promenade @ git+https://opendev.org/airship/promenade.git@5602e85fb936eeafdc3e4699fd3afbe1576078bc#egg=promenade
prometheus-client==0.8.0
protobuf==4.22.3
psutil==5.9.5
psycopg2-binary==2.9.6
pure-sasl==0.6.2
py==1.11.0
pyarrow==6.0.1
pyasn1==0.5.0
pyasn1-modules==0.3.0
pycadf==3.1.1
pycodestyle==2.6.0
pycparser==2.21
pyflakes==2.2.0
Pygments==2.14.0
PyHive==0.6.5
PyJWT==1.7.1
pylibyaml==0.1.0
pymongo==4.3.3
pyparsing==2.4.7
pyperclip==1.8.2
pyproject_api==1.5.0
pyrsistent==0.19.3
pytest==3.5.0
pytest-cov==2.5.1
python-barbicanclient==5.2.0
python-daemon==2.3.2
python-dateutil==2.8.1
python-editor==1.0.4
python-keystoneclient==5.1.0
python-memcached==1.59
python-mimeparse==1.6.0
python-nvd3==0.15.0
python-slugify==4.0.1
python-subunit==1.4.0
python3-openid==3.2.0
pytz==2023.3
pytzdata==2020.1
PyYAML==5.4.1
regex==2020.11.13
reno==4.0.0
repoze.lru==0.7
requests==2.23.0
requests-oauthlib==1.3.1
resolver==0.2.1
responses==0.12.1
retry==0.9.2
rfc3986==2.0.0
Routes==2.5.1
rsa==4.9
sasl==0.3.1
selector==0.10.1
setproctitle==1.3.2
setuptools==45.2.0
simplejson==3.19.1
six==1.16.0
smmap==5.0.0
snakebite-py3==3.0.5
snowballstemmer==2.2.0
Sphinx==3.3.1
sphinx-rtd-theme==0.5.0
sphinxcontrib-applehelp==1.0.4
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.1
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
SQLAlchemy==1.3.20
SQLAlchemy-JSONField==0.9.0
sqlalchemy-migrate==0.13.0
SQLAlchemy-Utils==0.41.0
sqlparse==0.4.4
statsd==4.0.1
stevedore==5.0.0
tabulate==0.8.10
Tempita==0.5.2
tenacity==4.12.0
testfixtures==7.1.0
testrepository==0.0.20
testresources==2.0.1
testscenarios==0.5.0
testtools==2.5.0
text-unidecode==1.3
thrift==0.16.0
thrift-sasl==0.4.3
tiddlyweb==2.4.3
toml==0.10.2
tomli==2.0.1
tomlkit==0.11.7
tornado==5.1.1
tox==3.12.1
typing-extensions==3.7.2
tzlocal==1.5.1
ulid==1.1
unicodecsv==0.14.1
urllib3==1.25.11
uWSGI==2.0.21
vine==1.3.0
virtualenv==20.22.0
wcwidth==0.2.6
WebOb==1.8.7
websocket-client==1.5.1
Werkzeug==0.16.1
wheel==0.40.0
wrapt==1.15.0
wsgi-intercept==1.11.0
WTForms==2.3.3
xmltodict==0.13.0
yappi==1.4.0
yq==3.2.1
zipp==3.15.0
zope.deprecation==4.4.0

64
src/bin/shipyard_airflow/requirements.txt
View File

@ -1,61 +1,3 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# API requirements
alembic==1.0.1
arrow==0.12.1
celery==4.3
kombu<4.7,>=4.6.10
vine<5.0.0a1,>=1.1.3
configparser==3.5.0
cryptography>=2.7
falcon==1.4.1
jsonschema>=3.0.1,<4
grpcio>=1.16.0
keystoneauth1>=3.18.0
keystonemiddleware==5.3.0
networkx==2.2 # common/deployment_group
oslo.config==7.0.0
oslo.policy==1.40.1
PasteDeploy==1.5.2
psycopg2-binary==2.8.4
pylibyaml~=0.1
oslo.cache==1.38.1
oslo.log==3.45.2
oslo.utils==3.42.1
oslo.serialization==2.29.2
pbr==5.4.5
python-dateutil==2.8.1
python-memcached==1.59
python-keystoneclient==3.22.0
requests!=2.20.0,>=2.14.2
setuptools==40.4.3
SQLAlchemy==1.3.15
ulid==1.1
urllib3==1.25.9
uwsgi~=2.0.19.1
# To support profiling in non-prod
Werkzeug==0.16.1
WTForms~=2.2.1
# Need to lock marshmellow-sqlalchemy and tabulate for compatibility issues
marshmallow-sqlalchemy==0.18.0
tabulate==0.8.03
# Dependencies for other UCP components
git+https://opendev.org/airship/deckhand.git@5cd799cc5d04527ac782270008ff647b3779ff05#egg=deckhand
git+https://opendev.org/airship/drydock.git@b431f7a2cfacd199fa5587487f372dc5589f9126#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@379c88d6195eed1a0ef61d0465d5746bcfda1a42#egg=armada
git+https://opendev.org/airship/promenade.git@c10165c144e0a18137596a3c89f1339d6ed30d0c#egg=promenade
# Warning: This file should be empty.
# Specify direct dependencies in requirements-direct.txt instead.
-r requirements-direct.txt

4
src/bin/shipyard_airflow/setup.cfg
View File

@ -12,8 +12,8 @@ classifier =
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 3
Programming Language :: Python :: 3.5
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.10
[files]
packages =

2
src/bin/shipyard_airflow/shipyard_airflow/common/document_validators/document_validator.py
View File

@ -117,7 +117,7 @@ class DocumentValidator(metaclass=abc.ABCMeta):
Triggers the specific checks after any common checks
"""
if self.missing_severity not in ["Error", "Warning", "Info"]:
LOG.warn("Document Validator for {}, {} does not have a valid "
LOG.warning("Document Validator for {}, {} does not have a valid "
"value set for missing_severity (got {}). "
"Assuming Error".format(
self.schema, self.doc_name, self.missing_severity

4
src/bin/shipyard_airflow/shipyard_airflow/common/notes/notes_helper.py
View File

@ -147,7 +147,7 @@ class NotesHelper:
note_timestamp=note_timestamp
)
except Exception as ex:
LOG.warn(
LOG.warning(
"Creating note for {} encountered a problem, exception info "
"follows, but processing is not halted for notes.",
assoc_id
@ -163,7 +163,7 @@ class NotesHelper:
q = Query(assoc_id_pattern, verbosity, exact_match)
return self.nm.retrieve(q)
except Exception as ex:
LOG.warn(
LOG.warning(
"Note retrieval for {} encountered a problem, exception "
"info follows, but processing is not halted for notes.",
assoc_id_pattern

4
src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py
View File

@ -109,7 +109,7 @@ class ActionsResource(BaseResource):
Return actions that have been invoked through shipyard.
:returns: a json array of action entities
"""
resp.body = self.to_json(self.get_all_actions(
resp.text = self.to_json(self.get_all_actions(
verbosity=req.context.verbosity)
)
resp.status = falcon.HTTP_200
@ -133,7 +133,7 @@ class ActionsResource(BaseResource):
LOG.info("Id %s generated for action %s", action['id'], action['name'])
# respond with the action and location for checking status
resp.status = falcon.HTTP_201
resp.body = self.to_json(action)
resp.text = self.to_json(action)
resp.location = '/api/v1.0/actions/{}'.format(action['id'])
def create_action(self, action, context, allow_intermediate_commits=False):

2
src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_id_api.py
View File

@ -36,7 +36,7 @@ class ActionsIdResource(BaseResource):
Return actions that have been invoked through shipyard.
:returns: a json array of action entities
"""
resp.body = self.to_json(self.get_action(
resp.text = self.to_json(self.get_action(
action_id=kwargs['action_id'],
verbosity=req.context.verbosity
))

2
src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_steps_id_api.py
View File

@ -32,7 +32,7 @@ class ActionsStepsResource(BaseResource):
Return step details for an action step
:returns: a json object describing a step
"""
resp.body = self.to_json(
resp.text = self.to_json(
self.get_action_step(
action_id=kwargs['action_id'],
step_id=kwargs['step_id'],

2
src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_steps_id_logs_api.py
View File

@ -51,7 +51,7 @@ class ActionsStepsLogsResource(BaseResource):
step_id = ActionsHelper.parse_step_id(**kwargs)
# Retrieve logs for the action step
resp.body = self.get_action_step_logs(action_id,
resp.text = self.get_action_step_logs(action_id,
step_id,
try_number)

2
src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_validations_id_api.py
View File

@ -31,7 +31,7 @@ class ActionsValidationsResource(BaseResource):
Return validation details for an action validation
:returns: a json object describing a validation
"""
resp.body = self.to_json(
resp.text = self.to_json(
self.get_action_validation(kwargs['action_id'],
kwargs['validation_id']))
resp.status = falcon.HTTP_200

4
src/bin/shipyard_airflow/shipyard_airflow/control/af_monitoring/workflows_api.py
View File

@ -38,7 +38,7 @@ class WorkflowResource(BaseResource):
"""
since_date = req.params.get('since')
helper = WorkflowHelper(req.context.external_marker)
resp.body = self.to_json(
resp.text = self.to_json(
self.get_all_workflows(helper=helper, since_date=since_date)
)
resp.status = falcon.HTTP_200
@ -67,7 +67,7 @@ class WorkflowIdResource(BaseResource):
:returns: a json object of a workflow entity
"""
helper = WorkflowHelper(req.context.external_marker)
resp.body = self.to_json(
resp.text = self.to_json(
self.get_workflow_detail(helper=helper, workflow_id=workflow_id)
)
resp.status = falcon.HTTP_200

4
src/bin/shipyard_airflow/shipyard_airflow/control/api.py
View File

@ -57,7 +57,7 @@ def start_api():
LoggingMiddleware(),
CommonParametersMiddleware()
]
control_api = falcon.API(
control_api = falcon.App(
request_type=ShipyardRequest, middleware=middlewares)
control_api.add_route('/versions', VersionsResource())
@ -113,7 +113,7 @@ class VersionsResource(BaseResource):
"""
def on_get(self, req, resp):
resp.body = self.to_json({
resp.text = self.to_json({
'v1.0': {
'path': '/api/v1.0',
'status': 'stable'

2
src/bin/shipyard_airflow/shipyard_airflow/control/base.py
View File

@ -33,7 +33,7 @@ class BaseResource(object):
"""
def on_options(self, req, resp, **kwargs):
"""Handle options requests"""
method_map = routing.create_http_method_map(self)
method_map = routing.map_http_methods(self)
for method in method_map:
if method_map.get(method).__name__ != 'method_not_allowed':
resp.append_header('Allow', method)

8
src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/configdocs_api.py
View File

@ -48,7 +48,7 @@ class ConfigDocsStatusResource(BaseResource):
"""Returns a list of the configdocs and their statuses"""
versions = req.params.get('versions') or None
helper = ConfigdocsHelper(req.context)
resp.body = self.to_json(helper.get_configdocs_status(versions))
resp.text = self.to_json(helper.get_configdocs_status(versions))
resp.status = falcon.HTTP_200
@ -90,7 +90,7 @@ class ConfigDocsResource(BaseResource):
if validations and validations['status'] == 'Success':
validations['code'] = resp.status
resp.location = '/api/v1.0/configdocs/{}'.format(collection_id)
resp.body = self.to_json(validations)
resp.text = self.to_json(validations)
def validate_content_length(self, content_length):
"""Validates that the content length header is valid
@ -131,7 +131,7 @@ class ConfigDocsResource(BaseResource):
policy.check_auth(req.context, policy.GET_CONFIGDOCS_CLRTXT)
# Not reformatting to JSON or YAML since just passing through
resp.body = self.get_collection(
resp.text = self.get_collection(
helper=helper, collection_id=collection_id, version=version,
cleartext_secrets=cleartext_secrets)
resp.append_header('Content-Type', 'application/x-yaml')
@ -277,7 +277,7 @@ class CommitConfigDocsResource(BaseResource):
dryrun = req.get_param_as_bool(name='dryrun') or False
helper = ConfigdocsHelper(req.context)
validations = self.commit_configdocs(helper, force, dryrun)
resp.body = self.to_json(validations)
resp.text = self.to_json(validations)
resp.status = validations.get('code', falcon.HTTP_200)
def commit_configdocs(self, helper, force, dryrun):

2
src/bin/shipyard_airflow/shipyard_airflow/control/configdocs/rendered_configdocs_api.py
View File

@ -52,7 +52,7 @@ class RenderedConfigDocsResource(BaseResource):
policy.check_auth(req.context,
policy.GET_RENDEREDCONFIGDOCS_CLRTXT)
resp.body = self.get_rendered_configdocs(
resp.text = self.get_rendered_configdocs(
helper=helper,
version=version,
cleartext_secrets=cleartext_secrets

5
src/bin/shipyard_airflow/shipyard_airflow/control/middleware/common_params.py
View File

@ -49,7 +49,10 @@ class CommonParametersMiddleware(object):
try:
verbosity = req.get_param_as_int(
'verbosity', required=False, min=0, max=MAX_VERBOSITY
'verbosity',
required=False,
min_value=0,
max_value=MAX_VERBOSITY
)
if verbosity is not None:
# if not set, retains the context default value.

2
src/bin/shipyard_airflow/shipyard_airflow/control/middleware/logging_mw.py
View File

@ -60,7 +60,7 @@ class LoggingMiddleware(object):
# have response scrubbing or way to categorize responses in the
# future, this may be an appropriate place to utilize it.
if resp_code >= 400:
LOG.debug('Errored Response body: %s', resp.body)
LOG.debug('Errored Response body: %s', resp.text)
def _log_headers(self, headers):
""" Log request headers, while scrubbing sensitive values

2
src/bin/shipyard_airflow/shipyard_airflow/control/notes/notedetails_api.py
View File

@ -47,7 +47,7 @@ class NoteDetailsResource(BaseResource):
note_id = kwargs['note_id']
self.validate_note_id(note_id)
note = self.get_note_with_access_check(req.context, note_id)
resp.body = self.get_note_details(note)
resp.text = self.get_note_details(note)
resp.status = falcon.HTTP_200
def validate_note_id(self, note_id):

2
src/bin/shipyard_airflow/shipyard_airflow/control/status/status_api.py
View File

@ -42,5 +42,5 @@ class StatusResource(BaseResource):
else:
fltrs = None
helper = StatusHelper(req.context)
resp.body = self.to_json(helper.get_site_statuses(fltrs))
resp.text = self.to_json(helper.get_site_statuses(fltrs))
resp.status = falcon.HTTP_200

3
src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_deployment_action.py
View File

@ -57,7 +57,8 @@ class ValidateDeploymentAction:
results = self.doc_val_mgr.validate()
if self.doc_val_mgr.errored:
if self.cont_on_fail:
LOG.warn("Validation failures occured, but 'continue-on-fail' "
LOG.warning("Validation failures occured, "
"but 'continue-on-fail' "
"is set to true. Processing continues")
else:
raise ApiError(

2
src/bin/shipyard_airflow/shipyard_airflow/errors.py
View File

@ -104,7 +104,7 @@ def format_error_resp(req,
'retry': retry
}
resp.body = json.dumps(error_response, default=str)
resp.text = json.dumps(error_response, default=str)
resp.content_type = 'application/json'
resp.status = status_code

11
src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_base_operator.py
View File

@ -404,7 +404,8 @@ class DrydockBaseOperator(UcpBaseOperator):
task_status = task_dict.get('status', "Not Specified")
task_result = task_dict.get('result')
if task_result is None:
LOG.warn("Task result is missing for task %s, with status %s."
LOG.warning("Task result is missing for task %s, "
"with status %s."
" Neither successes nor further details can be"
" extracted from this result",
task_id, task_status)
@ -429,7 +430,7 @@ class DrydockBaseOperator(UcpBaseOperator):
", ".join(success_nodes), task_id)
except KeyError:
# missing key on the path to getting nodes - don't add
LOG.warn(
LOG.warning(
"Missing successes field on result of task %s, "
"but a success field was expected. No successes"
" can be extracted from this result", task_id
@ -448,7 +449,7 @@ class DrydockBaseOperator(UcpBaseOperator):
except Exception:
# since we are reporting task results, if we can't get the
# results, do not block the processing.
LOG.warn("Failed to retrieve a result for task %s. Exception "
LOG.warning("Failed to retrieve a result for task %s. Exception "
"follows:", task_id, exc_info=True)
# deduplicate and return
@ -507,7 +508,7 @@ class DrydockBaseOperator(UcpBaseOperator):
note_timestamp=msg.get('ts'),
verbosity=3)
except Exception as ex:
LOG.warn("Error while creating a task result note, "
LOG.warning("Error while creating a task result note, "
"processing continues. Source info %s", msg)
LOG.exception(ex)
@ -528,7 +529,7 @@ class DrydockBaseOperator(UcpBaseOperator):
is_auth_link=True,
verbosity=5)
except Exception as ex:
LOG.warn("Error while creating a link-based note, "
LOG.warning("Error while creating a link-based note, "
"processing continues. Source info: %s", link)
LOG.exception(ex)

3
src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_destroy_nodes.py
View File

@ -78,7 +78,8 @@ class DrydockDestroyNodeOperator(DrydockBaseOperator):
"destroyed. The report at the end of processing "
"this step contains the results", task_name)
except DrydockTaskTimeoutException:
LOG.warn("Task %s has timed out after %s seconds. Some nodes may "
LOG.warning("Task %s has timed out after %s seconds. "
"Some nodes may "
"have been destroyed. The report at the end of "
"processing this step contains the results", task_name,
self.dest_timeout)

4
src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_nodes.py
View File

@ -252,7 +252,7 @@ class DrydockNodesOperator(DrydockBaseOperator):
except DrydockTaskFailedException:
# Task failure may be successful enough based on success criteria.
# This should not halt the overall flow of this workflow step.
LOG.warn(
LOG.warning(
"Task %s with Drydock task-id: %s has failed. Logs contain "
"details of the failure. Some nodes may be succesful, "
"processing continues", task_name, self.drydock_task_id
@ -263,7 +263,7 @@ class DrydockNodesOperator(DrydockBaseOperator):
except DrydockTaskTimeoutException:
# Task timeout may be successful enough based on success criteria.
# This should not halt the overall flow of this workflow step.
LOG.warn(
LOG.warning(
"Task %s with Drydock task-id: %s has timed out after %s "
"seconds. Logs contain details of the failure. Some nodes may "
"be succesful, processing continues", task_name,

3
src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_relabel_nodes.py
View File

@ -81,7 +81,8 @@ class DrydockRelabelNodesOperator(DrydockBaseOperator):
"relabeled. The report at the end of processing "
"this step contains the results", task_name)
except DrydockTaskTimeoutException:
LOG.warn("Task %s has timed out after %s seconds. Some nodes may "
LOG.warning("Task %s has timed out after %s seconds. "
"Some nodes may "
"have been relabeled. The report at the end of "
"processing this step contains the results", task_name,
self.task_timeout)

3
src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_validate_design.py
View File

@ -66,7 +66,8 @@ class DrydockValidateDesignOperator(DrydockBaseOperator):
try:
design_validate_response = requests.post(validation_endpoint,
headers=headers,
data=json.dumps(payload))
data=json.dumps(payload),
timeout=5)
except requests.exceptions.RequestException as e:
raise AirflowException(e)

3
src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py
View File

@ -66,7 +66,8 @@ class PromenadeValidateSiteDesignOperator(PromenadeBaseOperator):
try:
design_validate_response = requests.post(validation_endpoint,
headers=headers,
data=json.dumps(payload))
data=json.dumps(payload),
timeout=5)
except requests.exceptions.RequestException as e:
raise AirflowException(e)

23
src/bin/shipyard_airflow/test-requirements.txt
View File

@ -1,21 +1,8 @@
# Testing
pyflakes>=2.1.1
amqp~=2.6.0
bandit==1.6.0
flake8==3.8.4
pytest==3.5.0
pytest-cov==2.5.1
responses==0.10.2
testfixtures==5.1.1
testfixtures
tox<=4.0.0
typing-extensions==3.7.2
# Pin apache-airflow 1.10.5 to align with version in airflow images requirements.txt
apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.5
# install postgres seperately as apache-airflow 1.10.5 forces postgres to use
# psycopg2 instead of psycopg2-binary which requires additional apt packages
# to be installed, i.e. postgres-devel. Otherwise the following error is seen:
# Error: pg_config executable not found. Only seems to be an issue in xenial
postgres>=3.0.0
# Linting
flake8<3.8.0,>=3.6.0
# Security scanning
bandit>=1.5.0 # Apache-2.0
pyflakes>=2.1.1

4
src/bin/shipyard_airflow/tests/unit/control/common.py
View File

@ -39,12 +39,12 @@ def create_req(ctx, body):
env = testing.create_environ(
path='/',
query_string='',
protocol='HTTP/1.1',
http_version='1.1',
scheme='http',
host='falconframework.org',
port=None,
headers={'Content-Type': 'application/json'},
app='',
root_path='',
body=body,
method='POST',
wsgierrors=None,

8
src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py
View File

@ -80,12 +80,12 @@ def create_req(ctx, body):
env = testing.create_environ(
path='/',
query_string='',
protocol='HTTP/1.1',
http_version='1.1',
scheme='http',
host='falconframework.org',
port=None,
headers={'Content-Type': 'application/json'},
app='',
root_path='',
body=body,
method='POST',
wsgierrors=None,
@ -284,7 +284,7 @@ def test_on_get(mock_get_all_actions, mock_authorize):
mock_authorize.assert_called_once_with(
'workflow_orchestrator:list_actions', context)
assert mock_get_all_actions.call_count == 1
assert resp.body is not None
assert resp.text is not None
assert resp.status == '200 OK'
@ -318,7 +318,7 @@ def test_on_post(mock_info, mock_create_action, mock_authorize, *args):
mock_info.assert_called_with("Id %s generated for action %s", 'test_id',
'test_name')
assert resp.status == '201 Created'
assert resp.body is not None
assert resp.text is not None
assert '/api/v1.0/actions/' in resp.location

2
src/bin/shipyard_airflow/tests/unit/control/test_actions_id_api.py
View File

@ -166,7 +166,7 @@ def test_on_get(mock_authorize, mock_get_action):
mock_authorize.assert_called_once_with('workflow_orchestrator:get_action',
context)
mock_get_action.assert_called_once_with(action_id=None, verbosity=1)
assert resp.body == '"action_returned"'
assert resp.text == '"action_returned"'
assert resp.status == '200 OK'
@mock.patch('shipyard_airflow.control.helpers.action_helper.notes_helper',

2
src/bin/shipyard_airflow/tests/unit/control/test_actions_steps_id_logs_api.py
View File

@ -121,7 +121,7 @@ Starting attempt 1 of 1
--------------------------------------------------------------------------------
Executing Task(PythonOperator): action_xcom
Running: ['bash', '-c', 'airflow run deploy_site action_xcom \
Running: ['bash', '-c', 'airflow tasks run deploy_site action_xcom \
2018-04-11T07:30:37 --job_id 2 --raw -sd DAGS_FOLDER/deploy_site.py']
Running on host airflow-worker-0.airflow-worker-discovery.ucp.svc.cluster.local
Subtask: [2018-04-11 07:30:43,944] {{python_operator.py:90}} \

2
src/bin/shipyard_airflow/tests/unit/control/test_api.py
View File

@ -24,6 +24,6 @@ class TestVersionsResource():
req = create_req(context, None)
resp = create_resp()
version_resource.on_get(req, resp)
assert sorted(resp.body) == sorted(
assert sorted(resp.text) == sorted(
'{"v1.0": {"status": "stable", "path": "/api/v1.0"}}')
assert resp.status == '200 OK'

2
src/bin/shipyard_airflow/tests/unit/control/test_configdocs_api.py
View File

@ -46,7 +46,7 @@ class TestConfigDocsStatusResource():
assert result.status_code == 200
assert result.text == json.dumps(common.str_responder(), default=str)
assert result.headers[
'content-type'] == 'application/json; charset=UTF-8'
'content-type'] == 'application/json'
class TestConfigDocsResource():

2
src/bin/shipyard_airflow/tests/unit/control/test_status_api.py
View File

@ -33,4 +33,4 @@ class TestStatusResource():
assert result.status_code == 200
assert result.text == json.dumps(common.str_responder(), default=str)
assert result.headers[
'content-type'] == 'application/json; charset=UTF-8'
'content-type'] == 'application/json'

45
src/bin/shipyard_airflow/tox.ini
View File

@ -1,15 +1,29 @@
[tox]
skipsdist=True
envlist = py36, pep8, bandit
envlist = py38, pep8, bandit
[testenv]
setenv=
PYTHONWARNING=all
LANG=C.UTF-8
LC_ALL=C.UTF-8
deps = -r{toxinidir}/requirements.txt
deps = -r{toxinidir}/requirements-frozen.txt
[testenv:py36]
[testenv:freeze]
recreate = True
allowlist_externals=
rm
sh
deps=
-r{toxinidir}/requirements-direct.txt
-r{toxinidir}/test-requirements.txt
commands=
rm -f {toxinidir}/requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"
[testenv:py38]
skipsdist=True
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
@ -22,10 +36,10 @@ commands =
--cov-report=term-missing:skip-covered \
--cov-config=.coveragerc \
--cov=shipyard_airflow \
--cov-report=html
--cov-report=html \
-vv
[testenv:pep8]
basepython=python3
deps=
flake8>=3.3.0
bandit>=1.5.0
@ -34,7 +48,6 @@ commands =
bandit -r shipyard_airflow
[testenv:bandit]
basepython=python3
skipsdist= True
commands =
bandit -r shipyard_airflow -n 5
@ -42,20 +55,32 @@ commands =
[testenv:genconfig]
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
commands = oslo-config-generator --config-file=generator/config-generator.conf
deps =
-r{toxinidir}/test-requirements.txt
-r{toxinidir}/requirements-frozen.txt
commands =
pip install . --use-pep517
oslo-config-generator --config-file=generator/config-generator.conf
[testenv:genpolicy]
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
commands = oslopolicy-sample-generator --config-file=generator/policy-generator.conf
deps =
-r{toxinidir}/test-requirements.txt
-r{toxinidir}/requirements-frozen.txt
commands =
pip install . --use-pep517
oslopolicy-sample-generator --config-file=generator/policy-generator.conf
[testenv:gen_all]
# Combined to make these run together instead of setting up separately
basepython=python3
setenv=
SLUGIFY_USES_TEXT_UNIDECODE=yes
deps =
-r{toxinidir}/test-requirements.txt
-r{toxinidir}/requirements-frozen.txt
commands =
pip install -r{toxinidir}/test-requirements.txt
pip install . --use-pep517
oslo-config-generator --config-file=generator/config-generator.conf
oslopolicy-sample-generator --config-file=generator/policy-generator.conf

24
src/bin/shipyard_client/requirements-direct.txt Normal file
View File

@ -0,0 +1,24 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# CLI/Client requirements
arrow<=0.17.0
click==6.7
click-default-group==1.2
keystoneauth1<=5.1.1
requests==2.23.0
PTable==0.9.2
pylibyaml==0.1.0
PyYAML<=5.4.1

40
src/bin/shipyard_client/requirements-frozen.txt Normal file
View File

@ -0,0 +1,40 @@
amqp==2.6.1
arrow==0.17.0
attrs==23.1.0
bandit==1.6.0
certifi==2022.12.7
chardet==3.0.4
click==6.7
click-default-group==1.2
coverage==7.2.3
flake8==3.8.4
gitdb==4.0.10
GitPython==3.1.31
idna==2.10
iso8601==1.1.0
keystoneauth1==5.1.1
mccabe==0.6.1
more-itertools==9.1.0
os-service-types==1.7.0
pbr==5.11.1
pip==23.0.1
pluggy==0.6.0
PTable==0.9.2
py==1.11.0
pycodestyle==2.6.0
pyflakes==2.2.0
pylibyaml==0.1.0
pytest==3.5.0
pytest-cov==2.5.1
python-dateutil==2.8.2
PyYAML==5.4.1
requests==2.23.0
responses==0.10.2
setuptools==67.6.0
six==1.16.0
smmap==5.0.0
stevedore==5.0.0
testfixtures==5.1.1
urllib3==1.25.11
vine==1.3.0
wheel==0.40.0

27
src/bin/shipyard_client/requirements.txt
View File

@ -1,24 +1,3 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# CLI/Client requirements
arrow==0.12.1
click==6.7
click-default-group==1.2
keystoneauth1>=3.18.0
requests!=2.20.0,>=2.14.2
PTable==0.9.2
pylibyaml==0.1.0
pyyaml~=5.1
# Warning: This file should be empty.
# Specify direct dependencies in requirements-direct.txt instead.
-r requirements-direct.txt

4
src/bin/shipyard_client/setup.cfg
View File

@ -12,8 +12,8 @@ classifier =
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 3
Programming Language :: Python :: 3.5
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.10
[files]
packages =

8
src/bin/shipyard_client/test-requirements.txt
View File

@ -1,14 +1,14 @@
# Testing
pyflakes>=2.1.1
amqp~=2.6.0
pyflakes==2.2.0
amqp==2.6.1
pytest==3.5.0
pytest-cov==2.5.1
responses==0.10.2
testfixtures==5.1.1
# Linting
flake8<3.8.0,>=3.6.0
flake8==3.8.4
# Security scanning
bandit>=1.1.0 # Apache-2.0
bandit==1.6.0 # Apache-2.0

24
src/bin/shipyard_client/tox.ini
View File

@ -1,16 +1,29 @@
[tox]
skipsdist=True
envlist = py36, pep8, bandit
envlist = py38, pep8, bandit
[testenv]
setenv=
PYTHONWARNING=all
LC_ALL=C.UTF-8
LC_ALL=C.UTF-8
deps = -r{toxinidir}/requirements.txt
deps = -r{toxinidir}/requirements-frozen.txt
-r{toxinidir}/test-requirements.txt
[testenv:py36]
[testenv:freeze]
recreate = True
allowlist_externals=
rm
sh
deps=
-r{toxinidir}/requirements-direct.txt
-r{toxinidir}/test-requirements.txt
commands=
rm -f {toxinidir}/requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"
[testenv:py38]
skipsdist=True
commands =
pytest \
@ -19,10 +32,10 @@ commands =
--cov-report=term-missing:skip-covered \
--cov-config=.coveragerc \
--cov=shipyard_client \
--cov-report=html
--cov-report=html \
-vv
[testenv:pep8]
basepython=python3
deps=
flake8>=3.3.0
bandit>=1.5.0
@ -31,7 +44,6 @@ commands =
bandit -r shipyard_airflow
[testenv:bandit]
basepython=python3
skipsdist=True
commands =
bandit -r shipyard_client -n 5

2
tools/airflow_image_run.sh
View File

@ -47,7 +47,7 @@ fi
docker stop airflow_test
docker rm airflow_test
if [[ ${TEST_RESP:(-7)} == "success" ]]; then
if [ ${TEST_RESP:(-7)} == "success" ]; then
exit 0
else
exit 1

39
tools/gate/playbooks/airskiff-deploy.yaml
View File

@ -1,4 +1,4 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
# Copyright 2019, AT&T Intellectual Property
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -13,24 +13,15 @@
# limitations under the License.
- hosts: primary
roles:
- clear-firewall
- bindep
- disable-systemd-resolved
- ensure-python
- ensure-pip
- ensure-docker
tasks:
# Stop systemd-resolved service before starting docker.
- name: stop systemd-resolved service
systemd:
state: stopped
enabled: no
masked: yes
daemon_reload: yes
name: systemd-resolved
become: yes
- name: ensure pip3 installed
apt:
name: "{{ item }}"
with_items:
- python3-pip
become: yes
- name: Clone Required Repositories
shell: |
export CLONE_SHIPYARD={{ CLONE_SHIPYARD }}
@ -39,7 +30,7 @@
args:
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
- name: Setup AppArmor
- name: Setup Apparmor
shell: |
./tools/deployment/airskiff/developer/009-setup-apparmor.sh
args:
@ -58,16 +49,22 @@
chdir: "{{ zuul.project.src_dir }}"
become: yes
- name: Setup clients
shell: |
./tools/deployment/airskiff/developer/020-setup-client.sh
args:
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
- name: Deploy Airship components using Armada
shell: |
mkdir ~/.kube
mkdir -p ~/.kube
cp -rp /home/zuul/.kube/config ~/.kube/config
./tools/deployment/airskiff/developer/030-armada-bootstrap.sh
args:
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
become: yes
- name: Deploy OpenStack using Airship
- name: Deploy Memcached using Airship
shell: |
./tools/deployment/airskiff/developer/100-deploy-osh.sh
args:

30
tools/gate/playbooks/airskiff-reduce-site.yaml
View File

@ -14,40 +14,10 @@
- hosts: primary
tasks:
- name: ensure pip packages are installed
apt:
name: "{{ item }}"
with_items:
- python3-pip
- python3-git
- python3-requests
- python3-yaml
become: yes
- name: deploy iptables packages
include_role:
name: deploy-package
tasks_from: dist
vars:
packages:
deb:
- iptables
rpm:
- iptables
- command: sudo iptables -S
- command: sudo iptables -F
- command: sudo iptables -P INPUT ACCEPT
- command: sudo iptables -S
become: yes
- name: Overwrite Armada manifest
shell: |
git checkout v1.9
mv tools/gate/manifests/full-site.yaml \
type/skiff/manifests/full-site.yaml
# Overwrite versions.yaml with the latest Airship charts and xenial images
./tools/updater.py --in-file global/software/config/versions.yaml \
--tag-filter ubuntu_xenial --skip mariadb
args:
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"

10
tools/gate/playbooks/run-image.yaml
View File

@ -18,6 +18,16 @@
vars_files:
- vars.yaml
roles:
- bindep
- clear-firewall
- ensure-python
- ensure-pip
- ensure-docker
- build-images
tags:
- bindep
- clear-firewall
- ensure-python
- ensure-pip
- ensure-docker
- build-images

8
tools/gate/roles/build-images/files/docker-systemd.conf
View File

@ -1,8 +0,0 @@
# NOTE(SamYaple): CentOS cannot be build with userns-remap enabled. httpd uses
# cap_set_file capability and there is no way to pass that in at build as of
# docker 17.06.
# TODO(SamYaple): Periodically check to see if this is possible in newer
# versions of Docker
[Service]
ExecStart=
ExecStart=/usr/bin/dockerd

43
tools/gate/roles/build-images/tasks/airship-shipyard.yaml
View File

@ -38,52 +38,15 @@
debug:
var: image_tags
- name: Install Docker (Debian)
when: ansible_os_family == 'Debian'
- name: Install Docker python module for ansible docker login
block:
- file:
path: "{{ item }}"
state: directory
with_items:
- /etc/docker/
- /etc/systemd/system/docker.service.d/
- /var/lib/docker/
- mount:
path: /var/lib/docker/
src: tmpfs
fstype: tmpfs
opts: size=25g
state: mounted
- copy: "{{ item }}"
with_items:
- content: "{{ docker_daemon | to_json }}"
dest: /etc/docker/daemon.json
- src: files/docker-systemd.conf
dest: /etc/systemd/system/docker.service.d/
- apt_key:
url: https://download.docker.com/linux/ubuntu/gpg
- apt_repository:
repo: deb http://{{ zuul_site_mirror_fqdn }}/deb-docker/{{ ansible_distribution_release }} {{ ansible_distribution_release }} stable
- apt:
name: "{{ item }}"
allow_unauthenticated: True
with_items:
- docker-ce
- python3-pip
- python3-setuptools
- pip:
name: docker
version: 2.7.0
version: 4.4.4
executable: pip3
# NOTE(SamYaple): Allow all connections from containers to host so the
# containers can access the http server for git and wheels
- iptables:
action: insert
chain: INPUT
in_interface: docker0
jump: ACCEPT
become: True
- name: Make images
when: not publish
block:

37
tools/gate/roles/disable-systemd-resolved/tasks/disable-systemd-resolved.yaml Normal file
View File

@ -0,0 +1,37 @@
# Copyright 2020 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Disable systemd-resolved service
systemd:
state: stopped
enabled: no
masked: yes
daemon_reload: yes
name: systemd-resolved
become: yes
- name: Remove local stub dns from resolv.conf, if it exists
lineinfile:
path: /etc/resolv.conf
state: absent
regexp: '^nameserver.*127.0.0.1'
become: yes
- name: Add upstream nameservers in resolv.conf
blockinfile:
path: /etc/resolv.conf
block: |
nameserver 8.8.8.8
nameserver 8.8.4.4
become: yes

15
tools/gate/roles/disable-systemd-resolved/tasks/main.yaml Normal file
View File

@ -0,0 +1,15 @@
# Copyright 2020 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- include: disable-systemd-resolved.yaml

2
tools/helm_install.sh
View File

@ -17,7 +17,7 @@
set -x
HELM=$1
HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.6.3-linux-amd64.tar.gz"}
HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz"}
function install_helm_binary {

4
tools/helm_tk.sh
View File

@ -17,7 +17,9 @@
set -eux
HTK_REPO=${HTK_REPO:-"https://opendev.org/openstack/openstack-helm-infra.git"}
HTK_STABLE_COMMIT=${HTK_COMMIT:-"5c4056ad341afcc577e63902b6ddbfb222d757e1"}
HTK_STABLE_COMMIT=${HTK_COMMIT:-"a7cd689280cdbc0acd04a7a1b745941260e8700b"}
TMP_DIR=$(mktemp -d)

24
tox.ini
View File

@ -5,15 +5,20 @@ envlist = all_jobs, docs
[testenv]
setenv=
PYTHONWARNING=all
passenv=HTTP_PROXY HTTPS_PROXY http_proxy https_proxy NO_PROXY no_proxy
passenv=
HTTP_PROXY
HTTPS_PROXY
http_proxy
https_proxy
NO_PROXY
no_proxy
[testenv:venv]
commands = {posargs}
# Please use `make docs` instead
[testenv:docs]
basepython=python3
whitelist_externals=
allowlist_externals=
rm
cp
tox
@ -28,22 +33,21 @@ deps= -r{toxinidir}/doc/requirements.txt
# `all_jobs` is here to support invocation as simply `tox`, as is done in
# current pipeline.
[testenv:all_jobs]
whitelist_externals=
allowlist_externals=
tox
commands=
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini
[testenv:py36]
whitelist_externals=
[testenv:py38]
allowlist_externals=
tox
commands=
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py36
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py36
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py38
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py38
[testenv:pep8]
basepython=python3
whitelist_externals=
allowlist_externals=
tox
commands=
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e pep8