Rollback apache-airflow back to 1.10.5
This PS is created to : - roll back apache-airflow back to 1.10.5 - change default chart values from bionic to ubuntu_focal - save freshly generated config and policy files samples - in focal Dockerfile postgres client version is pubped up to v15 - change airflow docker image requirements from limited to fully frozen for shipyard-airflow project - adjusted requirements-direct.txt for shipyard_airflow - adjusted requirements-direct.txt for shipyard_client - regenerated requirements-frozen for both projects - fixed unit tests after upgrading click module - gen_all tox profile processing has been moved over to py8 gate because it requires focal zuul node to run - upgraded airskiff gate playbook to include latest treasuremap patchset with updated airskiff site admada manifests Change-Id: I47e44f5cfa19b2649697e7cc5a31557a6f4fcfea
This commit is contained in:
Sergiy Markin
parent
154a099b28
commit
b7e7d25308
@ -30,15 +30,15 @@ labels:
|
||||
|
||||
images:
|
||||
tags:
|
||||
airflow: quay.io/airshipit/airflow:latest-ubuntu_bionic
|
||||
shipyard: quay.io/airshipit/shipyard:latest-ubuntu_bionic
|
||||
airflow: quay.io/airshipit/airflow:latest-ubuntu_focal
|
||||
shipyard: quay.io/airshipit/shipyard:latest-ubuntu_focal
|
||||
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
|
||||
shipyard_db_init: docker.io/postgres:14.5
|
||||
shipyard_db_auxiliary: docker.io/postgres:14.5
|
||||
shipyard_db_sync: quay.io/airshipit/shipyard:latest-ubuntu_bionic
|
||||
shipyard_db_sync: quay.io/airshipit/shipyard:latest-ubuntu_focal
|
||||
airflow_db_init: docker.io/postgres:14.5
|
||||
rabbit_init: docker.io/rabbitmq:3.7-management
|
||||
airflow_db_sync: quay.io/airshipit/airflow:latest-ubuntu_bionic
|
||||
airflow_db_sync: quay.io/airshipit/airflow:latest-ubuntu_focal
|
||||
ks_user: docker.io/openstackhelm/heat:ocata
|
||||
ks_service: docker.io/openstackhelm/heat:ocata
|
||||
ks_endpoints: docker.io/openstackhelm/heat:ocata
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#
|
||||
# Requirements for creating documentation only.
|
||||
#
|
||||
Sphinx
|
||||
Sphinx<=6.2.1
|
||||
sphinx-rtd-theme==0.5.0
|
||||
Jinja2<=3.0.3
|
||||
@ -180,9 +180,13 @@
|
||||
# and will be removed in the S release.
|
||||
#auth_uri = <None>
|
||||
|
||||
# API version of the admin Identity API endpoint. (string value)
|
||||
# API version of the Identity API endpoint. (string value)
|
||||
#auth_version = <None>
|
||||
|
||||
# Interface to use for the Identity API endpoint. Valid values are "public",
|
||||
# "internal" (default) or "admin". (string value)
|
||||
#interface = internal
|
||||
|
||||
# Do not handle authorization requests within the middleware, but delegate the
|
||||
# authorization decision to downstream WSGI components. (boolean value)
|
||||
#delay_auth_decision = false
|
||||
@ -217,14 +221,6 @@
|
||||
# The region in which the identity server can be found. (string value)
|
||||
#region_name = <None>
|
||||
|
||||
# DEPRECATED: Directory used to cache files related to PKI tokens. This option
|
||||
# has been deprecated in the Ocata release and will be removed in the P
|
||||
# release. (string value)
|
||||
# This option is deprecated for removal since Ocata.
|
||||
# Its value may be silently ignored in the future.
|
||||
# Reason: PKI token format is no longer supported.
|
||||
#signing_dir = <None>
|
||||
|
||||
# Optionally specify a list of memcached server(s) to use for caching. If left
|
||||
# undefined, tokens will instead be cached in-process. (list value)
|
||||
# Deprecated group/name - [keystone_authtoken]/memcache_servers
|
||||
@ -235,16 +231,6 @@
|
||||
# to -1 to disable caching completely. (integer value)
|
||||
#token_cache_time = 300
|
||||
|
||||
# DEPRECATED: Determines the frequency at which the list of revoked tokens is
|
||||
# retrieved from the Identity service (in seconds). A high number of revocation
|
||||
# events combined with a low cache duration may significantly reduce
|
||||
# performance. Only valid for PKI tokens. This option has been deprecated in
|
||||
# the Ocata release and will be removed in the P release. (integer value)
|
||||
# This option is deprecated for removal since Ocata.
|
||||
# Its value may be silently ignored in the future.
|
||||
# Reason: PKI token format is no longer supported.
|
||||
#revocation_cache_time = 10
|
||||
|
||||
# (Optional) If defined, indicate whether token data should be authenticated or
|
||||
# authenticated and encrypted. If MAC, token data is authenticated (with HMAC)
|
||||
# in the cache. If ENCRYPT, token data is encrypted and authenticated in the
|
||||
@ -280,9 +266,9 @@
|
||||
# client connection from the pool. (integer value)
|
||||
#memcache_pool_conn_get_timeout = 10
|
||||
|
||||
# (Optional) Use the advanced (eventlet safe) memcached client pool. The
|
||||
# advanced pool will only work under python 2.x. (boolean value)
|
||||
#memcache_use_advanced_pool = false
|
||||
# (Optional) Use the advanced (eventlet safe) memcached client pool. (boolean
|
||||
# value)
|
||||
#memcache_use_advanced_pool = true
|
||||
|
||||
# (Optional) Indicate whether to set the X-Service-Catalog header. If False,
|
||||
# middleware will not ask for service catalog on token validation and will not
|
||||
@ -298,27 +284,6 @@
|
||||
# (string value)
|
||||
#enforce_token_bind = permissive
|
||||
|
||||
# DEPRECATED: If true, the revocation list will be checked for cached tokens.
|
||||
# This requires that PKI tokens are configured on the identity server. (boolean
|
||||
# value)
|
||||
# This option is deprecated for removal since Ocata.
|
||||
# Its value may be silently ignored in the future.
|
||||
# Reason: PKI token format is no longer supported.
|
||||
#check_revocations_for_cached = false
|
||||
|
||||
# DEPRECATED: Hash algorithms to use for hashing PKI tokens. This may be a
|
||||
# single algorithm or multiple. The algorithms are those supported by Python
|
||||
# standard hashlib.new(). The hashes will be tried in the order given, so put
|
||||
# the preferred one first for performance. The result of the first hash will be
|
||||
# stored in the cache. This will typically be set to multiple values only while
|
||||
# migrating from a less secure algorithm to a more secure one. Once all the old
|
||||
# tokens are expired this option should be set to a single value for better
|
||||
# performance. (list value)
|
||||
# This option is deprecated for removal since Ocata.
|
||||
# Its value may be silently ignored in the future.
|
||||
# Reason: PKI token format is no longer supported.
|
||||
#hash_algorithms = md5
|
||||
|
||||
# A choice of roles that must be present in a service token. Service tokens are
|
||||
# allowed to request that an expired token can be used and so this check should
|
||||
# tightly control that only actual services should be sending this token. Roles
|
||||
@ -333,6 +298,10 @@
|
||||
# possible. (boolean value)
|
||||
#service_token_roles_required = false
|
||||
|
||||
# The name or type of the service as it appears in the service catalog. This is
|
||||
# used to validate tokens that have restricted access rules. (string value)
|
||||
#service_type = <None>
|
||||
|
||||
# Authentication type to load (string value)
|
||||
# Deprecated group/name - [keystone_authtoken]/auth_plugin
|
||||
#auth_type = <None>
|
||||
|
||||
@ -106,3 +106,4 @@
|
||||
# targeted release
|
||||
# POST /api/v1.0/actions
|
||||
#"workflow_orchestrator:action_test_site": "rule:admin_required"
|
||||
|
||||
|
||||
@ -184,7 +184,7 @@ command. In this example we will execute the ``get actions`` command::
|
||||
-e 'OS_PASSWORD=password' -e 'OS_PROJECT_DOMAIN_NAME=default' \
|
||||
-e 'OS_PROJECT_NAME=service' -e 'OS_USERNAME=shipyard' \
|
||||
-e 'OS_USER_DOMAIN_NAME=default' -e 'OS_IDENTITY_API_VERSION=3' \
|
||||
--rm --net=host airshipit/shipyard:latest-ubuntu_bionic get actions
|
||||
--rm --net=host airshipit/shipyard:latest-ubuntu_focal get actions
|
||||
|
||||
The output will resemble the following::
|
||||
|
||||
|
||||
@ -45,7 +45,7 @@ Following parameters need to be passed as environment/shell variable to make
|
||||
command:
|
||||
|
||||
DISTRO
|
||||
parameter to identify distro specific Dockerfile, ubuntu_bionic (Default)
|
||||
parameter to identify distro specific Dockerfile, ubuntu_focal (Default)
|
||||
|
||||
DISTRO_BASE_IMAGE
|
||||
parameter to use different base image other than what's used in DISTRO
|
||||
|
||||
@ -43,7 +43,7 @@ ARG AIRFLOW_HOME=/usr/local/airflow
|
||||
# Moved celery to images/airflow/requirements.txt as apache-airflow uses a
|
||||
# version of celery incompatibile with the version of kombu needed by other
|
||||
# Airship components
|
||||
ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs]==1.10.15"
|
||||
ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs,jdbc]==1.10.5"
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
ARG ctx_base=src/bin
|
||||
|
||||
@ -53,30 +53,39 @@ ARG KUBECTL_VERSION=1.26.3
|
||||
# Needed from apache-airflow 1.10.2, since core.airflow_home config is deprecated
|
||||
ENV AIRFLOW_HOME=${AIRFLOW_HOME}
|
||||
|
||||
RUN set -ex && \
|
||||
apt-get -qq update && \
|
||||
apt-get -y install \
|
||||
automake \
|
||||
ca-certificates \
|
||||
curl \
|
||||
git \
|
||||
g++ \
|
||||
libkrb5-dev \
|
||||
libffi-dev \
|
||||
libssl-dev \
|
||||
libpq-dev \
|
||||
libtool \
|
||||
libsasl2-dev \
|
||||
locales \
|
||||
netcat \
|
||||
netbase \
|
||||
python3 \
|
||||
python3-setuptools \
|
||||
python3-pip \
|
||||
python3-dev \
|
||||
python3-dateutil \
|
||||
make \
|
||||
--no-install-recommends \
|
||||
RUN set -ex \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y wget curl \
|
||||
apt-transport-https ca-certificates gnupg \
|
||||
&& echo "deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
|
||||
&& curl -o /etc/apt/trusted.gpg.d/postgresql_release_signing_key.asc 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' \
|
||||
&& apt update \
|
||||
&& apt -y install \
|
||||
automake \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
git \
|
||||
g++ \
|
||||
libkrb5-dev \
|
||||
libffi-dev \
|
||||
libssl-dev \
|
||||
libpq-dev \
|
||||
libpq5 \
|
||||
libtool \
|
||||
libsasl2-dev \
|
||||
locales \
|
||||
netcat \
|
||||
netbase \
|
||||
postgresql-client \
|
||||
postgresql-common \
|
||||
python3 \
|
||||
python3-setuptools \
|
||||
python3-pip \
|
||||
python3-dev \
|
||||
python3-dateutil \
|
||||
make \
|
||||
--no-install-recommends \
|
||||
&& python3 -m pip install -U pip \
|
||||
&& apt-get clean \
|
||||
&& rm -rf \
|
||||
@ -102,6 +111,7 @@ RUN set -ex \
|
||||
&& cd .. \
|
||||
&& rm -fr libyaml
|
||||
|
||||
|
||||
# Things that change mostly infrequently
|
||||
RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
|
||||
&& curl -L -o /usr/local/bin/kubectl \
|
||||
@ -114,11 +124,11 @@ RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
|
||||
# Install Airflow directly to allow overriding source
|
||||
# COPY images/airflow/requirements.txt /tmp/
|
||||
COPY ${ctx_base}/shipyard_airflow/requirements-frozen.txt /tmp/requirements.txt
|
||||
RUN pip3 install -r /tmp/requirements.txt --no-cache-dir \
|
||||
&& pip3 install $AIRFLOW_SRC --no-cache-dir \
|
||||
&& (pip3 uninstall -y snakebite || true) \
|
||||
&& (pip3 uninstall -y psycopg2 || true) \
|
||||
&& (pip3 install --no-cache-dir --force-reinstall $(pip freeze | grep psycopg2-binary) || true)
|
||||
RUN pip3 install -r /tmp/requirements.txt --no-cache-dir
|
||||
# && pip3 install $AIRFLOW_SRC --no-cache-dir
|
||||
# && (pip3 uninstall -y snakebite || true) \
|
||||
# && (pip3 uninstall -y psycopg2 || true) \
|
||||
# && (pip3 install --no-cache-dir --force-reinstall $(pip freeze | grep psycopg2-binary) || true)
|
||||
|
||||
# Copy scripts used in the container:
|
||||
COPY images/airflow/script/*.sh ${AIRFLOW_HOME}/
|
||||
@ -143,7 +153,7 @@ RUN pip3 install -r /tmp/api_requirements.txt --no-cache-dir
|
||||
|
||||
COPY ${ctx_base}/shipyard_airflow /tmp/shipyard/
|
||||
RUN cd /tmp/shipyard \
|
||||
&& pip3 install $(pwd)
|
||||
&& pip3 install $(pwd) --use-pep517
|
||||
|
||||
# Note: The value for the dags and plugins directories that are sourced
|
||||
# from the values.yaml of the Shipyard Helm chart need to align with these
|
||||
|
||||
@ -41,9 +41,9 @@ elif [[ $cmd == *scheduler* ]]; then
|
||||
elif [[ $cmd == 'quicktest' ]]; then
|
||||
${python3_path} ${airflow_path} initdb
|
||||
${python3_path} ${airflow_path} webserver -p 8080 &
|
||||
airflow tasks run example_bash_operator runme_0 2018-01-01
|
||||
airflow run example_bash_operator runme_0 2018-01-01
|
||||
airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
|
||||
airflow dags state example_bash_operator 2018-01-01
|
||||
airflow dag_state example_bash_operator 2018-01-01
|
||||
else
|
||||
echo "Invalid Command!"
|
||||
exit 1
|
||||
|
||||
@ -112,10 +112,10 @@ COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/
|
||||
&& python3 -m pip install -U pip \
|
||||
&& pip3 install -r /home/shipyard/client_requirements.txt --no-cache-dir \
|
||||
&& cd /home/shipyard/shipyard_client \
|
||||
&& pip3 install $(pwd) --use-pep517 \
|
||||
&& python3 setup.py install \
|
||||
&& pip3 install -r /home/shipyard/api_requirements.txt --no-cache-dir \
|
||||
&& cd /home/shipyard/shipyard \
|
||||
&& pip3 install $(pwd) --use-pep517 \
|
||||
&& python3 setup.py install \
|
||||
&& apt-get purge -y --auto-remove $buildDeps \
|
||||
&& apt-get autoremove -yqq --purge \
|
||||
&& apt-get clean \
|
||||
@ -125,7 +125,9 @@ COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/
|
||||
/var/tmp/* \
|
||||
/usr/share/man \
|
||||
/usr/share/doc \
|
||||
/usr/share/doc-base
|
||||
/usr/share/doc-base \
|
||||
&& python3 -m pip install -U pip
|
||||
|
||||
|
||||
# Entrypoint
|
||||
ENTRYPOINT ["/home/shipyard/entrypoint.sh"]
|
||||
|
||||
@ -1,15 +1,22 @@
|
||||
alembic==1.0.1
|
||||
apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.5
|
||||
arrow<=0.17.0
|
||||
# apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.15
|
||||
apache-airflow[crypto,celery,hive,hdfs]==1.10.15
|
||||
celery==4.3
|
||||
Flask==1.1.0
|
||||
Flask-AppBuilder @ git+https://github.com/dpgaspar/Flask-AppBuilder.git@airflow_v1.13.2
|
||||
pytz==2018.5
|
||||
coverage==5.3
|
||||
falcon
|
||||
jsonschema<=3.2.0
|
||||
keystoneauth1<=5.1.1
|
||||
kombu
|
||||
kubernetes==26.1.0
|
||||
marshmallow-sqlalchemy==0.18.0
|
||||
networkx
|
||||
oslo.config<=8.7.1
|
||||
oslo.policy<=3.10.1
|
||||
oslo.utils<=4.12.3
|
||||
psycopg2-binary==2.8.4
|
||||
pyarrow==6.0.1
|
||||
pylibyaml==0.1.0
|
||||
PyYAML<=5.4.1
|
||||
@ -17,12 +24,16 @@ regex==2020.11.13
|
||||
requests==2.23.0
|
||||
responses==0.12.1
|
||||
setuptools<=45.2.0
|
||||
SQLAlchemy<=1.3.20
|
||||
SQLAlchemy==1.3.15
|
||||
tabulate==0.8.03
|
||||
ulid==1.1
|
||||
uWSGI==2.0.21
|
||||
Werkzeug
|
||||
Werkzeug<=0.16.1
|
||||
WTForms<=2.3.3
|
||||
|
||||
|
||||
itsdangerous==2.0.1
|
||||
|
||||
git+https://opendev.org/airship/deckhand.git@13c5199f18664d66ccdd3a1e54bb40ad2c293d1a#egg=deckhand
|
||||
git+https://opendev.org/airship/drydock.git@d00eaf0303f50478d6503c2c2096e931134189ac#egg=drydock_provisioner&subdirectory=python
|
||||
git+https://opendev.org/airship/armada.git@a6dcf17d178a89bad626b0e5ef6fafc428dca7e4#egg=armada
|
||||
|
||||
@ -1,23 +1,21 @@
|
||||
alabaster==0.7.13
|
||||
alembic==1.4.3
|
||||
alembic==1.0.1
|
||||
amqp==2.6.1
|
||||
apache-airflow==1.10.15
|
||||
apache-airflow==1.10.5
|
||||
apispec==1.3.3
|
||||
argcomplete==1.12.3
|
||||
Armada @ git+https://opendev.org/airship/armada.git@a6dcf17d178a89bad626b0e5ef6fafc428dca7e4#egg=armada
|
||||
argcomplete==3.0.8
|
||||
Armada @ git+https://opendev.org/airship/armada.git@a6dcf17d178a89bad626b0e5ef6fafc428dca7e4
|
||||
arrow==0.17.0
|
||||
attrs==20.3.0
|
||||
attrs==23.1.0
|
||||
autopage==0.5.1
|
||||
Babel==2.12.1
|
||||
bandit==1.6.0
|
||||
bcrypt==4.0.1
|
||||
Beaker==1.12.0
|
||||
billiard==3.6.4.0
|
||||
cached-property==1.5.2
|
||||
cachetools==5.3.0
|
||||
cattrs==1.10.0
|
||||
celery==4.4.7
|
||||
certifi==2022.12.7
|
||||
celery==4.3.0
|
||||
certifi==2023.5.7
|
||||
cffi==1.15.1
|
||||
chardet==3.0.4
|
||||
charset-normalizer==3.1.0
|
||||
@ -31,18 +29,18 @@ coverage==5.3
|
||||
croniter==0.3.37
|
||||
cryptography==3.4.8
|
||||
debtcollector==2.5.0
|
||||
Deckhand @ git+https://opendev.org/airship/deckhand.git@13c5199f18664d66ccdd3a1e54bb40ad2c293d1a#egg=deckhand
|
||||
Deckhand @ git+https://opendev.org/airship/deckhand.git@13c5199f18664d66ccdd3a1e54bb40ad2c293d1a
|
||||
decorator==5.1.1
|
||||
deepdiff==5.8.1
|
||||
defusedxml==0.7.1
|
||||
dill==0.3.6
|
||||
dill==0.2.9
|
||||
distlib==0.3.6
|
||||
dnspython==2.3.0
|
||||
docutils==0.17
|
||||
dogpile.cache==1.2.0
|
||||
drydock-provisioner @ git+https://opendev.org/airship/drydock.git@d00eaf0303f50478d6503c2c2096e931134189ac#egg=drydock_provisioner&subdirectory=python
|
||||
dulwich==0.21.3
|
||||
email-validator==1.3.1
|
||||
drydock-provisioner @ git+https://opendev.org/airship/drydock.git@d00eaf0303f50478d6503c2c2096e931134189ac#subdirectory=python
|
||||
dulwich==0.21.5
|
||||
dumb-init==1.2.5.post1
|
||||
eventlet==0.33.3
|
||||
extras==1.0.0
|
||||
falcon==3.1.1
|
||||
@ -50,27 +48,26 @@ fasteners==0.18
|
||||
filelock==3.12.0
|
||||
fixtures==3.0.0
|
||||
flake8==3.8.4
|
||||
Flask==1.1.4
|
||||
Flask-Admin==1.5.4
|
||||
Flask-AppBuilder==2.3.4
|
||||
Flask-Babel==1.0.0
|
||||
Flask==1.1.0
|
||||
Flask-Admin==1.5.3
|
||||
Flask-AppBuilder @ git+https://github.com/dpgaspar/Flask-AppBuilder.git@b71789d85b632935eca79c0b53fb77f20bf17fe6
|
||||
Flask-Babel==0.12.2
|
||||
Flask-Caching==1.3.3
|
||||
Flask-JWT-Extended==3.25.1
|
||||
Flask-Login==0.4.1
|
||||
Flask-OpenID==1.3.0
|
||||
Flask-SQLAlchemy==2.5.1
|
||||
flask-swagger==0.2.14
|
||||
flask-swagger==0.2.13
|
||||
Flask-WTF==0.14.3
|
||||
flower==0.9.7
|
||||
funcsigs==1.0.2
|
||||
future==0.18.3
|
||||
funcsigs==1.0.0
|
||||
future==0.16.0
|
||||
futurist==2.4.1
|
||||
gitdb==4.0.10
|
||||
GitPython==3.1.31
|
||||
google-auth==2.17.3
|
||||
graphviz==0.20.1
|
||||
greenlet==2.0.2
|
||||
gunicorn==20.1.0
|
||||
gunicorn==19.10.0
|
||||
hacking==4.1.0
|
||||
hmsclient==0.1.1
|
||||
html5lib==0.9999999
|
||||
@ -79,10 +76,11 @@ humanize==4.6.0
|
||||
idna==2.10
|
||||
imagesize==1.4.1
|
||||
importlib-metadata==2.1.3
|
||||
importlib-resources==1.5.0
|
||||
iso8601==1.1.0
|
||||
itsdangerous==1.1.0
|
||||
Jinja2==2.11.3
|
||||
itsdangerous==2.0.1
|
||||
JayDeBeApi==1.2.3
|
||||
Jinja2==2.10.3
|
||||
JPype1==1.4.1
|
||||
json-merge-patch==0.2
|
||||
jsonpath-ng==1.5.3
|
||||
jsonpath-rw==1.4.0
|
||||
@ -91,19 +89,18 @@ jsonpickle==1.4.1
|
||||
jsonschema==3.2.0
|
||||
keystoneauth1==5.1.1
|
||||
keystonemiddleware==10.2.0
|
||||
kombu==4.6.11
|
||||
kombu==4.6.3
|
||||
kubernetes==26.1.0
|
||||
lazy-object-proxy==1.4.3
|
||||
lazy-object-proxy==1.9.0
|
||||
lockfile==0.12.2
|
||||
Mako==1.2.4
|
||||
Markdown==2.6.11
|
||||
MarkupSafe==2.0.1
|
||||
marshmallow==2.21.0
|
||||
marshmallow==2.19.5
|
||||
marshmallow-enum==1.5.1
|
||||
marshmallow-sqlalchemy==0.23.1
|
||||
marshmallow-sqlalchemy==0.18.0
|
||||
mccabe==0.6.1
|
||||
mock==5.0.2
|
||||
more-itertools==9.1.0
|
||||
msgpack==1.0.5
|
||||
natsort==8.3.1
|
||||
netaddr==0.8.0
|
||||
@ -113,6 +110,7 @@ nose==1.3.7
|
||||
numpy==1.24.3
|
||||
oauthlib==3.2.2
|
||||
ordered-set==4.1.0
|
||||
ordereddict==1.1
|
||||
os-service-types==1.7.0
|
||||
oslo.cache==2.10.1
|
||||
oslo.concurrency==5.1.1
|
||||
@ -121,32 +119,29 @@ oslo.context==4.1.0
|
||||
oslo.db==10.0.0
|
||||
oslo.i18n==6.0.0
|
||||
oslo.log==4.6.0
|
||||
oslo.messaging==12.13.0
|
||||
oslo.metrics==0.6.0
|
||||
oslo.messaging==10.3.0
|
||||
oslo.middleware==4.4.0
|
||||
oslo.policy==3.10.1
|
||||
oslo.serialization==4.2.0
|
||||
oslo.service==3.1.1
|
||||
oslo.utils==4.12.3
|
||||
packaging==21.3
|
||||
pandas==1.5.3
|
||||
pandas==0.25.3
|
||||
Paste==3.5.0
|
||||
PasteDeploy==3.0.1
|
||||
PasteScript==3.3.0
|
||||
pbr==5.5.1
|
||||
pendulum==1.4.4
|
||||
pip==23.0.1
|
||||
platformdirs==3.4.0
|
||||
pluggy==0.6.0
|
||||
platformdirs==3.5.0
|
||||
ply==3.11
|
||||
prettytable==3.7.0
|
||||
prison==0.2.1
|
||||
promenade @ git+https://opendev.org/airship/promenade.git@5602e85fb936eeafdc3e4699fd3afbe1576078bc#egg=promenade
|
||||
prison==0.1.0
|
||||
promenade @ git+https://opendev.org/airship/promenade.git@5602e85fb936eeafdc3e4699fd3afbe1576078bc
|
||||
prometheus-client==0.8.0
|
||||
protobuf==4.22.3
|
||||
protobuf==4.22.4
|
||||
psutil==5.9.5
|
||||
psycopg2-binary==2.9.6
|
||||
pure-sasl==0.6.2
|
||||
psycopg2-binary==2.8.4
|
||||
py==1.11.0
|
||||
pyarrow==6.0.1
|
||||
pyasn1==0.5.0
|
||||
@ -164,20 +159,16 @@ pyparsing==2.4.7
|
||||
pyperclip==1.8.2
|
||||
pyproject_api==1.5.0
|
||||
pyrsistent==0.19.3
|
||||
pytest==3.5.0
|
||||
pytest-cov==2.5.1
|
||||
python-barbicanclient==5.2.0
|
||||
python-daemon==2.3.2
|
||||
python-daemon==2.1.2
|
||||
python-dateutil==2.8.1
|
||||
python-editor==1.0.4
|
||||
python-keystoneclient==5.1.0
|
||||
python-memcached==1.59
|
||||
python-mimeparse==1.6.0
|
||||
python-nvd3==0.15.0
|
||||
python-slugify==4.0.1
|
||||
python-subunit==1.4.0
|
||||
python3-openid==3.2.0
|
||||
pytz==2023.3
|
||||
pytz==2018.5
|
||||
pytzdata==2020.1
|
||||
PyYAML==5.4.1
|
||||
regex==2020.11.13
|
||||
@ -191,14 +182,13 @@ retry==0.9.2
|
||||
rfc3986==2.0.0
|
||||
Routes==2.5.1
|
||||
rsa==4.9
|
||||
sasl==0.3.1
|
||||
selector==0.10.1
|
||||
setproctitle==1.3.2
|
||||
setuptools==45.2.0
|
||||
simplejson==3.19.1
|
||||
six==1.16.0
|
||||
smmap==5.0.0
|
||||
snakebite-py3==3.0.5
|
||||
snakebite==2.11.0
|
||||
snowballstemmer==2.2.0
|
||||
Sphinx==3.3.1
|
||||
sphinx-rtd-theme==0.5.0
|
||||
@ -208,30 +198,25 @@ sphinxcontrib-htmlhelp==2.0.1
|
||||
sphinxcontrib-jsmath==1.0.1
|
||||
sphinxcontrib-qthelp==1.0.3
|
||||
sphinxcontrib-serializinghtml==1.1.5
|
||||
SQLAlchemy==1.3.20
|
||||
SQLAlchemy-JSONField==0.9.0
|
||||
SQLAlchemy==1.3.15
|
||||
sqlalchemy-migrate==0.13.0
|
||||
SQLAlchemy-Utils==0.41.0
|
||||
sqlparse==0.4.4
|
||||
statsd==4.0.1
|
||||
stevedore==5.0.0
|
||||
tabulate==0.8.10
|
||||
tabulate==0.8.3
|
||||
Tempita==0.5.2
|
||||
tenacity==4.12.0
|
||||
testfixtures==7.1.0
|
||||
termcolor==1.1.0
|
||||
testrepository==0.0.20
|
||||
testresources==2.0.1
|
||||
testscenarios==0.5.0
|
||||
testtools==2.5.0
|
||||
text-unidecode==1.3
|
||||
text-unidecode==1.2
|
||||
thrift==0.16.0
|
||||
thrift-sasl==0.4.3
|
||||
tiddlyweb==2.4.3
|
||||
toml==0.10.2
|
||||
tomli==2.0.1
|
||||
tomlkit==0.11.7
|
||||
tomlkit==0.11.8
|
||||
tornado==5.1.1
|
||||
tox==3.12.1
|
||||
typing-extensions==3.7.2
|
||||
tzlocal==1.5.1
|
||||
ulid==1.1
|
||||
@ -239,7 +224,7 @@ unicodecsv==0.14.1
|
||||
urllib3==1.25.11
|
||||
uWSGI==2.0.21
|
||||
vine==1.3.0
|
||||
virtualenv==20.22.0
|
||||
virtualenv==20.23.0
|
||||
wcwidth==0.2.6
|
||||
WebOb==1.8.7
|
||||
websocket-client==1.5.1
|
||||
|
||||
@ -6,3 +6,10 @@ testfixtures
|
||||
tox<=4.0.0
|
||||
typing-extensions==3.7.2
|
||||
pyflakes>=2.1.1
|
||||
# Pin apache-airflow 1.10.5 to align with version in airflow images requirements.txt
|
||||
apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.5
|
||||
# install postgres seperately as apache-airflow 1.10.5 forces postgres to use
|
||||
# psycopg2 instead of psycopg2-binary which requires additional apt packages
|
||||
# to be installed, i.e. postgres-devel. Otherwise the following error is seen:
|
||||
# Error: pg_config executable not found. Only seems to be an issue in xenial
|
||||
postgres>=3.0.0
|
||||
@ -7,7 +7,9 @@ setenv=
|
||||
PYTHONWARNING=all
|
||||
LANG=C.UTF-8
|
||||
LC_ALL=C.UTF-8
|
||||
deps = -r{toxinidir}/requirements-frozen.txt
|
||||
deps =
|
||||
-r{toxinidir}/requirements-frozen.txt
|
||||
-r{toxinidir}/test-requirements.txt
|
||||
|
||||
|
||||
[testenv:freeze]
|
||||
@ -17,7 +19,6 @@ allowlist_externals=
|
||||
sh
|
||||
deps=
|
||||
-r{toxinidir}/requirements-direct.txt
|
||||
-r{toxinidir}/test-requirements.txt
|
||||
commands=
|
||||
rm -f {toxinidir}/requirements-frozen.txt
|
||||
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"
|
||||
@ -27,9 +28,16 @@ commands=
|
||||
skipsdist=True
|
||||
setenv=
|
||||
SLUGIFY_USES_TEXT_UNIDECODE=yes
|
||||
allowlist_externals=
|
||||
bash
|
||||
airflow
|
||||
commands =
|
||||
bash -c "rm -f ${HOME}/airflow/airflow.db"
|
||||
pip install -r{toxinidir}/test-requirements.txt
|
||||
airflow initdb
|
||||
airflow run example_bash_operator runme_0 2018-01-01
|
||||
airflow backfill example_bash_operator -s 2018-01-01 -e 2018-01-02
|
||||
airflow dag_state example_bash_operator 2018-01-01
|
||||
pytest \
|
||||
{posargs} \
|
||||
--cov-branch \
|
||||
@ -74,6 +82,7 @@ commands =
|
||||
|
||||
[testenv:gen_all]
|
||||
# Combined to make these run together instead of setting up separately
|
||||
skipsdist=True
|
||||
setenv=
|
||||
SLUGIFY_USES_TEXT_UNIDECODE=yes
|
||||
deps =
|
||||
|
||||
@ -15,10 +15,19 @@
|
||||
# CLI/Client requirements
|
||||
|
||||
arrow<=0.17.0
|
||||
click==6.7
|
||||
click==7.1.2
|
||||
click-default-group==1.2
|
||||
keystoneauth1<=5.1.1
|
||||
requests==2.23.0
|
||||
PTable==0.9.2
|
||||
pylibyaml==0.1.0
|
||||
PyYAML<=5.4.1
|
||||
Flask==1.1.0
|
||||
coverage==5.3
|
||||
itsdangerous==2.0.1
|
||||
Jinja2==2.10.3
|
||||
MarkupSafe==2.0.1
|
||||
pbr==5.5.1
|
||||
python-dateutil==2.8.1
|
||||
setuptools<=45.2.0
|
||||
Werkzeug==0.16.1
|
||||
@ -1,40 +1,27 @@
|
||||
amqp==2.6.1
|
||||
arrow==0.17.0
|
||||
attrs==23.1.0
|
||||
bandit==1.6.0
|
||||
certifi==2022.12.7
|
||||
certifi==2023.5.7
|
||||
chardet==3.0.4
|
||||
click==6.7
|
||||
click==7.1.2
|
||||
click-default-group==1.2
|
||||
coverage==7.2.3
|
||||
flake8==3.8.4
|
||||
gitdb==4.0.10
|
||||
GitPython==3.1.31
|
||||
coverage==5.3
|
||||
Flask==1.1.0
|
||||
idna==2.10
|
||||
iso8601==1.1.0
|
||||
itsdangerous==2.0.1
|
||||
Jinja2==2.10.3
|
||||
keystoneauth1==5.1.1
|
||||
mccabe==0.6.1
|
||||
more-itertools==9.1.0
|
||||
MarkupSafe==2.0.1
|
||||
os-service-types==1.7.0
|
||||
pbr==5.11.1
|
||||
pbr==5.5.1
|
||||
pip==23.0.1
|
||||
pluggy==0.6.0
|
||||
PTable==0.9.2
|
||||
py==1.11.0
|
||||
pycodestyle==2.6.0
|
||||
pyflakes==2.2.0
|
||||
pylibyaml==0.1.0
|
||||
pytest==3.5.0
|
||||
pytest-cov==2.5.1
|
||||
python-dateutil==2.8.2
|
||||
python-dateutil==2.8.1
|
||||
PyYAML==5.4.1
|
||||
requests==2.23.0
|
||||
responses==0.10.2
|
||||
setuptools==67.6.0
|
||||
setuptools==45.2.0
|
||||
six==1.16.0
|
||||
smmap==5.0.0
|
||||
stevedore==5.0.0
|
||||
testfixtures==5.1.1
|
||||
urllib3==1.25.11
|
||||
vine==1.3.0
|
||||
Werkzeug==0.16.1
|
||||
wheel==0.40.0
|
||||
|
||||
@ -187,7 +187,7 @@ def test_create_configdocs_directory_empty():
|
||||
'--directory=' + dir1
|
||||
])
|
||||
_method.assert_not_called()
|
||||
assert b'directory does not contain any YAML files' in result.output_bytes
|
||||
assert b'directory does not contain any YAML files' in result.stdout_bytes
|
||||
|
||||
|
||||
def test_create_configdocs_multi_directory():
|
||||
|
||||
@ -17,7 +17,6 @@ allowlist_externals=
|
||||
sh
|
||||
deps=
|
||||
-r{toxinidir}/requirements-direct.txt
|
||||
-r{toxinidir}/test-requirements.txt
|
||||
commands=
|
||||
rm -f {toxinidir}/requirements-frozen.txt
|
||||
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"
|
||||
|
||||
@ -47,6 +47,7 @@ fi
|
||||
docker stop airflow_test
|
||||
docker rm airflow_test
|
||||
|
||||
|
||||
if [ ${TEST_RESP:(-7)} == "success" ]; then
|
||||
exit 0
|
||||
else
|
||||
|
||||
@ -22,6 +22,7 @@
|
||||
- ensure-docker
|
||||
|
||||
tasks:
|
||||
|
||||
- name: Clone Required Repositories
|
||||
shell: |
|
||||
export CLONE_SHIPYARD={{ CLONE_SHIPYARD }}
|
||||
@ -45,15 +46,23 @@
|
||||
- name: Build Shipyard and Airflow with submitted changes
|
||||
shell: |
|
||||
make
|
||||
docker pull quay.io/airshipit/shipyard:latest-ubuntu_bionic
|
||||
docker rm quay.io/airshipit/shipyard:latest-ubuntu_bionic
|
||||
docker tag quay.io/airshipit/shipyard:latest-ubuntu_focal quay.io/airshipit/shipyard:latest-ubuntu_bionic
|
||||
docker pull quay.io/airshipit/airflow:latest-ubuntu_bionic
|
||||
docker rm quay.io/airshipit/airflow:latest-ubuntu_bionic
|
||||
docker tag quay.io/airshipit/airflow:latest-ubuntu_focal quay.io/airshipit/airflow:latest-ubuntu_bionic
|
||||
args:
|
||||
chdir: "{{ zuul.project.src_dir }}"
|
||||
become: yes
|
||||
|
||||
- name: Setup clients
|
||||
- name: Build all charts locally
|
||||
shell: |
|
||||
./tools/deployment/airskiff/developer/020-setup-client.sh
|
||||
set -ex
|
||||
./tools/deployment/airskiff/developer/015-make-all-charts.sh
|
||||
args:
|
||||
chdir: "{{ zuul.projects['opendev.org/airship/treasuremap'].src_dir }}"
|
||||
become: yes
|
||||
|
||||
- name: Deploy Airship components using Armada
|
||||
shell: |
|
||||
|
||||
@ -38,7 +38,7 @@ set -e
|
||||
# Source Base Docker Command
|
||||
SHIPYARD_HOSTPATH=${SHIPYARD_HOSTPATH:-"/target"}
|
||||
NAMESPACE="${NAMESPACE:-ucp}"
|
||||
DISTRO="${DISTRO:-ubuntu_bionic}"
|
||||
DISTRO="${DISTRO:-ubuntu_focal}"
|
||||
SHIPYARD_IMAGE="${SHIPYARD_IMAGE:-quay.io/airshipit/shipyard:master-${DISTRO}}"
|
||||
# set default value for OS_PASSWORD if it's not set
|
||||
# this doesn't actually get exported to environment
|
||||
|
||||
@ -19,7 +19,7 @@
|
||||
# before executing this script if they differ from the default values.
|
||||
#
|
||||
NAMESPACE="${NAMESPACE:-ucp}"
|
||||
DISTRO="${DISTRO:-ubuntu_bionic}"
|
||||
DISTRO="${DISTRO:-ubuntu_focal}"
|
||||
SHIPYARD_IMAGE="${SHIPYARD_IMAGE:-quay.io/airshipit/shipyard:master-${DISTRO}}"
|
||||
|
||||
# Define Base Docker Command
|
||||
|
||||
35
tox.ini
35
tox.ini
@ -23,9 +23,6 @@ allowlist_externals=
|
||||
cp
|
||||
tox
|
||||
commands=
|
||||
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e gen_all
|
||||
cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample {toxinidir}/doc/source/_static/shipyard.policy.yaml.sample
|
||||
cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample {toxinidir}/doc/source/_static/shipyard.conf.sample
|
||||
rm -rf {toxinidir}/doc/build
|
||||
sphinx-build -W -b html {toxinidir}/doc/source {toxinidir}/doc/build/html
|
||||
deps= -r{toxinidir}/doc/requirements.txt
|
||||
@ -40,11 +37,17 @@ commands=
|
||||
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini
|
||||
|
||||
[testenv:py38]
|
||||
deps =
|
||||
tox<=4.0.0
|
||||
allowlist_externals=
|
||||
tox
|
||||
bash
|
||||
commands=
|
||||
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e py38
|
||||
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e py38
|
||||
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e gen_all
|
||||
bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample {toxinidir}/doc/source/_static/shipyard.policy.yaml.sample"
|
||||
bash -c "cp {toxinidir}/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample {toxinidir}/doc/source/_static/shipyard.conf.sample"
|
||||
|
||||
[testenv:pep8]
|
||||
allowlist_externals=
|
||||
@ -52,3 +55,29 @@ allowlist_externals=
|
||||
commands=
|
||||
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e pep8
|
||||
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e pep8
|
||||
|
||||
|
||||
[testenv:freeze]
|
||||
allowlist_externals=
|
||||
tox
|
||||
commands=
|
||||
tox -c {toxinidir}/src/bin/shipyard_airflow/tox.ini -e freeze
|
||||
tox -c {toxinidir}/src/bin/shipyard_client/tox.ini -e freeze
|
||||
|
||||
|
||||
[testenv:shipyard_client]
|
||||
deps =
|
||||
-r{toxinidir}/src/bin/shipyard_airflow/requirements-frozen.txt
|
||||
-r{toxinidir}/src/bin/shipyard_client/requirements-frozen.txt
|
||||
skipsdist=True
|
||||
allowlist_externals=
|
||||
pytest
|
||||
commands =
|
||||
pytest \
|
||||
{posargs} \
|
||||
--cov-branch \
|
||||
--cov-report=term-missing:skip-covered \
|
||||
--cov-config=.coveragerc \
|
||||
--cov=shipyard_client \
|
||||
--cov-report=html \
|
||||
-vv
|
||||
Loading…
x
Reference in New Issue
Block a user