Airflow and Openstack modules upgrade

This PS upgrades Airflow to 2.10.4 and Openstack
modules to 2024.1 Caracal versions

Change-Id: I034cdf334f04f3d08af1891ac8db9ed9fd7177c8
This commit is contained in:
Sergiy Markin 2024-12-17 19:25:24 +00:00
parent dc275f7e2a
commit 96c9ddd64f
8 changed files with 261 additions and 183 deletions

View File

@ -16,8 +16,8 @@
apiVersion: v1
description: A Helm chart for Shipyard and Airflow
name: shipyard
version: 0.2.7
appVersion: 2.10.2
version: 0.2.8
appVersion: 2.10.4
keywords:
- shipyard
- airflow

View File

@ -44,7 +44,7 @@ ARG AIRFLOW_HOME=/usr/local/airflow
# Moved celery to images/airflow/requirements.txt as apache-airflow uses a
# version of celery incompatibile with the version of kombu needed by other
# Airship components
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.10.2"
ARG AIRFLOW_SRC="apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.10.4"
ARG DEBIAN_FRONTEND=noninteractive
ARG ctx_base=src/bin

View File

@ -1,5 +1,5 @@
alembic
apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.10.2
apache-airflow[crypto,celery,hive,hdfs,jdbc,postgres]==2.10.4
arrow
celery
Flask
@ -30,36 +30,36 @@ Werkzeug
WTForms
itsdangerous
# Openstack Antelope 2023.1
# https://releases.openstack.org/antelope/index.html
barbican==16.0.0
# Openstack Caracal 2024.1
# https://releases.openstack.org/caracal/index.html
barbican==18.0.0
python-barbicanclient==5.5.0
python-keystoneclient==5.1.0
python-barbicanclient==5.7.0
python-keystoneclient==5.4.0
keystoneauth1==5.1.2
keystonemiddleware==10.2.0
keystoneauth1==5.6.0
keystonemiddleware==10.6.0
oslo.cache==3.3.1
oslo.concurrency==5.1.1
oslo.config==9.1.1
oslo.context==5.1.1
oslo.db==12.3.1
oslo.i18n==6.0.0
oslo.log==5.2.0
oslo.messaging==14.2.4
oslo.metrics==0.6.0
oslo.middleware==5.1.1
oslo.policy==4.1.1
oslo.serialization==5.1.1
oslo.service==3.1.1
oslo.upgradecheck==2.1.1
oslo.utils==6.1.0
oslo.versionedobjects==3.1.0
oslo.cache==3.7.0
oslo.concurrency==6.0.0
oslo.config==9.4.0
oslo.context==5.5.0
oslo.db==15.0.0
oslo.i18n==6.3.0
oslo.log==5.5.1
oslo.messaging==14.7.2
oslo.metrics==0.8.0
oslo.middleware==6.1.0
oslo.policy==4.3.0
oslo.serialization==5.4.1
oslo.service==3.4.1
oslo.upgradecheck==2.3.0
oslo.utils==7.1.0
oslo.versionedobjects==3.3.0
# Airship dependencies
git+https://opendev.org/airship/deckhand.git@be9f97b846b1edc6b74777e5e639aa9edce8cfa6#egg=deckhand
git+https://opendev.org/airship/drydock.git@146f1debbaf85f66e22cf8c546c3f343f0180ec9#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@edf8b5b88b726691f0dae23ec911e8d539c49211#egg=armada
git+https://opendev.org/airship/promenade.git@ac573b9fb53fd543172c50bbbcdfac3656fb8e66#egg=promenade
git+https://opendev.org/airship/deckhand.git@dacedae17b71acd249dc01ad0540e58136245fe1#egg=deckhand
git+https://opendev.org/airship/drydock.git@683b03dfe11101586ed9e37bd8a667cd59a4f2f3#egg=drydock_provisioner&subdirectory=python
git+https://opendev.org/airship/armada.git@54c3ebd63efbe21ed964cd8ee11274a5c2d95390#egg=armada
git+https://opendev.org/airship/promenade.git@4283b05cbee451007edde96951d31ee6b243a642#egg=promenade

View File

@ -1,139 +1,137 @@
aiohappyeyeballs==2.4.0
aiohttp==3.10.5
aiohappyeyeballs==2.4.4
aiohttp==3.10.11
aiosignal==1.3.1
alembic==1.13.2
amqp==5.2.0
anyio==4.4.0
apache-airflow==2.10.2
apache-airflow-providers-apache-hdfs==4.5.0
apache-airflow-providers-apache-hive==8.2.0
apache-airflow-providers-celery==3.8.1
apache-airflow-providers-common-compat==1.2.0
apache-airflow-providers-common-io==1.4.0
apache-airflow-providers-common-sql==1.16.0
apache-airflow-providers-fab==1.3.0
apache-airflow-providers-ftp==3.11.0
apache-airflow-providers-http==4.13.0
alembic==1.14.0
amqp==5.3.1
anyio==4.7.0
apache-airflow==2.10.4
apache-airflow-providers-apache-hdfs==4.6.0
apache-airflow-providers-apache-hive==8.2.1
apache-airflow-providers-celery==3.8.5
apache-airflow-providers-common-compat==1.2.2
apache-airflow-providers-common-io==1.4.2
apache-airflow-providers-common-sql==1.20.0
apache-airflow-providers-fab==1.5.1
apache-airflow-providers-ftp==3.11.1
apache-airflow-providers-http==4.13.3
apache-airflow-providers-imap==3.7.0
apache-airflow-providers-jdbc==4.5.0
apache-airflow-providers-postgres==5.12.0
apache-airflow-providers-smtp==1.8.0
apache-airflow-providers-sqlite==3.9.0
apispec==6.6.1
argcomplete==3.5.0
Armada @ git+https://opendev.org/airship/armada.git@edf8b5b88b726691f0dae23ec911e8d539c49211
apache-airflow-providers-jdbc==4.5.3
apache-airflow-providers-postgres==5.14.0
apache-airflow-providers-smtp==1.8.1
apache-airflow-providers-sqlite==3.9.1
apispec==6.8.0
argcomplete==3.5.2
Armada @ git+https://opendev.org/airship/armada.git@54c3ebd63efbe21ed964cd8ee11274a5c2d95390
arrow==1.3.0
asgiref==3.8.1
async-timeout==4.0.3
async-timeout==5.0.1
attrs==24.2.0
autopage==0.5.2
babel==2.16.0
barbican==16.0.0
bcrypt==4.2.0
barbican==18.0.0
bcrypt==4.2.1
Beaker==1.13.0
billiard==4.2.0
blinker==1.8.2
billiard==4.2.1
blinker==1.9.0
cachelib==0.9.0
cachetools==5.5.0
castellan==5.1.1
castellan==5.2.0
celery==5.4.0
certifi==2024.8.30
cffi==1.17.1
charset-normalizer==3.3.2
charset-normalizer==3.4.0
click==8.1.7
click-didyoumean==0.3.1
click-plugins==1.1.1
click-repl==0.3.0
clickclick==20.10.2
cliff==4.7.0
cmd2==2.4.3
cliff==4.8.0
cmd2==2.5.8
colorama==0.4.6
colorlog==6.8.2
colorlog==6.9.0
ConfigUpdater==3.2
connexion==2.14.2
cron-descriptor==1.4.5
croniter==3.0.3
croniter==5.0.1
cryptography==42.0.8
debtcollector==3.0.0
Deckhand @ git+https://opendev.org/airship/deckhand.git@be9f97b846b1edc6b74777e5e639aa9edce8cfa6
Deckhand @ git+https://opendev.org/airship/deckhand.git@dacedae17b71acd249dc01ad0540e58136245fe1
decorator==5.1.1
deepdiff==8.0.1
Deprecated==1.2.14
deepdiff==8.1.1
Deprecated==1.2.15
dill==0.3.1.1
dnspython==2.6.1
dnspython==2.7.0
docopt==0.6.2
docutils==0.16
dogpile.cache==1.3.3
drydock_provisioner @ git+https://opendev.org/airship/drydock.git@146f1debbaf85f66e22cf8c546c3f343f0180ec9#subdirectory=python
drydock_provisioner @ git+https://opendev.org/airship/drydock.git@683b03dfe11101586ed9e37bd8a667cd59a4f2f3#subdirectory=python
email_validator==2.2.0
eventlet==0.37.0
eventlet==0.38.1
exceptiongroup==1.2.2
falcon==3.1.3
falcon==4.0.2
fastavro==1.9.7
fasteners==0.19
fixtures==4.1.0
Flask==2.2.5
Flask-AppBuilder==4.5.0
Flask-AppBuilder==4.5.2
Flask-Babel==2.0.0
Flask-Caching==2.3.0
Flask-JWT-Extended==4.6.0
Flask-Limiter==3.8.0
Flask-JWT-Extended==4.7.1
Flask-Limiter==3.9.2
Flask-Login==0.6.3
Flask-Session==0.5.0
Flask-SQLAlchemy==2.5.1
Flask-WTF==1.2.1
Flask-WTF==1.2.2
flower==2.0.1
frozenlist==1.4.1
fsspec==2024.9.0
frozenlist==1.5.0
fsspec==2024.10.0
future==1.0.0
futurist==3.0.0
gitdb==4.0.11
GitPython==3.1.43
google-auth==2.34.0
google-auth==2.36.0
google-re2==1.1.20240702
googleapis-common-protos==1.65.0
googleapis-common-protos==1.66.0
graphviz==0.20.3
greenlet==3.1.0
grpcio==1.66.1
gssapi==1.8.3
greenlet==3.1.1
grpcio==1.65.5
gssapi==1.9.0
gunicorn==23.0.0
h11==0.14.0
hdfs==2.7.3
hmsclient==0.1.1
html5lib==0.9999999
httpcore==1.0.5
httpcore==1.0.7
httpexceptor==1.4.0
httpx==0.27.0
humanize==4.10.0
humanize==4.11.0
idna==3.10
importlib-metadata==6.11.0
importlib_resources==6.4.5
inflection==0.5.1
iso8601==2.1.0
itsdangerous==2.2.0
JayDeBeApi==1.2.3
Jinja2==3.1.4
jmespath==0.10.0
JPype1==1.5.0
jsonpath-ng==1.6.1
jsonpickle==3.3.0
jpype1==1.5.1
jsonpath-ng==1.7.0
jsonpickle==3.4.2
jsonschema==4.23.0
jsonschema-specifications==2023.12.1
keystoneauth1==5.1.2
keystonemiddleware==10.2.0
kombu==5.4.1
krb5==0.6.0
keystoneauth1==5.6.0
keystonemiddleware==10.6.0
kombu==5.4.2
krb5==0.7.0
kubernetes==30.1.0
lazy-object-proxy==1.10.0
ldap3==2.9.1
limits==3.13.0
limits==3.14.1
linkify-it-py==2.0.3
lockfile==0.12.2
logutils==0.3.5
Mako==1.3.5
Mako==1.3.8
markdown-it-py==3.0.0
MarkupSafe==2.1.5
marshmallow==3.22.0
MarkupSafe==3.0.2
marshmallow==3.23.1
marshmallow-oneofschema==3.1.1
marshmallow-sqlalchemy==0.28.2
mdit-py-plugins==0.4.2
@ -146,7 +144,7 @@ msgpack==1.1.0
multidict==6.1.0
netaddr==1.3.0
netifaces==0.11.0
networkx==3.3
networkx==3.4.2
numpy==1.26.4
oauthlib==3.2.2
opentelemetry-api==1.27.0
@ -158,25 +156,25 @@ opentelemetry-proto==1.27.0
opentelemetry-sdk==1.27.0
opentelemetry-semantic-conventions==0.48b0
ordered-set==4.1.0
orderly-set==5.2.2
orderly-set==5.2.3
os-service-types==1.7.0
oslo.cache==3.3.1
oslo.concurrency==5.1.1
oslo.config==9.1.1
oslo.context==5.1.1
oslo.db==12.3.1
oslo.i18n==6.0.0
oslo.log==5.2.0
oslo.messaging==14.2.4
oslo.metrics==0.6.0
oslo.middleware==5.1.1
oslo.policy==4.1.1
oslo.serialization==5.1.1
oslo.service==3.1.1
oslo.upgradecheck==2.1.1
oslo.utils==6.1.0
oslo.versionedobjects==3.1.0
packaging==24.1
oslo.cache==3.7.0
oslo.concurrency==6.0.0
oslo.config==9.4.0
oslo.context==5.5.0
oslo.db==15.0.0
oslo.i18n==6.3.0
oslo.log==5.5.1
oslo.messaging==14.7.2
oslo.metrics==0.8.0
oslo.middleware==6.1.0
oslo.policy==4.3.0
oslo.serialization==5.4.1
oslo.service==3.4.1
oslo.upgradecheck==2.3.0
oslo.utils==7.1.0
oslo.versionedobjects==3.3.0
packaging==24.2
pandas==2.1.4
Paste==3.10.1
PasteDeploy==3.1.0
@ -188,33 +186,34 @@ pendulum==3.0.0
pip==24.1
pluggy==1.5.0
ply==3.11
prettytable==3.11.0
prettytable==3.12.0
prison==0.2.1
promenade @ git+https://opendev.org/airship/promenade.git@ac573b9fb53fd543172c50bbbcdfac3656fb8e66
prometheus_client==0.20.0
prompt_toolkit==3.0.47
protobuf==4.25.4
psutil==6.0.0
psycopg2-binary==2.9.9
promenade @ git+https://opendev.org/airship/promenade.git@4283b05cbee451007edde96951d31ee6b243a642
prometheus_client==0.21.1
prompt_toolkit==3.0.48
propcache==0.2.1
protobuf==4.25.5
psutil==6.1.0
psycopg2-binary==2.9.10
pure-sasl==0.6.2
py==1.11.0
pyarrow==16.1.0
pyasn1==0.6.1
pyasn1_modules==0.4.0
pycadf==3.1.1
pycadf==4.0.0
pycparser==2.22
Pygments==2.18.0
PyHive==0.7.0
PyJWT==2.9.0
PyJWT==2.10.1
pylibyaml==0.1.0
pyOpenSSL==24.2.1
pyparsing==3.1.4
pyOpenSSL==24.3.0
pyparsing==3.2.0
pyperclip==1.9.0
pyspnego==0.11.1
python-barbicanclient==5.5.0
python-daemon==3.0.1
pyspnego==0.11.2
python-barbicanclient==5.7.0
python-daemon==3.1.2
python-dateutil==2.9.0.post0
python-keystoneclient==5.1.0
python-keystoneclient==5.4.0
python-memcached==1.62
python-mimeparse==2.0.0
python-nvd3==0.16.0
@ -222,9 +221,9 @@ python-slugify==8.0.4
python3-memcached==1.51
pytz==2024.2
PyYAML==6.0.2
redis==5.0.8
redis==5.2.1
referencing==0.35.1
regex==2024.9.11
regex==2024.11.6
repoze.lru==0.7
requests==2.32.3
requests-kerberos==0.15.0
@ -235,29 +234,27 @@ responses==0.25.3
retry==0.9.2
rfc3339-validator==0.1.4
rfc3986==2.0.0
rich==13.8.1
rich-argparse==1.5.2
rich==13.9.4
rich-argparse==1.6.0
Routes==2.5.1
rpds-py==0.20.0
rpds-py==0.22.3
rsa==4.9
selector==0.10.1
setproctitle==1.3.3
setproctitle==1.3.4
setuptools==70.1.0
simplejson==3.19.3
six==1.16.0
six==1.17.0
smmap==5.0.1
sniffio==1.3.1
SQLAlchemy==1.4.54
SQLAlchemy-JSONField==1.0.2
sqlalchemy-migrate==0.13.0
SQLAlchemy-Utils==0.41.2
sqlparse==0.5.1
sqlparse==0.5.3
statsd==4.0.1
stevedore==5.3.0
stevedore==5.4.0
tabulate==0.9.0
Tempita==0.5.2
tenacity==8.5.0
termcolor==2.4.0
tenacity==9.0.0
termcolor==2.5.0
testresources==2.0.1
testscenarios==0.5.0
testtools==2.7.2
@ -265,27 +262,26 @@ text-unidecode==1.3
thrift==0.16.0
thrift-sasl==0.4.3
tiddlyweb==2.4.3
time-machine==2.15.0
tornado==6.4.1
types-python-dateutil==2.9.0.20240906
time-machine==2.16.0
tornado==6.4.2
types-python-dateutil==2.9.0.20241206
typing_extensions==4.12.2
tzdata==2024.1
tzdata==2024.2
uc-micro-py==1.0.3
ulid==1.1
unicodecsv==0.14.1
universal_pathlib==0.2.5
urllib3==2.2.2
uWSGI==2.0.27
urllib3==2.2.3
uWSGI==2.0.28
vine==5.1.0
wcwidth==0.2.13
WebOb==1.8.8
WebOb==1.8.9
websocket-client==1.8.0
Werkzeug==2.2.3
wheel==0.43.0
wirerope==0.4.7
wrapt==1.16.0
WTForms==3.1.2
wirerope==0.4.8
wrapt==1.17.0
WTForms==3.2.1
xattr==0.10.1
yappi==1.6.0
yarl==1.11.1
zipp==3.20.2
yappi==1.6.10
yarl==1.18.3
zipp==3.21.0

View File

@ -17,7 +17,7 @@ allowlist_externals=
sh
deps=
-r{toxinidir}/requirements-direct.txt
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.4/constraints-3.10.txt
commands=
rm -f {toxinidir}/requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"

View File

@ -29,13 +29,29 @@ pbr
python-dateutil
Werkzeug
# Openstack Antelope 2023.1
# https://releases.openstack.org/antelope/index.html
python-keystoneclient==5.1.0
# Openstack Caracal 2024.1
# https://releases.openstack.org/caracal/index.html
barbican==18.0.0
keystoneauth1==5.1.2
python-barbicanclient==5.7.0
python-keystoneclient==5.4.0
oslo.config==9.1.1
oslo.i18n==6.0.0
oslo.serialization==5.1.1
oslo.utils==6.1.0
keystoneauth1==5.6.0
keystonemiddleware==10.6.0
oslo.cache==3.7.0
oslo.concurrency==6.0.0
oslo.config==9.4.0
oslo.context==5.5.0
oslo.db==15.0.0
oslo.i18n==6.3.0
oslo.log==5.5.1
oslo.messaging==14.7.2
oslo.metrics==0.8.0
oslo.middleware==6.1.0
oslo.policy==4.3.0
oslo.serialization==5.4.1
oslo.service==3.4.1
oslo.upgradecheck==2.3.0
oslo.utils==7.1.0
oslo.versionedobjects==3.3.0

View File

@ -1,41 +1,107 @@
alembic==1.14.0
amqp==5.3.1
arrow==1.3.0
attrs==24.2.0
autopage==0.5.2
barbican==18.0.0
bcrypt==4.2.1
cachetools==5.5.0
castellan==5.2.0
certifi==2024.8.30
charset-normalizer==3.3.2
cffi==1.17.1
charset-normalizer==3.4.0
click==8.1.7
click-default-group==1.2.4
cliff==4.8.0
cmd2==2.5.8
cryptography==42.0.8
debtcollector==3.0.0
decorator==5.1.1
dnspython==2.7.0
dogpile.cache==1.3.3
eventlet==0.38.1
fasteners==0.19
Flask==2.2.5
futurist==3.0.0
greenlet==3.1.1
idna==3.10
iso8601==2.1.0
itsdangerous==2.2.0
Jinja2==3.1.4
keystoneauth1==5.1.2
MarkupSafe==2.1.5
jsonschema==4.23.0
jsonschema-specifications==2023.12.1
keystoneauth1==5.6.0
keystonemiddleware==10.6.0
kombu==5.4.2
ldap3==2.9.1
logutils==0.3.5
Mako==1.3.8
MarkupSafe==3.0.2
microversion-parse==2.0.0
msgpack==1.1.0
netaddr==1.3.0
netifaces==0.11.0
os-service-types==1.7.0
oslo.config==9.1.1
oslo.i18n==6.0.0
oslo.serialization==5.1.1
oslo.utils==6.1.0
packaging==24.1
oslo.cache==3.7.0
oslo.concurrency==6.0.0
oslo.config==9.4.0
oslo.context==5.5.0
oslo.db==15.0.0
oslo.i18n==6.3.0
oslo.log==5.5.1
oslo.messaging==14.7.2
oslo.metrics==0.8.0
oslo.middleware==6.1.0
oslo.policy==4.3.0
oslo.serialization==5.4.1
oslo.service==3.4.1
oslo.upgradecheck==2.3.0
oslo.utils==7.1.0
oslo.versionedobjects==3.3.0
packaging==24.2
Paste==3.10.1
PasteDeploy==3.1.0
pbr==6.1.0
pecan==1.5.1
pip==24.1
prettytable==3.12.0
prometheus_client==0.21.1
PTable==0.9.2
pyasn1==0.6.1
pycadf==4.0.0
pycparser==2.22
PyJWT==2.10.1
pylibyaml==0.1.0
pyparsing==3.1.4
pyOpenSSL==24.3.0
pyparsing==3.2.0
pyperclip==1.9.0
python-barbicanclient==5.7.0
python-dateutil==2.9.0.post0
python-keystoneclient==5.1.0
python-keystoneclient==5.4.0
pytz==2024.2
PyYAML==6.0.2
referencing==0.35.1
repoze.lru==0.7
requests==2.32.3
rfc3986==2.0.0
Routes==2.5.1
rpds-py==0.22.3
setuptools==70.1.0
six==1.16.0
stevedore==5.3.0
types-python-dateutil==2.9.0.20240906
urllib3==2.2.2
six==1.17.0
SQLAlchemy==1.4.54
statsd==4.0.1
stevedore==5.4.0
testresources==2.0.1
testscenarios==0.5.0
testtools==2.7.2
types-python-dateutil==2.9.0.20241206
typing_extensions==4.12.2
tzdata==2024.2
urllib3==2.2.3
vine==5.1.0
wcwidth==0.2.13
WebOb==1.8.9
Werkzeug==2.2.3
wheel==0.43.0
wrapt==1.16.0
wrapt==1.17.0
yappi==1.6.10

View File

@ -18,7 +18,7 @@ allowlist_externals=
sh
deps=
-r{toxinidir}/requirements-direct.txt
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.2/constraints-3.10.txt
-c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.4/constraints-3.10.txt
commands=
rm -f {toxinidir}/requirements-frozen.txt
sh -c "pip freeze --all | grep -vE 'shipyard_airflow|pyinotify|pkg-resources' > requirements-frozen.txt"