From 35e5a7f796a5341e02fb0d3f2d27b41e2fb39131 Mon Sep 17 00:00:00 2001 From: Rick Bartra Date: Mon, 28 Sep 2020 15:14:25 -0400 Subject: [PATCH] Update pip package versions in preparation of pip 20.3 When pip is upgraded to 20.3, the pip dependency resolver is much more strict and will no longer install a combination of packages that is mutually inconsistent[0]. These changes account for the fact that Shipyard imports Armada, Drydock, Promenade, and Deckhand. Having said that, with pip 20.3, the pip packages amongst those projects cannot conflict. A follow-up change may be needed if more conflicts are found. Change-Id: Idd9ea0d57b5be063b133036cfc9ebaa69956f4fc --- images/airflow/Dockerfile.opensuse_15 | 5 ++- images/airflow/Dockerfile.ubuntu_bionic | 5 ++- images/airflow/Dockerfile.ubuntu_xenial | 5 ++- images/airflow/requirements.txt | 21 +++++++--- src/bin/shipyard_airflow/requirements.txt | 38 ++++++++++++------- .../shipyard_airflow/test-requirements.txt | 11 +++++- src/bin/shipyard_client/requirements.txt | 4 +- src/bin/shipyard_client/test-requirements.txt | 2 +- 8 files changed, 64 insertions(+), 27 deletions(-) diff --git a/images/airflow/Dockerfile.opensuse_15 b/images/airflow/Dockerfile.opensuse_15 index 9be4e295..06f8651e 100644 --- a/images/airflow/Dockerfile.opensuse_15 +++ b/images/airflow/Dockerfile.opensuse_15 @@ -40,7 +40,10 @@ EXPOSE $WORKER_PORT # Set ARG for usage during build ARG AIRFLOW_HOME=/usr/local/airflow -ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.5" +# Moved celery to images/airflow/requirements.txt as apache-airflow uses a +# version of celery incompatibile with the version of kombu needed by other +# Airship components +ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs,jdbc]==1.10.5" ARG ctx_base=src/bin # Kubectl version diff --git a/images/airflow/Dockerfile.ubuntu_bionic b/images/airflow/Dockerfile.ubuntu_bionic index feefb0ce..436b45d4 100644 --- a/images/airflow/Dockerfile.ubuntu_bionic +++ b/images/airflow/Dockerfile.ubuntu_bionic @@ -40,7 +40,10 @@ EXPOSE $WORKER_PORT # Set ARG for usage during build ARG AIRFLOW_HOME=/usr/local/airflow -ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.5" +# Moved celery to images/airflow/requirements.txt as apache-airflow uses a +# version of celery incompatibile with the version of kombu needed by other +# Airship components +ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs,jdbc]==1.10.5" ARG DEBIAN_FRONTEND=noninteractive ARG ctx_base=src/bin diff --git a/images/airflow/Dockerfile.ubuntu_xenial b/images/airflow/Dockerfile.ubuntu_xenial index a9eace88..da20d03a 100644 --- a/images/airflow/Dockerfile.ubuntu_xenial +++ b/images/airflow/Dockerfile.ubuntu_xenial @@ -40,7 +40,10 @@ EXPOSE $WORKER_PORT # Set ARG for usage during build ARG AIRFLOW_HOME=/usr/local/airflow -ARG AIRFLOW_SRC="apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.5" +# Moved celery to images/airflow/requirements.txt as apache-airflow uses a +# version of celery incompatibile with the version of kombu needed by other +# Airship components +ARG AIRFLOW_SRC="apache-airflow[crypto,postgres,hive,hdfs,jdbc]==1.10.5" ARG DEBIAN_FRONTEND=noninteractive ARG ctx_base=src/bin diff --git a/images/airflow/requirements.txt b/images/airflow/requirements.txt index d0cf1449..d15a37c6 100644 --- a/images/airflow/requirements.txt +++ b/images/airflow/requirements.txt @@ -12,22 +12,33 @@ # See the License for the specific language governing permissions and # limitations under the License. +celery==4.3 pytz==2018.5 +python-dateutil==2.8.1 pyOpenSSL==18.0.0 ndg-httpsclient==0.5.1 -pyasn1==0.4.4 +oslo.utils==3.42.1 +oslo.config==7.0.0 +oslo.serialization==2.29.2 +pyasn1<0.5.0,>=0.4.6 psycopg2-binary==2.8.4 -docker==3.5.0 +docker==3.7.2 # Airflow is now installed in the Dockerfile directory to allow for # overriding where it is sourced from python-openstackclient==3.16.1 +python-keystoneclient==3.22.0 kubernetes>=6.0.0 # Need to lock marshmellow-sqlalchemy and tabulate for compatibility issues marshmallow-sqlalchemy==0.18.0 WTForms~=2.2.1 tabulate==0.8.03 +pbr==5.4.5 +setuptools>=40.4.3 +six>=1.15.0 +urllib3==1.25.9 +vine<5.0.0a1,>=1.1.3 # Dependencies for other UCP components -git+https://opendev.org/airship/deckhand.git@e7ba6828a0a1ca27fae596f6e0ee5a857f28001d#egg=deckhand -git+https://opendev.org/airship/drydock.git@586bcf8ebed430b4de82edd9a527566ed39704b7#egg=drydock_provisioner&subdirectory=python -git+https://opendev.org/airship/armada.git@af8a9ffd0873c2fbc915794e235dbd357f2adab1#egg=armada +git+https://opendev.org/airship/deckhand.git@1f0c011a1708c1235a2be65edada7d386cd55d2a#egg=deckhand +git+https://opendev.org/airship/drydock.git@96db568d4b28f5f163efbf606854243614a6c821#egg=drydock_provisioner&subdirectory=python +git+https://opendev.org/airship/armada.git@0c3aff1ba1f0e836e3dd56de81b2ee560e11c179#egg=armada diff --git a/src/bin/shipyard_airflow/requirements.txt b/src/bin/shipyard_airflow/requirements.txt index 8b433417..2a425f5b 100644 --- a/src/bin/shipyard_airflow/requirements.txt +++ b/src/bin/shipyard_airflow/requirements.txt @@ -14,27 +14,37 @@ # API requirements -alembic==1.0.0 +alembic==1.0.1 arrow==0.12.1 -celery~=4.4.2 +celery==4.3 +kombu<4.7,>=4.6.10 +vine<5.0.0a1,>=1.1.3 configparser==3.5.0 -cryptography==2.3 +cryptography>=2.7 falcon==1.4.1 -jsonschema==2.6.0 +jsonschema>=3.0.1<4 grpcio>=1.16.0 -keystoneauth1==3.11.0 -keystonemiddleware==5.2.0 -networkx==2.1 # common/deployment_group -oslo.config==6.4.0 -oslo.policy==1.38.1 +keystoneauth1>=3.18.0 +keystonemiddleware==5.3.0 +networkx==2.2 # common/deployment_group +oslo.config==7.0.0 +oslo.policy==1.40.1 PasteDeploy==1.5.2 psycopg2-binary==2.8.4 pylibyaml~=0.1 -python-dateutil==2.7.3 +oslo.cache==1.38.1 +oslo.log==3.45.2 +oslo.utils==3.42.1 +oslo.serialization==2.29.2 +pbr==5.4.5 +python-dateutil==2.8.1 python-memcached==1.59 -requests==2.20.0 +python-keystoneclient==3.22.0 +requests!=2.20.0,>=2.14.2 +setuptools==40.4.3 SQLAlchemy==1.3.15 ulid==1.1 +urllib3==1.25.9 uwsgi~=2.0.19.1 # To support profiling in non-prod @@ -45,6 +55,6 @@ marshmallow-sqlalchemy==0.18.0 tabulate==0.8.03 # Dependencies for other UCP components -git+https://opendev.org/airship/deckhand.git@e7ba6828a0a1ca27fae596f6e0ee5a857f28001d#egg=deckhand -git+https://opendev.org/airship/drydock.git@586bcf8ebed430b4de82edd9a527566ed39704b7#egg=drydock_provisioner&subdirectory=python -git+https://opendev.org/airship/armada.git@af8a9ffd0873c2fbc915794e235dbd357f2adab1#egg=armada +git+https://opendev.org/airship/deckhand.git@1f0c011a1708c1235a2be65edada7d386cd55d2a#egg=deckhand +git+https://opendev.org/airship/drydock.git@96db568d4b28f5f163efbf606854243614a6c821#egg=drydock_provisioner&subdirectory=python +git+https://opendev.org/airship/armada.git@0c3aff1ba1f0e836e3dd56de81b2ee560e11c179#egg=armada diff --git a/src/bin/shipyard_airflow/test-requirements.txt b/src/bin/shipyard_airflow/test-requirements.txt index 75ba6c2f..7e5e50f1 100644 --- a/src/bin/shipyard_airflow/test-requirements.txt +++ b/src/bin/shipyard_airflow/test-requirements.txt @@ -5,14 +5,21 @@ pytest==3.5.0 pytest-cov==2.5.1 responses==0.10.2 testfixtures==5.1.1 -apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.10.1 +# Had to upgrade apache-airflow as 1.10.1 requires a version of jinja too old +# for promenade +apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.2 +# install postgres seperately as apache-airflow 1.10.2 forces postgres to use +# psycopg2 instead of psycopg2-binary which requires additional apt packages +# to be installed, i.e. postgres-devel. Otherwise the following error is seen: +# Error: pg_config executable not found. Only seems to be an issue in xenial +postgres>=3.0.0 # TODO(bryan-strassner) Pin to version for airflow when added to the # requirements.txt in the airflow images directory git+https://opendev.org/airship/promenade.git@master#egg=promenade # Linting -flake8>=3.3.0 +flake8<3.8.0,>=3.6.0 # Security scanning bandit>=1.5.0 # Apache-2.0 diff --git a/src/bin/shipyard_client/requirements.txt b/src/bin/shipyard_client/requirements.txt index 6e1b6e21..c4eed4f6 100644 --- a/src/bin/shipyard_client/requirements.txt +++ b/src/bin/shipyard_client/requirements.txt @@ -17,8 +17,8 @@ arrow==0.12.1 click==6.7 click-default-group==1.2 -keystoneauth1==3.11.1 -requests==2.20.0 +keystoneauth1>=3.18.0 +requests!=2.20.0,>=2.14.2 PTable==0.9.2 pylibyaml==0.1.0 pyyaml==5.1 diff --git a/src/bin/shipyard_client/test-requirements.txt b/src/bin/shipyard_client/test-requirements.txt index a5f9b1fb..515ab1ae 100644 --- a/src/bin/shipyard_client/test-requirements.txt +++ b/src/bin/shipyard_client/test-requirements.txt @@ -7,7 +7,7 @@ responses==0.10.2 testfixtures==5.1.1 # Linting -flake8>=3.3.0 +flake8<3.8.0,>=3.6.0 # Security scanning bandit>=1.1.0 # Apache-2.0