From ebe2497d3cfe158ba39003c51e94e6ad96d4942f Mon Sep 17 00:00:00 2001 From: "anthony.bellino" Date: Thu, 20 May 2021 07:18:53 -0700 Subject: [PATCH] [WIP] Gate fixes for Shipyard * Install older version of pip<21.0 for ubuntu_xenial images * Install setuptools via pip for ubuntu_xenial images Change-Id: Iee4ce59fdceacb165120a69d11c44e6e47feaea8 --- images/airflow/Dockerfile.ubuntu_xenial | 4 ++-- images/shipyard/Dockerfile.ubuntu_xenial | 4 ++-- src/bin/shipyard_airflow/test-requirements.txt | 5 +++-- .../deployment_group/test_deployment_group_manager.py | 8 +++++--- src/bin/shipyard_airflow/tox.ini | 7 +++++++ src/bin/shipyard_client/tox.ini | 2 ++ 6 files changed, 21 insertions(+), 9 deletions(-) diff --git a/images/airflow/Dockerfile.ubuntu_xenial b/images/airflow/Dockerfile.ubuntu_xenial index da20d03a..aaad425d 100644 --- a/images/airflow/Dockerfile.ubuntu_xenial +++ b/images/airflow/Dockerfile.ubuntu_xenial @@ -69,13 +69,13 @@ RUN set -ex && \ netcat \ netbase \ python3 \ - python3-setuptools \ python3-pip \ python3-dev \ python3-dateutil \ make \ --no-install-recommends \ - && python3 -m pip install -U pip \ + && python3 -m pip install -U 'pip<21.0' \ + && python3 -m pip install -U setuptools \ && apt-get clean \ && rm -rf \ /var/lib/apt/lists/* \ diff --git a/images/shipyard/Dockerfile.ubuntu_xenial b/images/shipyard/Dockerfile.ubuntu_xenial index 996f8f18..0ff2e621 100644 --- a/images/shipyard/Dockerfile.ubuntu_xenial +++ b/images/shipyard/Dockerfile.ubuntu_xenial @@ -46,7 +46,6 @@ RUN set -ex && \ make \ netbase \ python3-dev \ - python3-setuptools \ --no-install-recommends \ && apt-get autoremove -yqq --purge \ && apt-get clean \ @@ -101,7 +100,8 @@ COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/ ' \ && apt-get -qq update \ && apt-get -y install -y $buildDeps --no-install-recommends \ - && python3 -m pip install -U pip \ + && python3 -m pip install -U 'pip<21.0' \ + && python3 -m pip install -U setuptools \ && pip3 install -r /home/shipyard/client_requirements.txt --no-cache-dir \ && cd /home/shipyard/shipyard_client \ && python3 setup.py install \ diff --git a/src/bin/shipyard_airflow/test-requirements.txt b/src/bin/shipyard_airflow/test-requirements.txt index 7e5e50f1..7a8f9b22 100644 --- a/src/bin/shipyard_airflow/test-requirements.txt +++ b/src/bin/shipyard_airflow/test-requirements.txt @@ -5,9 +5,10 @@ pytest==3.5.0 pytest-cov==2.5.1 responses==0.10.2 testfixtures==5.1.1 +typing-extensions==3.7.2 # Had to upgrade apache-airflow as 1.10.1 requires a version of jinja too old # for promenade -apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.2 +apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.4 # install postgres seperately as apache-airflow 1.10.2 forces postgres to use # psycopg2 instead of psycopg2-binary which requires additional apt packages # to be installed, i.e. postgres-devel. Otherwise the following error is seen: @@ -16,7 +17,7 @@ postgres>=3.0.0 # TODO(bryan-strassner) Pin to version for airflow when added to the # requirements.txt in the airflow images directory -git+https://opendev.org/airship/promenade.git@master#egg=promenade +#git+https://opendev.org/airship/promenade.git@master#egg=promenade # Linting flake8<3.8.0,>=3.6.0 diff --git a/src/bin/shipyard_airflow/tests/unit/common/deployment_group/test_deployment_group_manager.py b/src/bin/shipyard_airflow/tests/unit/common/deployment_group/test_deployment_group_manager.py index 4c459b68..53a1c8d6 100644 --- a/src/bin/shipyard_airflow/tests/unit/common/deployment_group/test_deployment_group_manager.py +++ b/src/bin/shipyard_airflow/tests/unit/common/deployment_group/test_deployment_group_manager.py @@ -15,6 +15,7 @@ module """ import pytest +import re import yaml from shipyard_airflow.common.deployment_group.deployment_group import ( @@ -197,13 +198,14 @@ class TestDeploymentGroupManager: dgm.mark_group_prepared('compute-nodes-1') dgm.mark_group_failed('compute-nodes-2') dgm.report_group_summary() + ansi_re = re.compile(r'\x1b\[[0-9;]*m') assert "===== Group Summary =====" in caplog.text assert ("Group ntp-node [Critical] ended with stage: " - "Stage.NOT_STARTED") in caplog.text + "Stage.NOT_STARTED") in re.sub(ansi_re, '', caplog.text) caplog.clear() dgm.report_node_summary() - assert "Nodes Stage.PREPARED: node2" in caplog.text - assert "Nodes Stage.FAILED: node3" in caplog.text + assert "Nodes Stage.PREPARED: node2" in re.sub(ansi_re, '', caplog.text) + assert "Nodes Stage.FAILED: node3" in re.sub(ansi_re, '', caplog.text) assert "===== End Node Summary =====" in caplog.text assert "It was the best of times" not in caplog.text diff --git a/src/bin/shipyard_airflow/tox.ini b/src/bin/shipyard_airflow/tox.ini index 537cb57f..3b54a02c 100644 --- a/src/bin/shipyard_airflow/tox.ini +++ b/src/bin/shipyard_airflow/tox.ini @@ -1,17 +1,23 @@ [tox] +skipsdist=True envlist = py35, py36, pep8, bandit [testenv] setenv= PYTHONWARNING=all + LANG=C.UTF-8 + LC_ALL=C.UTF-8 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt +commands = + python3 {toxinidir}/setup.py sdist --formats=zip --dist-dir {toxinidir}/.tox/dist [testenv:py35] skipsdist=True setenv= SLUGIFY_USES_TEXT_UNIDECODE=yes commands = + airflow initdb pytest \ {posargs} \ --cov-branch \ @@ -25,6 +31,7 @@ skipsdist=True setenv= SLUGIFY_USES_TEXT_UNIDECODE=yes commands = + airflow initdb pytest \ {posargs} \ --cov-branch \ diff --git a/src/bin/shipyard_client/tox.ini b/src/bin/shipyard_client/tox.ini index 6a030ffa..256f569e 100644 --- a/src/bin/shipyard_client/tox.ini +++ b/src/bin/shipyard_client/tox.ini @@ -5,6 +5,8 @@ envlist = py35, py36, pep8, bandit [testenv] setenv= PYTHONWARNING=all + LC_ALL=C.UTF-8 + LC_ALL=C.UTF-8 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt