[WIP] Gate fixes for Shipyard

* Install older version of pip<21.0 for ubuntu_xenial images
* Install setuptools via pip for ubuntu_xenial images

Change-Id: Iee4ce59fdceacb165120a69d11c44e6e47feaea8

images/airflow/Dockerfile.ubuntu_xenial

@@ -69,13 +69,13 @@ RUN set -ex && \
     netcat \
     netbase \
     python3 \
-    python3-setuptools \
     python3-pip \
     python3-dev \
     python3-dateutil \
     make \
     --no-install-recommends \
-    && python3 -m pip install -U pip \
+    && python3 -m pip install -U 'pip<21.0' \
+    && python3 -m pip install -U setuptools \
     && apt-get clean \
     && rm -rf \
         /var/lib/apt/lists/* \

images/shipyard/Dockerfile.ubuntu_xenial

@@ -46,7 +46,6 @@ RUN set -ex && \
     make \
     netbase \
     python3-dev \
-    python3-setuptools \
     --no-install-recommends \
     && apt-get autoremove -yqq --purge \
     && apt-get clean \
@@ -101,7 +100,8 @@ COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/
     ' \
     && apt-get -qq update \
     && apt-get -y install -y $buildDeps --no-install-recommends \
-    && python3 -m pip install -U pip \
+    && python3 -m pip install -U 'pip<21.0' \
+    && python3 -m pip install -U setuptools \
     && pip3 install -r /home/shipyard/client_requirements.txt --no-cache-dir \
     && cd /home/shipyard/shipyard_client \
     && python3 setup.py install \

src/bin/shipyard_airflow/test-requirements.txt

@@ -5,9 +5,10 @@ pytest==3.5.0
 pytest-cov==2.5.1
 responses==0.10.2
 testfixtures==5.1.1
+typing-extensions==3.7.2
 # Had to upgrade apache-airflow as 1.10.1 requires a version of jinja too old
 # for promenade
-apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.2
+apache-airflow[crypto,celery,hive,hdfs,jdbc]==1.10.5
 # install postgres seperately as apache-airflow 1.10.2 forces postgres to use
 # psycopg2 instead of psycopg2-binary which requires additional apt packages
 # to be installed, i.e. postgres-devel. Otherwise the following error is seen:
@@ -16,7 +17,7 @@ postgres>=3.0.0
 
 # TODO(bryan-strassner) Pin to version for airflow when added to the
 # requirements.txt in the airflow images directory
-git+https://opendev.org/airship/promenade.git@master#egg=promenade
+#git+https://opendev.org/airship/promenade.git@master#egg=promenade
 
 # Linting
 flake8<3.8.0,>=3.6.0

src/bin/shipyard_airflow/tests/unit/common/deployment_group/test_deployment_group_manager.py

@@ -15,6 +15,7 @@
 module
 """
 import pytest
+import re
 import yaml
 
 from shipyard_airflow.common.deployment_group.deployment_group import (
@@ -197,13 +198,14 @@ class TestDeploymentGroupManager:
         dgm.mark_group_prepared('compute-nodes-1')
         dgm.mark_group_failed('compute-nodes-2')
         dgm.report_group_summary()
+        ansi_re = re.compile(r'\x1b\[[0-9;]*m')
         assert "=====   Group Summary   =====" in caplog.text
         assert ("Group ntp-node [Critical] ended with stage: "
-                "Stage.NOT_STARTED") in caplog.text
+                "Stage.NOT_STARTED") in re.sub(ansi_re, '', caplog.text)
         caplog.clear()
         dgm.report_node_summary()
-        assert "Nodes Stage.PREPARED: node2" in caplog.text
-        assert "Nodes Stage.FAILED: node3" in caplog.text
+        assert "Nodes Stage.PREPARED: node2" in re.sub(ansi_re, '', caplog.text)
+        assert "Nodes Stage.FAILED: node3" in re.sub(ansi_re, '', caplog.text)
         assert "===== End Node Summary =====" in caplog.text
         assert "It was the best of times" not in caplog.text
 

src/bin/shipyard_airflow/tox.ini

@@ -1,17 +1,21 @@
 [tox]
+skipsdist=True
 envlist = py35, py36, pep8, bandit
 
 [testenv]
 setenv=
   PYTHONWARNING=all
+  LANG=C.UTF-8
+  LC_ALL=C.UTF-8
 deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
 
 [testenv:py35]
 skipsdist=True
 setenv=
   SLUGIFY_USES_TEXT_UNIDECODE=yes
 commands =
+  pip install -r{toxinidir}/test-requirements.txt
+  airflow initdb
   pytest \
     {posargs} \
     --cov-branch \
@@ -25,6 +29,8 @@ skipsdist=True
 setenv=
   SLUGIFY_USES_TEXT_UNIDECODE=yes
 commands =
+  pip install -r{toxinidir}/test-requirements.txt
+  airflow initdb
   pytest \
     {posargs} \
     --cov-branch \

src/bin/shipyard_client/tox.ini

@@ -5,6 +5,8 @@ envlist = py35, py36, pep8, bandit
 [testenv]
 setenv=
   PYTHONWARNING=all
+  LC_ALL=C.UTF-8
+  LC_ALL=C.UTF-8
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt