(fix) Address image build issues, bionic

- With bionic image based shipyard docker images, uwsgi crashes
  with segmentation fault, when it tries to load the psycopg2 library,
  causing the api become unreachable on both shipyard docker images.
  This happens because psycopg2 2.7.x and uwsgi binary wheels are built
  with incompatible ssl libraries. This patch upgrades psycopg2 to the
  latest release to address this issue.

- The existing image build script cannot run in a docker or a pod,
  based pipeline because of two reasons:
  - The build script runs a docker (docker-in-docker) and mounts a
    volume.
    In a dind case, volume bind mounts will not work, because the nested
    container will need the host file system's path for the source path.
  - The shipyard service listens to its exposed service port in the
    nested docker network namespace, which is not reachable from the host
    pod/container.
This patch address both of the above issues. It first creates the
container, copies needed config files to the container and then starts
it. Also it execs into the nested docker to access the shipyard services
in a dind (docker-in-dcoker) case.

Change-Id: Ifdfed539babab01608bfaef37001bb79cd3a080d

images/airflow/requirements.txt

@@ -16,7 +16,7 @@ pytz==2018.5
 pyOpenSSL==18.0.0
 ndg-httpsclient==0.5.1
 pyasn1==0.4.4
-psycopg2==2.7.5
+psycopg2-binary==2.8.4
 docker==3.5.0
 # Airflow is now installed in the Dockerfile directory to allow for
 # overriding where it is sourced from
@@ -27,6 +27,6 @@ marshmallow-sqlalchemy==0.18.0
 tabulate==0.8.03
 
 # Dependencies for other UCP components
-git+https://opendev.org/airship/deckhand.git@7e5d81f50f1f8c0d58d0973ae0b3065fd5b62451#egg=deckhand
+git+https://opendev.org/airship/deckhand.git@e7ba6828a0a1ca27fae596f6e0ee5a857f28001d#egg=deckhand
-git+https://opendev.org/airship/drydock.git@22a4f01cb7880828f7b955c56d53603b6170415a#egg=drydock_provisioner&subdirectory=python
+git+https://opendev.org/airship/drydock.git@586bcf8ebed430b4de82edd9a527566ed39704b7#egg=drydock_provisioner&subdirectory=python
 git+https://opendev.org/airship/armada.git@af8a9ffd0873c2fbc915794e235dbd357f2adab1#egg=armada

src/bin/shipyard_airflow/requirements.txt

@@ -26,7 +26,7 @@ networkx==2.1 # common/deployment_group
 oslo.config==6.4.0
 oslo.policy==1.38.1
 PasteDeploy==1.5.2
-psycopg2==2.7.4
+psycopg2-binary==2.8.4
 python-dateutil==2.7.3
 python-memcached==1.59
 requests==2.20.0
@@ -42,6 +42,6 @@ marshmallow-sqlalchemy==0.18.0
 tabulate==0.8.03
 
 # Dependencies for other UCP components
-git+https://opendev.org/airship/deckhand.git@7e5d81f50f1f8c0d58d0973ae0b3065fd5b62451#egg=deckhand
+git+https://opendev.org/airship/deckhand.git@e7ba6828a0a1ca27fae596f6e0ee5a857f28001d#egg=deckhand
-git+https://opendev.org/airship/drydock.git@22a4f01cb7880828f7b955c56d53603b6170415a#egg=drydock_provisioner&subdirectory=python
+git+https://opendev.org/airship/drydock.git@586bcf8ebed430b4de82edd9a527566ed39704b7#egg=drydock_provisioner&subdirectory=python
 git+https://opendev.org/airship/armada.git@af8a9ffd0873c2fbc915794e235dbd357f2adab1#egg=armada

tools/shipyard_image_run.sh

@@ -18,19 +18,27 @@ set -x
 IMAGE=$1
 USE_PROXY=${USE_PROXY:-false}
 
-# Collect necessary files and run shipyard image in docker
 mkdir -p build/.tmprun/etc
-cp $PWD/etc/shipyard/api-paste.ini build/.tmprun/etc
+docker create \
-cp $PWD/tools/resources/shipyard.conf build/.tmprun/etc
-docker run \
     -v $PWD/build/.tmprun/etc:/etc/shipyard \
     -p 9000:9000 \
-    --name shipyard_test ${IMAGE} \
+    --name shipyard_test ${IMAGE}
-    &
 
+docker cp $PWD/etc/shipyard/api-paste.ini shipyard_test:/etc/shipyard
+docker cp $PWD/tools/resources/shipyard.conf shipyard_test:/etc/shipyard
+docker start shipyard_test &
 sleep 5
 
+# If the image build pipeline is running in a pod/docker (docker-in-docker),
+# we'll need to exec into the nested container's network namespace to acces the
+# shipyard api.
+GOOD="HTTP/1.1 200 OK"
 RESULT="$(curl -i 'http://127.0.0.1:9000/versions' --noproxy '*' | tr '\r' '\n' | head -1)"
+if [[ "${RESULT}" != "${GOOD}" ]]; then
+  if docker exec -t shipyard_test /bin/bash -c "curl -i 'http://127.0.0.1:9000/versions' --noproxy '*' | tr '\r' '\n' | head -1 | grep 'HTTP/1.1 200 OK'"; then
+    RESULT="${GOOD}"
+  fi
+fi
 
 if [ "${USE_PROXY}" == "true" ]; then
   CLI_RESULT="$(docker run -t --rm --net=host --env HTTP_PROXY="${PROXY}" --env HTTPS_PROXY="${PROXY}" ${IMAGE} help | tr '\r' '\n' | head -1)"
@@ -40,8 +48,7 @@ fi
 
 docker stop shipyard_test
 docker rm shipyard_test
-rm -r build/.tmprun
+rm -rf $PWD/build/.tmprun
-GOOD="HTTP/1.1 200 OK"
 CLI_GOOD="THE SHIPYARD COMMAND"
 if [[ ${RESULT} == ${GOOD} && ${CLI_RESULT} == ${CLI_GOOD} ]]; then
     exit 0