(zuul) Improve image building

Improve image building to get closer to
support zuul package mirrors

Change-Id: I57285b242ddc50e8e902ad2820982a2d4b351bc3

Makefile

@@ -38,8 +38,8 @@ run_images: run_drydock
 .PHONY: tests
 tests: pep8 security docs unit_tests
 
-# Intall external (not managed by tox/pip) dependencies
-external_dep: requirements-host.txt
+# Install external (not managed by tox/pip) dependencies
+external_dep: requirements-host.txt requirements-host-test.txt
 	sudo ./hostdeps.sh
 	touch external_dep
 
@@ -102,17 +102,7 @@ helm-install:
 
 .PHONY: build_drydock
 build_drydock: external_dep
-ifeq ($(USE_PROXY), true)
-	docker build --network host -t $(IMAGE) --label $(LABEL) -f images/drydock/Dockerfile \
-		--build-arg http_proxy=$(PROXY) \
-		--build-arg https_proxy=$(PROXY) \
-		--build-arg HTTP_PROXY=$(PROXY) \
-		--build-arg HTTPS_PROXY=$(PROXY) \
-		--build-arg no_proxy=$(NO_PROXY) \
-		--build-arg NO_PROXY=$(NO_PROXY) .
-else
-	docker build --network host -t $(IMAGE) --label $(LABEL) -f images/drydock/Dockerfile .
-endif
+	export; tools/drydock_image_build.sh
 ifeq ($(PUSH_IMAGE), true)
 	docker push $(IMAGE)
 endif

hostdeps.sh

@@ -1,9 +1,40 @@
 #!/bin/bash
+
 # Install host-level package dependencies
 # needed for local testing
+set -x
+
 if [[ ! -z $(uname -a | grep Ubuntu) ]]
 then
-  apt install -y --no-install-recommends $(grep -v '^#' requirements-host.txt)
+  apt update
+  installed_pkgs=$(apt list --installed | cut -d'/' -f1)
+  set -a added_pkgs
+  for reqfile in $(ls requirements-host*.txt)
+  do
+    for l in $(grep -vE '(^ *#)|(^$)' "${reqfile}")
+    do
+      # Do extra magic to support a list of alternative packages separated by '|'
+      # none of the packages are found, install the first one listed
+      IFS='|' read -a pkgalts <<< "${l}"
+      pkgfound=0
+      for a in "${pkgalts[@]}"
+      do
+        if grep -qE "^${a}$" <<< "${installed_pkgs}"
+        then
+          pkgfound=1
+          break
+        fi
+      done
+      if [[ "${pkgfound}" -eq 0 ]]
+      then
+        added_pkgs+=("${pkgalts[0]}")
+      fi
+    done
+  done
+  if [[ ${#added_pkgs[@]} -gt 0 ]]
+  then
+    apt install -y --no-install-recommends "${added_pkgs[@]}"
+  fi
 else
   echo "Only support testing on Ubuntu hosts at this time."
 fi

images/drydock/Dockerfile

@@ -11,22 +11,38 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-FROM python:3.5
+FROM ubuntu:16.04
 
+ARG UBUNTU_REPO=http://archive.ubuntu.com/ubuntu
+ARG TRUSTED_UBUNTU_REPO=no
+ARG ALLOW_UNAUTHENTICATED=false
+ARG PIP_TRUSTED_HOST=foo.com
+ARG PIP_INDEX_URL=https://pypi.org/simple
 ENV container docker
 ENV PORT 9000
 ENV LC_ALL C.UTF-8
 ENV LANG C.UTF-8
 
 # Copy direct dependency requirements only to build a dependency layer
-RUN DEBIAN_FRONTEND=noninteractive apt update && \
-    apt install -y libvirt-dev --no-install-recommends
+RUN echo  "deb  ${UBUNTU_REPO} xenial main restricted universe multiverse" > /etc/apt/sources.list; \
+    echo "deb  ${UBUNTU_REPO} xenial-security main restricted universe multiverse" >> /etc/apt/sources.list; \
+    echo "deb  ${UBUNTU_REPO} xenial-updates main restricted universe multiverse" >> /etc/apt/sources.list; \
+    cat /etc/apt/sources.list; \
+    echo "APT::Get::AllowUnauthenticated ${ALLOW_UNAUTHENTICATED};" >> /etc/apt/apt.conf.d/00-local-mirrors;
+
+
+COPY ./requirements-host.txt /tmp/drydock/
+COPY ./hostdeps.sh /tmp/drydock
+WORKDIR /tmp/drydock
+RUN ./hostdeps.sh; \
+    rm -r /var/lib/apt/lists/*
 
 COPY ./requirements-lock.txt /tmp/drydock/
 RUN pip3 install \
     --no-cache-dir \
     -r /tmp/drydock/requirements-lock.txt
 
+
 COPY . /tmp/drydock
 
 WORKDIR /tmp/drydock

requirements-host-test.txt

@@ -0,0 +1,5 @@
+# These are host packages needed for Drydock
+# that don't come on a minimal Ubuntu install
+python-tox
+docker.io|docker-ce
+plantuml

requirements-host.txt

@@ -3,7 +3,9 @@
 libvirt-dev
 pkg-config
 python3-dev
-python-tox
-docker.io
 gcc
-plantuml
+netbase
+python3-pip
+python3-setuptools
+ssh
+curl

tools/drydock_image_build.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -x
+
+UBUNTU_REPO=${UBUNTU_REPO:-""}
+TRUSTED_UBUNTU_REPO=${TRUSTED_UBUNTU_REPO:-"no"}
+ALLOW_UNATHENTICATED=${ALLOW_UNAUTHENTICATED:-"false"}
+PIP_INDEX_URL=${PIP_INDEX_URL:-""}
+PIP_TRUSTED_HOST=${PIP_TRUSTED_HOST:-""}
+
+ADDL_BUILD_ARGS=""
+
+if [[ ! -z "${UBUNTU_REPO}" ]]
+then
+  ADDL_BUILD_ARGS="${ADDL_BUILD_ARGS} --build-arg UBUNTU_REPO=${UBUNTU_REPO}"
+  ADDL_BUILD_ARGS="${ADDL_BUILD_ARGS} --build-arg TRUSTED_UBUNTU_REPO=${TRUSTED_UBUNTU_REPO}"
+  ADDL_BUILD_ARGS="${ADDL_BUILD_ARGS} --build-arg ALLOW_UNAUTHENTICATED=${ALLOW_UNAUTHENTICATED}"
+fi
+
+if [[ ! -z "${PIP_INDEX_URL}" ]]
+then
+  ADDL_BUILD_ARGS="${ADDL_BUILD_ARGS}| --build-arg PIP_INDEX_URL=${PIP_INDEX_URL}"
+  ADDL_BUILD_ARGS="${ADDL_BUILD_ARGS}| --build-arg PIP_TRUSTED_HOST=${PIP_TRUSTED_HOST}"
+fi
+
+docker build --network host -t ${IMAGE} --label ${LABEL} -f images/drydock/Dockerfile \
+  ${ADDL_BUILD_ARGS} \
+  --build-arg http_proxy=${http_proxy} \
+  --build-arg https_proxy=${https_proxy} \
+  --build-arg HTTP_PROXY=${HTTP_PROXY} \
+  --build-arg HTTPS_PROXY=${HTTPS_PROXY} \
+  --build-arg no_proxy=${no_proxy} \
+  --build-arg NO_PROXY=${NO_PROXY} .
+