From 812cef9335c1cf0f06336d2554cb48db3f69ff06 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Tue, 31 Jul 2018 13:23:51 -0500 Subject: [PATCH] (zuul) Improve image building Improve image building to get closer to support zuul package mirrors Change-Id: I57285b242ddc50e8e902ad2820982a2d4b351bc3 --- Makefile | 16 +++------------- hostdeps.sh | 33 ++++++++++++++++++++++++++++++++- images/drydock/Dockerfile | 22 +++++++++++++++++++--- requirements-host-test.txt | 5 +++++ requirements-host.txt | 8 +++++--- tools/drydock_image_build.sh | 33 +++++++++++++++++++++++++++++++++ 6 files changed, 97 insertions(+), 20 deletions(-) create mode 100644 requirements-host-test.txt create mode 100755 tools/drydock_image_build.sh diff --git a/Makefile b/Makefile index 89fbcabf..de58804c 100644 --- a/Makefile +++ b/Makefile @@ -38,8 +38,8 @@ run_images: run_drydock .PHONY: tests tests: pep8 security docs unit_tests -# Intall external (not managed by tox/pip) dependencies -external_dep: requirements-host.txt +# Install external (not managed by tox/pip) dependencies +external_dep: requirements-host.txt requirements-host-test.txt sudo ./hostdeps.sh touch external_dep @@ -102,17 +102,7 @@ helm-install: .PHONY: build_drydock build_drydock: external_dep -ifeq ($(USE_PROXY), true) - docker build --network host -t $(IMAGE) --label $(LABEL) -f images/drydock/Dockerfile \ - --build-arg http_proxy=$(PROXY) \ - --build-arg https_proxy=$(PROXY) \ - --build-arg HTTP_PROXY=$(PROXY) \ - --build-arg HTTPS_PROXY=$(PROXY) \ - --build-arg no_proxy=$(NO_PROXY) \ - --build-arg NO_PROXY=$(NO_PROXY) . -else - docker build --network host -t $(IMAGE) --label $(LABEL) -f images/drydock/Dockerfile . -endif + export; tools/drydock_image_build.sh ifeq ($(PUSH_IMAGE), true) docker push $(IMAGE) endif diff --git a/hostdeps.sh b/hostdeps.sh index 8561d030..64f52eed 100755 --- a/hostdeps.sh +++ b/hostdeps.sh @@ -1,9 +1,40 @@ #!/bin/bash + # Install host-level package dependencies # needed for local testing +set -x + if [[ ! -z $(uname -a | grep Ubuntu) ]] then - apt install -y --no-install-recommends $(grep -v '^#' requirements-host.txt) + apt update + installed_pkgs=$(apt list --installed | cut -d'/' -f1) + set -a added_pkgs + for reqfile in $(ls requirements-host*.txt) + do + for l in $(grep -vE '(^ *#)|(^$)' "${reqfile}") + do + # Do extra magic to support a list of alternative packages separated by '|' + # none of the packages are found, install the first one listed + IFS='|' read -a pkgalts <<< "${l}" + pkgfound=0 + for a in "${pkgalts[@]}" + do + if grep -qE "^${a}$" <<< "${installed_pkgs}" + then + pkgfound=1 + break + fi + done + if [[ "${pkgfound}" -eq 0 ]] + then + added_pkgs+=("${pkgalts[0]}") + fi + done + done + if [[ ${#added_pkgs[@]} -gt 0 ]] + then + apt install -y --no-install-recommends "${added_pkgs[@]}" + fi else echo "Only support testing on Ubuntu hosts at this time." fi diff --git a/images/drydock/Dockerfile b/images/drydock/Dockerfile index 5abf3a70..28cb214f 100644 --- a/images/drydock/Dockerfile +++ b/images/drydock/Dockerfile @@ -11,22 +11,38 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM python:3.5 +FROM ubuntu:16.04 +ARG UBUNTU_REPO=http://archive.ubuntu.com/ubuntu +ARG TRUSTED_UBUNTU_REPO=no +ARG ALLOW_UNAUTHENTICATED=false +ARG PIP_TRUSTED_HOST=foo.com +ARG PIP_INDEX_URL=https://pypi.org/simple ENV container docker ENV PORT 9000 ENV LC_ALL C.UTF-8 ENV LANG C.UTF-8 # Copy direct dependency requirements only to build a dependency layer -RUN DEBIAN_FRONTEND=noninteractive apt update && \ - apt install -y libvirt-dev --no-install-recommends +RUN echo "deb ${UBUNTU_REPO} xenial main restricted universe multiverse" > /etc/apt/sources.list; \ + echo "deb ${UBUNTU_REPO} xenial-security main restricted universe multiverse" >> /etc/apt/sources.list; \ + echo "deb ${UBUNTU_REPO} xenial-updates main restricted universe multiverse" >> /etc/apt/sources.list; \ + cat /etc/apt/sources.list; \ + echo "APT::Get::AllowUnauthenticated ${ALLOW_UNAUTHENTICATED};" >> /etc/apt/apt.conf.d/00-local-mirrors; + + +COPY ./requirements-host.txt /tmp/drydock/ +COPY ./hostdeps.sh /tmp/drydock +WORKDIR /tmp/drydock +RUN ./hostdeps.sh; \ + rm -r /var/lib/apt/lists/* COPY ./requirements-lock.txt /tmp/drydock/ RUN pip3 install \ --no-cache-dir \ -r /tmp/drydock/requirements-lock.txt + COPY . /tmp/drydock WORKDIR /tmp/drydock diff --git a/requirements-host-test.txt b/requirements-host-test.txt new file mode 100644 index 00000000..9cdb2ba6 --- /dev/null +++ b/requirements-host-test.txt @@ -0,0 +1,5 @@ +# These are host packages needed for Drydock +# that don't come on a minimal Ubuntu install +python-tox +docker.io|docker-ce +plantuml diff --git a/requirements-host.txt b/requirements-host.txt index 6b7b68ad..23b8a681 100644 --- a/requirements-host.txt +++ b/requirements-host.txt @@ -3,7 +3,9 @@ libvirt-dev pkg-config python3-dev -python-tox -docker.io gcc -plantuml +netbase +python3-pip +python3-setuptools +ssh +curl diff --git a/tools/drydock_image_build.sh b/tools/drydock_image_build.sh new file mode 100755 index 00000000..067db8bc --- /dev/null +++ b/tools/drydock_image_build.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -x + +UBUNTU_REPO=${UBUNTU_REPO:-""} +TRUSTED_UBUNTU_REPO=${TRUSTED_UBUNTU_REPO:-"no"} +ALLOW_UNATHENTICATED=${ALLOW_UNAUTHENTICATED:-"false"} +PIP_INDEX_URL=${PIP_INDEX_URL:-""} +PIP_TRUSTED_HOST=${PIP_TRUSTED_HOST:-""} + +ADDL_BUILD_ARGS="" + +if [[ ! -z "${UBUNTU_REPO}" ]] +then + ADDL_BUILD_ARGS="${ADDL_BUILD_ARGS} --build-arg UBUNTU_REPO=${UBUNTU_REPO}" + ADDL_BUILD_ARGS="${ADDL_BUILD_ARGS} --build-arg TRUSTED_UBUNTU_REPO=${TRUSTED_UBUNTU_REPO}" + ADDL_BUILD_ARGS="${ADDL_BUILD_ARGS} --build-arg ALLOW_UNAUTHENTICATED=${ALLOW_UNAUTHENTICATED}" +fi + +if [[ ! -z "${PIP_INDEX_URL}" ]] +then + ADDL_BUILD_ARGS="${ADDL_BUILD_ARGS}| --build-arg PIP_INDEX_URL=${PIP_INDEX_URL}" + ADDL_BUILD_ARGS="${ADDL_BUILD_ARGS}| --build-arg PIP_TRUSTED_HOST=${PIP_TRUSTED_HOST}" +fi + +docker build --network host -t ${IMAGE} --label ${LABEL} -f images/drydock/Dockerfile \ + ${ADDL_BUILD_ARGS} \ + --build-arg http_proxy=${http_proxy} \ + --build-arg https_proxy=${https_proxy} \ + --build-arg HTTP_PROXY=${HTTP_PROXY} \ + --build-arg HTTPS_PROXY=${HTTPS_PROXY} \ + --build-arg no_proxy=${no_proxy} \ + --build-arg NO_PROXY=${NO_PROXY} . +