From e9696dca0a2f608627ddd6b369f30dd191b3e487 Mon Sep 17 00:00:00 2001 From: Dustin Specker Date: Thu, 18 Feb 2021 08:27:01 -0600 Subject: [PATCH] feat: support setting up Vagrant behind corporate proxy with TLS This is a squashed commit, keeping previous messages intact for history. - chore(tools/gate/jarvis): remove unused http_proxy - fix(tools/gate/deploy-k8s): pre-pull Calico images By pre-pulling Calico images, we can better ensure the timeout for `kubectl wait` for `k8s-app=kube-dns` is sufficient, since most of the time spent is on pulling images. - fix(tools/gate/jarvis): skip loki Helm test when proxy is set The Loki test attempts to install `curl` and `jq`, which will fail when a proxy is required since the pod doesn't setup proxy environment variables. - feat(tools/deployment/vagrant): support providing a cert for proxy - feat(ubuntu-base/standard-container): support internal-certs The Vagrant file mounts an additional synced folder to /airship_charts/tools/gate/jarvis/ubuntu-base/internal-certs. This internal-certs dir has been added to this Git repository using a placeholder `.gitkeep` file to keep the directory non-empty. This directory has also been added to .gitignore to prevent any changes such as the mounted internal certs from being committed. The ubuntu-base image sets the proxy env vars as well as contains the internal certs. The standard container is then based on the ubuntu-base image. The ubuntu-base image is published as library/ubuntu:focal in harbor. - fix(tools/gate/jarvis): support Harbor behind proxy with cert Change-Id: I602dfa3b04b798a1a2096242ffb6dfe7f2ba92e4 --- .gitignore | 3 +++ doc/source/install/jarvis/proxy.rst | 12 +++++++++ tools/deployment/vagrant/Vagrantfile | 9 +++++++ tools/gate/deploy-k8s.sh | 5 ++++ tools/gate/jarvis/010-pre-setup.sh | 5 +++- tools/gate/jarvis/300-deploy-loki.sh | 8 +++++- tools/gate/jarvis/400-deploy-harbor.sh | 22 +++++++++++++--- tools/gate/jarvis/ubuntu-base/Dockerfile | 25 +++++++++++++++++++ .../ubuntu-base/internal-certs/.gitkeep | 0 9 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 tools/gate/jarvis/ubuntu-base/Dockerfile create mode 100644 tools/gate/jarvis/ubuntu-base/internal-certs/.gitkeep diff --git a/.gitignore b/.gitignore index 42969320..6759312b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ tools/deployment/vagrant/.vagrant # Helm dependencies lock file Chart.lock + +# ignore any added internal certs for ubuntu-build +tools/gate/jarvis/ubuntu-base/internal-certs/ diff --git a/doc/source/install/jarvis/proxy.rst b/doc/source/install/jarvis/proxy.rst index d46294a2..8ecf3f5d 100644 --- a/doc/source/install/jarvis/proxy.rst +++ b/doc/source/install/jarvis/proxy.rst @@ -10,6 +10,18 @@ On the host machine, ensure the following environment variables are set with the to an IP address of a corporate name server that will resolve internal URLs. ``PRIVATE_NS`` can support multiple corporate name servers by creating a space separated list, such as ``PRIVATE_NS="172.18.0.17 172.19.0.17"``. +Certificate Authority for Corporate Proxy +========================================= + +If the proxy requires a certificate to trust then: + +#. Run ``mkdir ~/internal-certs/`` +#. Download the required ``*.crt`` file +#. Move the ``*.crt`` file into ``~/internal-certs/`` +#. Define an environment variable named ``INTERNAL_CERTS_DIR`` with the value of ``~/internal-certs/`` + +The ``Vagrantfile`` will handle using the internal certificates during the ``vagrant up`` process. + Vagrant Plugin ============== diff --git a/tools/deployment/vagrant/Vagrantfile b/tools/deployment/vagrant/Vagrantfile index 59248165..c5e4ad6d 100644 --- a/tools/deployment/vagrant/Vagrantfile +++ b/tools/deployment/vagrant/Vagrantfile @@ -16,6 +16,15 @@ Vagrant.configure("2") do |config| config.vm.synced_folder "../../../", "/airship_charts" + if ENV["INTERNAL_CERTS_DIR"] + # for guest OS to trust proxy itself + config.vm.synced_folder ENV["INTERNAL_CERTS_DIR"], "/usr/local/share/ca-certificates/internal-certs/" + # for containerd/docker to trust proxy when pulling images within kubernetes cluster + config.vm.synced_folder ENV["INTERNAL_CERTS_DIR"], "/etc/containerd/cert.d/" + # for use by ubuntu-base to trust proxy + config.vm.synced_folder ENV["INTERNAL_CERTS_DIR"], "/airship_charts/tools/gate/jarvis/ubuntu-base/internal-certs/" + end + config.vm.network "private_network", ip: "192.168.56.10" config.vm.provider "libvirt" do |libvirt| diff --git a/tools/gate/deploy-k8s.sh b/tools/gate/deploy-k8s.sh index 67c3943c..ffaed25e 100755 --- a/tools/gate/deploy-k8s.sh +++ b/tools/gate/deploy-k8s.sh @@ -203,6 +203,11 @@ curl https://docs.projectcalico.org/"${CALICO_VERSION}"/manifests/calico.yaml -o # in image pull error due to dockerhub's rate limiting policy. To avoid potential conflict, # use calico's quay.io repository to mitigate this issue. sed -i -e 's#docker.io/calico/#quay.io/calico/#g' /tmp/calico.yaml + +# Download images needed for calico before applying manifests, so that `kubectl wait` timeout +# for `k8s-app=kube-dns` isn't reached by slow download speeds +awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} sudo docker pull {} + kubectl apply -f /tmp/calico.yaml # Note: Patch calico daemonset to enable Prometheus metrics and annotations diff --git a/tools/gate/jarvis/010-pre-setup.sh b/tools/gate/jarvis/010-pre-setup.sh index 14cf3034..b2a6cd11 100755 --- a/tools/gate/jarvis/010-pre-setup.sh +++ b/tools/gate/jarvis/010-pre-setup.sh @@ -7,7 +7,6 @@ set -ex # Add the necessary corporate nameserver to systemd-resolved so it # propagates properly and prevent it from overwriting. # Replace 123.123.123.4 with the correct IP -: "${HTTP_PROXY:=""}" : "${PRIVATE_NS:=""}" if [ -n "${PRIVATE_NS}" ]; then sudo -E sed -i -e 's/^DNS=/#DNS=/' /etc/systemd/resolved.conf @@ -16,3 +15,7 @@ if [ -n "${PRIVATE_NS}" ]; then sudo ln -s /run/systemd/resolve/resolv.conf /etc/resolv.conf sudo systemctl restart systemd-resolved fi + +# Update CA certificates in case internal certs have been mounted to /usr/share/local/ca-certificates +# via Vagrantfile. Need to run update-ca-certificates before any `curl`, etc. commands are done. +sudo update-ca-certificates diff --git a/tools/gate/jarvis/300-deploy-loki.sh b/tools/gate/jarvis/300-deploy-loki.sh index f6e28457..16fb7c48 100755 --- a/tools/gate/jarvis/300-deploy-loki.sh +++ b/tools/gate/jarvis/300-deploy-loki.sh @@ -14,4 +14,10 @@ helm upgrade \ ./tools/deployment/common/wait-for-pods.sh loki -helm -n loki test loki --logs +# TODO(dustinspecker): remove this if condition and run loki test behind proxy +# loki pod's container downloads jq and curl, which won't work +# since the proxies are not configured for the pod, so skip test loki test for now +# when proxy is defined +if [ -z "$http_proxy" ]; then + helm -n loki test loki --logs +fi diff --git a/tools/gate/jarvis/400-deploy-harbor.sh b/tools/gate/jarvis/400-deploy-harbor.sh index 8130ad26..2c41c6f5 100755 --- a/tools/gate/jarvis/400-deploy-harbor.sh +++ b/tools/gate/jarvis/400-deploy-harbor.sh @@ -8,12 +8,27 @@ helm upgrade \ --create-namespace \ --install \ --namespace=harbor \ + --set harbor.proxy.httpProxy="$http_proxy" \ + --set harbor.proxy.httpsProxy="$https_proxy" \ harbor \ ./charts/harbor \ $(./tools/deployment/common/get-values-overrides.sh harbor) ./tools/deployment/common/wait-for-pods.sh harbor +# Setup internal certs so that Trivy can download its database. +# Harbor's Trivy statefulset image is based on PhotonOS, which doesn't have +# an update-ca-certificates package. +# PhotonOS has support to generate the ca-bundle, but it's not installed in this +# image. The statefulset also prevents privilege escalation, so we can't intervene and +# install that. +# So, hackily append .crt files to the ca-bundle.crt. +for cert in ./tools/gate/jarvis/ubuntu-base/internal-certs/*.crt; do + kubectl exec harbor-harbor-trivy-0 \ + --namespace harbor \ + -- /bin/bash -c "echo \"$(cat $cert)\" >> /etc/pki/tls/certs/ca-bundle.crt" +done + helm -n harbor test harbor --logs function validate() { @@ -74,9 +89,10 @@ EOF sudo -E docker pull harbor-core.jarvis.local/library/busybox:latest sudo -E docker trust inspect --pretty harbor-core.jarvis.local/library/busybox:latest - #Required for pipelines - sudo docker pull docker.io/library/ubuntu:focal - sudo docker tag docker.io/library/ubuntu:focal harbor-core.jarvis.local/library/ubuntu:focal + #Required for pipelines and standard-container + pushd ./tools/gate/jarvis/ubuntu-base/ + sudo docker build --build-arg http_proxy="$http_proxy" --build-arg https_proxy="$https_proxy" --build-arg no_proxy="$no_proxy" --build-arg HTTP_PROXY="$http_proxy" --build-arg HTTPS_PROXY="$https_proxy" --build-arg NO_PROXY="$no_proxy" -t harbor-core.jarvis.local/library/ubuntu:focal . + popd sudo -E notary init -p harbor-core.jarvis.local/library/ubuntu:focal sudo -E docker push harbor-core.jarvis.local/library/ubuntu:focal } diff --git a/tools/gate/jarvis/ubuntu-base/Dockerfile b/tools/gate/jarvis/ubuntu-base/Dockerfile new file mode 100644 index 00000000..5df96165 --- /dev/null +++ b/tools/gate/jarvis/ubuntu-base/Dockerfile @@ -0,0 +1,25 @@ +ARG BASE_IMAGE=ubuntu:focal +FROM ${BASE_IMAGE} + +ARG http_proxy +ARG https_proxy +ARG no_proxy +ARG HTTP_PROXY +ARG HTTPS_PROXY +ARG NO_PROXY + +COPY ./internal-certs /usr/local/share/ca-certificates + +RUN apt-get update \ + && apt-get install \ + --no-install-recommends \ + --yes \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +ENV http_proxy $http_proxy +ENV https_proxy $https_proxy +ENV no_proxy $no_proxy +ENV HTTP_PROXY $HTTP_PROXY +ENV HTTPS_PROXY $HTTPS_PROXY +ENV NO_PROXY $NO_PROXY diff --git a/tools/gate/jarvis/ubuntu-base/internal-certs/.gitkeep b/tools/gate/jarvis/ubuntu-base/internal-certs/.gitkeep new file mode 100644 index 00000000..e69de29b