feat: support setting up Vagrant behind corporate proxy with TLS

This is a squashed commit, keeping previous messages intact for history.

- chore(tools/gate/jarvis): remove unused http_proxy

- fix(tools/gate/deploy-k8s): pre-pull Calico images

By pre-pulling Calico images, we can better ensure the timeout for
`kubectl wait` for `k8s-app=kube-dns` is sufficient, since most of the
time spent is on pulling images.

- fix(tools/gate/jarvis): skip loki Helm test when proxy is set

The Loki test attempts to install `curl` and `jq`, which will fail when
a proxy is required since the pod doesn't setup proxy environment
variables.

- feat(tools/deployment/vagrant): support providing a cert for proxy

- feat(ubuntu-base/standard-container): support internal-certs

The Vagrant file mounts an additional synced folder to
/airship_charts/tools/gate/jarvis/ubuntu-base/internal-certs.

This internal-certs dir has been added to this Git repository using a
placeholder `.gitkeep` file to keep the directory non-empty. This
directory has also been added to .gitignore to prevent any changes such
as the mounted internal certs from being committed.

The ubuntu-base image sets the proxy env vars as well as contains the
internal certs. The standard container is then based on the ubuntu-base
image.

The ubuntu-base image is published as library/ubuntu:focal in harbor.

- fix(tools/gate/jarvis): support Harbor behind proxy with cert

Change-Id: I602dfa3b04b798a1a2096242ffb6dfe7f2ba92e4

.gitignore

@@ -13,3 +13,6 @@ tools/deployment/vagrant/.vagrant
 
 # Helm dependencies lock file
 Chart.lock
+
+# ignore any added internal certs for ubuntu-build
+tools/gate/jarvis/ubuntu-base/internal-certs/

doc/source/install/jarvis/proxy.rst

@@ -10,6 +10,18 @@ On the host machine, ensure the following environment variables are set with the
 to an IP address of a corporate name server that will resolve internal URLs. ``PRIVATE_NS`` can support multiple
 corporate name servers by creating a space separated list, such as ``PRIVATE_NS="172.18.0.17 172.19.0.17"``.
 
+Certificate Authority for Corporate Proxy
+=========================================
+
+If the proxy requires a certificate to trust then:
+
+#. Run ``mkdir ~/internal-certs/``
+#. Download the required ``*.crt`` file
+#. Move the ``*.crt`` file into ``~/internal-certs/``
+#. Define an environment variable named ``INTERNAL_CERTS_DIR`` with the value of ``~/internal-certs/``
+
+The ``Vagrantfile`` will handle using the internal certificates during the ``vagrant up`` process.
+
 Vagrant Plugin
 ==============
 

tools/deployment/vagrant/Vagrantfile

@@ -16,6 +16,15 @@ Vagrant.configure("2") do |config|
 
   config.vm.synced_folder "../../../", "/airship_charts"
 
+  if ENV["INTERNAL_CERTS_DIR"]
+    # for guest OS to trust proxy itself
+    config.vm.synced_folder ENV["INTERNAL_CERTS_DIR"], "/usr/local/share/ca-certificates/internal-certs/"
+    # for containerd/docker to trust proxy when pulling images within kubernetes cluster
+    config.vm.synced_folder ENV["INTERNAL_CERTS_DIR"], "/etc/containerd/cert.d/"
+    # for use by ubuntu-base to trust proxy
+    config.vm.synced_folder ENV["INTERNAL_CERTS_DIR"], "/airship_charts/tools/gate/jarvis/ubuntu-base/internal-certs/"
+  end
+
   config.vm.network "private_network", ip: "192.168.56.10"
 
   config.vm.provider "libvirt" do |libvirt|

tools/gate/deploy-k8s.sh

@@ -203,6 +203,11 @@ curl https://docs.projectcalico.org/"${CALICO_VERSION}"/manifests/calico.yaml -o
 # in image pull error due to dockerhub's rate limiting policy. To avoid potential conflict,
 # use calico's quay.io repository to mitigate this issue.
 sed -i -e 's#docker.io/calico/#quay.io/calico/#g' /tmp/calico.yaml
+
+# Download images needed for calico before applying manifests, so that `kubectl wait` timeout
+# for `k8s-app=kube-dns` isn't reached by slow download speeds
+awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} sudo docker pull {}
+
 kubectl apply -f /tmp/calico.yaml
 
 # Note: Patch calico daemonset to enable Prometheus metrics and annotations

tools/gate/jarvis/010-pre-setup.sh

@@ -7,7 +7,6 @@ set -ex
 # Add the necessary corporate nameserver to systemd-resolved so it
 # propagates properly and prevent it from overwriting.
 # Replace 123.123.123.4 with the correct IP
-: "${HTTP_PROXY:=""}"
 : "${PRIVATE_NS:=""}"
 if [ -n "${PRIVATE_NS}" ]; then
   sudo -E sed -i -e 's/^DNS=/#DNS=/' /etc/systemd/resolved.conf
@@ -16,3 +15,7 @@ if [ -n "${PRIVATE_NS}" ]; then
   sudo ln -s /run/systemd/resolve/resolv.conf /etc/resolv.conf
   sudo systemctl restart systemd-resolved
 fi
+
+# Update CA certificates in case internal certs have been mounted to /usr/share/local/ca-certificates
+# via Vagrantfile. Need to run update-ca-certificates before any `curl`, etc. commands are done.
+sudo update-ca-certificates

tools/gate/jarvis/300-deploy-loki.sh

@@ -14,4 +14,10 @@ helm upgrade \
 
 ./tools/deployment/common/wait-for-pods.sh loki
 
+# TODO(dustinspecker): remove this if condition and run loki test behind proxy
+# loki pod's container downloads jq and curl, which won't work
+# since the proxies are not configured for the pod, so skip test loki test for now
+# when proxy is defined
+if [ -z "$http_proxy" ]; then
      helm -n loki test loki --logs
+fi

tools/gate/jarvis/400-deploy-harbor.sh

@@ -8,12 +8,27 @@ helm upgrade \
     --create-namespace \
     --install \
     --namespace=harbor \
+    --set harbor.proxy.httpProxy="$http_proxy" \
+    --set harbor.proxy.httpsProxy="$https_proxy" \
     harbor \
     ./charts/harbor \
     $(./tools/deployment/common/get-values-overrides.sh harbor)
 
 ./tools/deployment/common/wait-for-pods.sh harbor
 
+# Setup internal certs so that Trivy can download its database.
+# Harbor's Trivy statefulset image is based on PhotonOS, which doesn't have
+# an update-ca-certificates package.
+# PhotonOS has support to generate the ca-bundle, but it's not installed in this
+# image. The statefulset also prevents privilege escalation, so we can't intervene and
+# install that.
+# So, hackily append .crt files to the ca-bundle.crt.
+for cert in ./tools/gate/jarvis/ubuntu-base/internal-certs/*.crt; do
+    kubectl exec harbor-harbor-trivy-0 \
+        --namespace harbor \
+        -- /bin/bash -c "echo \"$(cat $cert)\" >> /etc/pki/tls/certs/ca-bundle.crt"
+done
+
 helm -n harbor test harbor --logs
 
 function validate() {
@@ -74,9 +89,10 @@ EOF
     sudo -E docker pull harbor-core.jarvis.local/library/busybox:latest
     sudo -E docker trust inspect --pretty harbor-core.jarvis.local/library/busybox:latest
 
-    #Required for pipelines
+    #Required for pipelines and standard-container
-    sudo docker pull docker.io/library/ubuntu:focal
+    pushd ./tools/gate/jarvis/ubuntu-base/
-    sudo docker tag docker.io/library/ubuntu:focal harbor-core.jarvis.local/library/ubuntu:focal
+    sudo docker build --build-arg http_proxy="$http_proxy" --build-arg https_proxy="$https_proxy" --build-arg no_proxy="$no_proxy" --build-arg HTTP_PROXY="$http_proxy" --build-arg HTTPS_PROXY="$https_proxy" --build-arg NO_PROXY="$no_proxy" -t harbor-core.jarvis.local/library/ubuntu:focal .
+    popd
     sudo -E notary init -p harbor-core.jarvis.local/library/ubuntu:focal
     sudo -E docker push harbor-core.jarvis.local/library/ubuntu:focal
 }

tools/gate/jarvis/ubuntu-base/Dockerfile

@@ -0,0 +1,25 @@
+ARG BASE_IMAGE=ubuntu:focal
+FROM ${BASE_IMAGE}
+
+ARG http_proxy
+ARG https_proxy
+ARG no_proxy
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ARG NO_PROXY
+
+COPY ./internal-certs /usr/local/share/ca-certificates
+
+RUN apt-get update \
+ && apt-get install \
+      --no-install-recommends \
+      --yes \
+      ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV http_proxy $http_proxy
+ENV https_proxy $https_proxy
+ENV no_proxy $no_proxy
+ENV HTTP_PROXY $HTTP_PROXY
+ENV HTTPS_PROXY $HTTPS_PROXY
+ENV NO_PROXY $NO_PROXY