feat: support setting up Vagrant behind corporate proxy with TLS

This is a squashed commit, keeping previous messages intact for history.

- chore(tools/gate/jarvis): remove unused http_proxy

- fix(tools/gate/deploy-k8s): pre-pull Calico images

By pre-pulling Calico images, we can better ensure the timeout for
`kubectl wait` for `k8s-app=kube-dns` is sufficient, since most of the
time spent is on pulling images.

- fix(tools/gate/jarvis): skip loki Helm test when proxy is set

The Loki test attempts to install `curl` and `jq`, which will fail when
a proxy is required since the pod doesn't setup proxy environment
variables.

- feat(tools/deployment/vagrant): support providing a cert for proxy

- feat(ubuntu-base/standard-container): support internal-certs

The Vagrant file mounts an additional synced folder to
/airship_charts/tools/gate/jarvis/ubuntu-base/internal-certs.

This internal-certs dir has been added to this Git repository using a
placeholder `.gitkeep` file to keep the directory non-empty. This
directory has also been added to .gitignore to prevent any changes such
as the mounted internal certs from being committed.

The ubuntu-base image sets the proxy env vars as well as contains the
internal certs. The standard container is then based on the ubuntu-base
image.

The ubuntu-base image is published as library/ubuntu:focal in harbor.

- fix(tools/gate/jarvis): support Harbor behind proxy with cert

Change-Id: I602dfa3b04b798a1a2096242ffb6dfe7f2ba92e4
This commit is contained in:
Dustin Specker 2021-02-18 08:27:01 -06:00
parent 877149a753
commit e9696dca0a
9 changed files with 84 additions and 5 deletions

3
.gitignore vendored
View File

@ -13,3 +13,6 @@ tools/deployment/vagrant/.vagrant
# Helm dependencies lock file # Helm dependencies lock file
Chart.lock Chart.lock
# ignore any added internal certs for ubuntu-build
tools/gate/jarvis/ubuntu-base/internal-certs/

View File

@ -10,6 +10,18 @@ On the host machine, ensure the following environment variables are set with the
to an IP address of a corporate name server that will resolve internal URLs. ``PRIVATE_NS`` can support multiple to an IP address of a corporate name server that will resolve internal URLs. ``PRIVATE_NS`` can support multiple
corporate name servers by creating a space separated list, such as ``PRIVATE_NS="172.18.0.17 172.19.0.17"``. corporate name servers by creating a space separated list, such as ``PRIVATE_NS="172.18.0.17 172.19.0.17"``.
Certificate Authority for Corporate Proxy
=========================================
If the proxy requires a certificate to trust then:
#. Run ``mkdir ~/internal-certs/``
#. Download the required ``*.crt`` file
#. Move the ``*.crt`` file into ``~/internal-certs/``
#. Define an environment variable named ``INTERNAL_CERTS_DIR`` with the value of ``~/internal-certs/``
The ``Vagrantfile`` will handle using the internal certificates during the ``vagrant up`` process.
Vagrant Plugin Vagrant Plugin
============== ==============

View File

@ -16,6 +16,15 @@ Vagrant.configure("2") do |config|
config.vm.synced_folder "../../../", "/airship_charts" config.vm.synced_folder "../../../", "/airship_charts"
if ENV["INTERNAL_CERTS_DIR"]
# for guest OS to trust proxy itself
config.vm.synced_folder ENV["INTERNAL_CERTS_DIR"], "/usr/local/share/ca-certificates/internal-certs/"
# for containerd/docker to trust proxy when pulling images within kubernetes cluster
config.vm.synced_folder ENV["INTERNAL_CERTS_DIR"], "/etc/containerd/cert.d/"
# for use by ubuntu-base to trust proxy
config.vm.synced_folder ENV["INTERNAL_CERTS_DIR"], "/airship_charts/tools/gate/jarvis/ubuntu-base/internal-certs/"
end
config.vm.network "private_network", ip: "192.168.56.10" config.vm.network "private_network", ip: "192.168.56.10"
config.vm.provider "libvirt" do |libvirt| config.vm.provider "libvirt" do |libvirt|

View File

@ -203,6 +203,11 @@ curl https://docs.projectcalico.org/"${CALICO_VERSION}"/manifests/calico.yaml -o
# in image pull error due to dockerhub's rate limiting policy. To avoid potential conflict, # in image pull error due to dockerhub's rate limiting policy. To avoid potential conflict,
# use calico's quay.io repository to mitigate this issue. # use calico's quay.io repository to mitigate this issue.
sed -i -e 's#docker.io/calico/#quay.io/calico/#g' /tmp/calico.yaml sed -i -e 's#docker.io/calico/#quay.io/calico/#g' /tmp/calico.yaml
# Download images needed for calico before applying manifests, so that `kubectl wait` timeout
# for `k8s-app=kube-dns` isn't reached by slow download speeds
awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} sudo docker pull {}
kubectl apply -f /tmp/calico.yaml kubectl apply -f /tmp/calico.yaml
# Note: Patch calico daemonset to enable Prometheus metrics and annotations # Note: Patch calico daemonset to enable Prometheus metrics and annotations

View File

@ -7,7 +7,6 @@ set -ex
# Add the necessary corporate nameserver to systemd-resolved so it # Add the necessary corporate nameserver to systemd-resolved so it
# propagates properly and prevent it from overwriting. # propagates properly and prevent it from overwriting.
# Replace 123.123.123.4 with the correct IP # Replace 123.123.123.4 with the correct IP
: "${HTTP_PROXY:=""}"
: "${PRIVATE_NS:=""}" : "${PRIVATE_NS:=""}"
if [ -n "${PRIVATE_NS}" ]; then if [ -n "${PRIVATE_NS}" ]; then
sudo -E sed -i -e 's/^DNS=/#DNS=/' /etc/systemd/resolved.conf sudo -E sed -i -e 's/^DNS=/#DNS=/' /etc/systemd/resolved.conf
@ -16,3 +15,7 @@ if [ -n "${PRIVATE_NS}" ]; then
sudo ln -s /run/systemd/resolve/resolv.conf /etc/resolv.conf sudo ln -s /run/systemd/resolve/resolv.conf /etc/resolv.conf
sudo systemctl restart systemd-resolved sudo systemctl restart systemd-resolved
fi fi
# Update CA certificates in case internal certs have been mounted to /usr/share/local/ca-certificates
# via Vagrantfile. Need to run update-ca-certificates before any `curl`, etc. commands are done.
sudo update-ca-certificates

View File

@ -14,4 +14,10 @@ helm upgrade \
./tools/deployment/common/wait-for-pods.sh loki ./tools/deployment/common/wait-for-pods.sh loki
helm -n loki test loki --logs # TODO(dustinspecker): remove this if condition and run loki test behind proxy
# loki pod's container downloads jq and curl, which won't work
# since the proxies are not configured for the pod, so skip test loki test for now
# when proxy is defined
if [ -z "$http_proxy" ]; then
helm -n loki test loki --logs
fi

View File

@ -8,12 +8,27 @@ helm upgrade \
--create-namespace \ --create-namespace \
--install \ --install \
--namespace=harbor \ --namespace=harbor \
--set harbor.proxy.httpProxy="$http_proxy" \
--set harbor.proxy.httpsProxy="$https_proxy" \
harbor \ harbor \
./charts/harbor \ ./charts/harbor \
$(./tools/deployment/common/get-values-overrides.sh harbor) $(./tools/deployment/common/get-values-overrides.sh harbor)
./tools/deployment/common/wait-for-pods.sh harbor ./tools/deployment/common/wait-for-pods.sh harbor
# Setup internal certs so that Trivy can download its database.
# Harbor's Trivy statefulset image is based on PhotonOS, which doesn't have
# an update-ca-certificates package.
# PhotonOS has support to generate the ca-bundle, but it's not installed in this
# image. The statefulset also prevents privilege escalation, so we can't intervene and
# install that.
# So, hackily append .crt files to the ca-bundle.crt.
for cert in ./tools/gate/jarvis/ubuntu-base/internal-certs/*.crt; do
kubectl exec harbor-harbor-trivy-0 \
--namespace harbor \
-- /bin/bash -c "echo \"$(cat $cert)\" >> /etc/pki/tls/certs/ca-bundle.crt"
done
helm -n harbor test harbor --logs helm -n harbor test harbor --logs
function validate() { function validate() {
@ -74,9 +89,10 @@ EOF
sudo -E docker pull harbor-core.jarvis.local/library/busybox:latest sudo -E docker pull harbor-core.jarvis.local/library/busybox:latest
sudo -E docker trust inspect --pretty harbor-core.jarvis.local/library/busybox:latest sudo -E docker trust inspect --pretty harbor-core.jarvis.local/library/busybox:latest
#Required for pipelines #Required for pipelines and standard-container
sudo docker pull docker.io/library/ubuntu:focal pushd ./tools/gate/jarvis/ubuntu-base/
sudo docker tag docker.io/library/ubuntu:focal harbor-core.jarvis.local/library/ubuntu:focal sudo docker build --build-arg http_proxy="$http_proxy" --build-arg https_proxy="$https_proxy" --build-arg no_proxy="$no_proxy" --build-arg HTTP_PROXY="$http_proxy" --build-arg HTTPS_PROXY="$https_proxy" --build-arg NO_PROXY="$no_proxy" -t harbor-core.jarvis.local/library/ubuntu:focal .
popd
sudo -E notary init -p harbor-core.jarvis.local/library/ubuntu:focal sudo -E notary init -p harbor-core.jarvis.local/library/ubuntu:focal
sudo -E docker push harbor-core.jarvis.local/library/ubuntu:focal sudo -E docker push harbor-core.jarvis.local/library/ubuntu:focal
} }

View File

@ -0,0 +1,25 @@
ARG BASE_IMAGE=ubuntu:focal
FROM ${BASE_IMAGE}
ARG http_proxy
ARG https_proxy
ARG no_proxy
ARG HTTP_PROXY
ARG HTTPS_PROXY
ARG NO_PROXY
COPY ./internal-certs /usr/local/share/ca-certificates
RUN apt-get update \
&& apt-get install \
--no-install-recommends \
--yes \
ca-certificates \
&& rm -rf /var/lib/apt/lists/*
ENV http_proxy $http_proxy
ENV https_proxy $https_proxy
ENV no_proxy $no_proxy
ENV HTTP_PROXY $HTTP_PROXY
ENV HTTPS_PROXY $HTTPS_PROXY
ENV NO_PROXY $NO_PROXY