From ff0eaeb1c88874a2becd77c130611ca01660d7b0 Mon Sep 17 00:00:00 2001 From: Dustin Specker Date: Wed, 24 Feb 2021 17:25:46 -0600 Subject: [PATCH] feat: enable development pipeline to pass behind proxy with TLS This is a squashed commit, keeping messages intact for history. - feat(charts/development-pipeline): work behind corporate proxy dockerd sidecar works behind proxy with cert. Proxy is only needed on the sidecar to pull public images. The cert is mounted via a host path so that the proxy may be trusted. - fix(standard-container/roles): remove installing Helm push plugin The standard-container Dockerfile already installs the Helm push plugin, so no reason to try to install it in multiple ansible roles. I suspect this was originally done because someone tried to use `helm push` in the ansible role, but Helm couldn't find even though it was installed in the image. But tekton defines a the HOME env var if you describe the pod in a cluster. So if we just define HELM_DATA_HOME to the location where the push plugin is installed we can remove having to install it in the ansible roles. - feat(standard-container/roles): use image.image_from for docker build Change-Id: Ibc3c5f400978cb98d2d2a37b737b56125f4c2aa7 --- .../templates/Task-createProjectAccess.yaml | 5 +++ charts/jarvis-system/values.yaml | 6 +++ tools/deployment/vagrant/Vagrantfile | 4 +- tools/gate/jarvis/700-deploy-jarvis-system.sh | 5 +++ .../templates/task-chart.yaml | 5 +++ .../templates/task-image.yaml | 41 ++++++++++++++++--- .../templates/task-promote.yaml | 4 ++ .../jarvis/development-pipeline/values.yaml | 6 +++ .../playbooks/roles/charts/tasks/main.yaml | 2 - .../playbooks/roles/images/tasks/main.yaml | 2 +- .../playbooks/roles/promote/tasks/main.yaml | 3 -- 11 files changed, 71 insertions(+), 12 deletions(-) diff --git a/charts/jarvis-system/templates/Task-createProjectAccess.yaml b/charts/jarvis-system/templates/Task-createProjectAccess.yaml index 5b74501b..52d51429 100644 --- a/charts/jarvis-system/templates/Task-createProjectAccess.yaml +++ b/charts/jarvis-system/templates/Task-createProjectAccess.yaml @@ -211,9 +211,14 @@ spec: cd "$(workspaces.output.path)"/jarvis + # escape commas in no_proxy because Helm tries to split the value on commas helm upgrade --install development-pipeline \ -n jarvis-$(params.changeNumber)-$(params.patchSetNumber) \ ./development-pipeline \ + --set proxy.http_proxy="{{ .Values.proxy.http_proxy }}" \ + --set proxy.https_proxy="{{ .Values.proxy.https_proxy }}" \ + --set proxy.no_proxy="$(echo {{ .Values.proxy.no_proxy }} | sed "s/,/\\\,/g")" \ + --set proxy.internal_certs_dir="{{ .Values.proxy.internal_certs_dir }}" \ --set "git_repo=$(params.repoRoot)/$(params.project)" \ --set "refspec=$change_ref" \ --set "namespace=jarvis-$(params.changeNumber)-$(params.patchSetNumber)" diff --git a/charts/jarvis-system/values.yaml b/charts/jarvis-system/values.yaml index 3a9a32ef..8f00cfb2 100644 --- a/charts/jarvis-system/values.yaml +++ b/charts/jarvis-system/values.yaml @@ -36,6 +36,12 @@ node_labels: key: kubernetes.io/os value: linux +proxy: + http_proxy: + https_proxy: + no_proxy: + internal_certs_dir: + over_rides: {} params: diff --git a/tools/deployment/vagrant/Vagrantfile b/tools/deployment/vagrant/Vagrantfile index c5e4ad6d..e1c21c4e 100644 --- a/tools/deployment/vagrant/Vagrantfile +++ b/tools/deployment/vagrant/Vagrantfile @@ -21,7 +21,9 @@ Vagrant.configure("2") do |config| config.vm.synced_folder ENV["INTERNAL_CERTS_DIR"], "/usr/local/share/ca-certificates/internal-certs/" # for containerd/docker to trust proxy when pulling images within kubernetes cluster config.vm.synced_folder ENV["INTERNAL_CERTS_DIR"], "/etc/containerd/cert.d/" - # for use by ubuntu-base to trust proxy + # for use by ubuntu-base , trivy, jarvis-system chart, and development-pipeline to trust proxy + # most will mount this path directly, but others copy files from it + # standard-container and 5G projects will use the ubuntu-base image as their base to get these certs config.vm.synced_folder ENV["INTERNAL_CERTS_DIR"], "/airship_charts/tools/gate/jarvis/ubuntu-base/internal-certs/" end diff --git a/tools/gate/jarvis/700-deploy-jarvis-system.sh b/tools/gate/jarvis/700-deploy-jarvis-system.sh index 785ca2b7..99ea6798 100755 --- a/tools/gate/jarvis/700-deploy-jarvis-system.sh +++ b/tools/gate/jarvis/700-deploy-jarvis-system.sh @@ -1,11 +1,16 @@ #!/bin/bash set -ex +# escape commas in no_proxy because Helm tries to split the value on commas # shellcheck disable=SC2046 helm upgrade \ --create-namespace \ --install \ --namespace=jarvis-system \ + --set proxy.http_proxy="$http_proxy" \ + --set proxy.https_proxy="$https_proxy" \ + --set proxy.no_proxy="$(echo $no_proxy | sed "s/,/\\\,/g")" \ + --set proxy.internal_certs_dir="$PWD/tools/gate/jarvis/ubuntu-base/internal-certs/" \ jarvis-system \ "./charts/jarvis-system" \ $(./tools/deployment/common/get-values-overrides.sh jarvis-system) diff --git a/tools/gate/jarvis/development-pipeline/templates/task-chart.yaml b/tools/gate/jarvis/development-pipeline/templates/task-chart.yaml index 2b3a218e..e4e34bb6 100644 --- a/tools/gate/jarvis/development-pipeline/templates/task-chart.yaml +++ b/tools/gate/jarvis/development-pipeline/templates/task-chart.yaml @@ -52,6 +52,11 @@ spec: subPath: harbor-ca - mountPath: /workspace/helm-creds name: helm-publish-creds + env: + # specify HELM_DATA_HOME since tekton defines HOME as /home/tekton, which is used by Helm by default meaning + # Helm won't find any plugins installed during image build time + - name: HELM_DATA_HOME + value: /root/.local/share/helm image: {{ $.Values.tasks.chart.buildChartImage }} script: | ls -ltr diff --git a/tools/gate/jarvis/development-pipeline/templates/task-image.yaml b/tools/gate/jarvis/development-pipeline/templates/task-image.yaml index e6188f33..4c13d479 100644 --- a/tools/gate/jarvis/development-pipeline/templates/task-image.yaml +++ b/tools/gate/jarvis/development-pipeline/templates/task-image.yaml @@ -95,11 +95,20 @@ spec: sidecars: - image: {{ $.Values.tasks.image.sidecarServer }} name: server - args: - - --storage-driver=overlay2 - - --userland-proxy=false - - --debug - - --insecure-registry={{ $.Values.tasks.image.insecureRegistry }} + script: | + # Alpine's update-ca-certificates does not support directories, so copy files + # Copy files instead of mounting to prevent overriding built-in certificates + if ls /internal-certs/*.crt ; then + cp /internal-certs/*.crt /usr/local/share/ca-certificates/ + fi + + update-ca-certificates + + /usr/local/bin/dockerd-entrypoint.sh \ + --storage-driver=overlay2 \ + --userland-proxy=false \ + --debug \ + --insecure-registry={{ $.Values.tasks.image.insecureRegistry }} ##TODO: Get rid of privileged true securityContext: privileged: true @@ -107,6 +116,18 @@ spec: # Write generated certs to the path shared with the client. - name: DOCKER_TLS_CERTDIR value: /certs + - name: http_proxy + value: {{ $.Values.proxy.http_proxy }} + - name: https_proxy + value: {{ $.Values.proxy.https_proxy }} + - name: no_proxy + value: {{ $.Values.proxy.no_proxy }} + - name: HTTP_PROXY + value: {{ $.Values.proxy.http_proxy }} + - name: HTTPS_PROXY + value: {{ $.Values.proxy.https_proxy }} + - name: NO_PROXY + value: {{ $.Values.proxy.no_proxy }} volumeMounts: - mountPath: /certs/client name: dind-certs @@ -115,6 +136,10 @@ spec: - mountPath: /usr/local/share/ca-certificates/harbor-ca.crt name: harbor-ca subPath: harbor-ca +{{ if .Values.proxy.internal_certs_dir }} + - mountPath: /internal-certs + name: internal-certs +{{ end }} # Wait for the dind daemon to generate the certs it will share with the # client. readinessProbe: @@ -135,3 +160,9 @@ spec: - name: harbor-ca secret: secretName: harbor-ca +{{ if .Values.proxy.internal_certs_dir }} + - name: internal-certs + hostPath: + path: {{ .Values.proxy.internal_certs_dir }} + type: Directory +{{ end }} diff --git a/tools/gate/jarvis/development-pipeline/templates/task-promote.yaml b/tools/gate/jarvis/development-pipeline/templates/task-promote.yaml index 6f227166..9baf4d9e 100644 --- a/tools/gate/jarvis/development-pipeline/templates/task-promote.yaml +++ b/tools/gate/jarvis/development-pipeline/templates/task-promote.yaml @@ -22,6 +22,10 @@ spec: # Use the certs generated by the sidecar daemon. - name: DOCKER_CERT_PATH value: /certs/client + # specify HELM_DATA_HOME since tekton defines HOME as /home/tekton, which is used by Helm by default meaning + # Helm won't find any plugins installed during image build time + - name: HELM_DATA_HOME + value: /root/.local/share/helm volumeMounts: - mountPath: /tekton/home/.docker/config.json name: image-push-creds diff --git a/tools/gate/jarvis/development-pipeline/values.yaml b/tools/gate/jarvis/development-pipeline/values.yaml index c15d49df..610f1b57 100644 --- a/tools/gate/jarvis/development-pipeline/values.yaml +++ b/tools/gate/jarvis/development-pipeline/values.yaml @@ -3,6 +3,12 @@ pvc: storageClass: standard size: 1Gi +proxy: + http_proxy: + https_proxy: + no_proxy: + internal_certs_dir: + tasks: kubernetes: createClusterImage: &base_image standard-container:1.0 diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/roles/charts/tasks/main.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/roles/charts/tasks/main.yaml index ec097920..f4636902 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/roles/charts/tasks/main.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/roles/charts/tasks/main.yaml @@ -99,8 +99,6 @@ - name: Helm Publish when: ( stage == "publish") block: - - name: Install Plugin - shell: helm plugin update push || helm plugin install https://github.com/chartmuseum/helm-push || true - name: Push chart to Harbor staging registry command: helm push "{{ chart.chart_name }}-{{ chart.version }}".tgz "{{ chart.project }}-staging" args: diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/roles/images/tasks/main.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/roles/images/tasks/main.yaml index c5d2aab3..7b067eea 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/roles/images/tasks/main.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/roles/images/tasks/main.yaml @@ -8,7 +8,7 @@ block: - name: Build Docker Image shell: | - docker build -t "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}:{{ image.tag }}" . + docker build --build-arg FROM="{{ image.image_from }}" -t "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}:{{ image.tag }}" . args: chdir: "{{ image.build.target_loc }}/{{ image.path }}" loop: "{{ images | selectattr('build_from_source', 'equalto', True) | list }}" diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/roles/promote/tasks/main.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/roles/promote/tasks/main.yaml index adcc5b07..b6131b9c 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/roles/promote/tasks/main.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/roles/promote/tasks/main.yaml @@ -11,9 +11,6 @@ - name: Promote Chart when: ( stage == "promote_chart") block: - - name: Install Plugin - shell: helm plugin update push || helm plugin install https://github.com/chartmuseum/helm-push || true - - name: Pull down Helm Chart shell: helm pull "{{ chart.project }}-staging/{{ chart.chart_name }}" --version="{{ chart.version }}" loop: "{{ items }}"