From 191342b6c3a1362c7c75b0d3171e25f5f37a7632 Mon Sep 17 00:00:00 2001 From: Cole Walker Date: Tue, 14 Jul 2020 15:10:12 -0400 Subject: [PATCH] Changed vault-mgr to statefulset for performance Also contains cleanup and formatting, removal of unused or duplicate files. Story: 2007718 Task: 40326 Change-Id: I64d5b2d14f8868da17c0e88b71b5b1d35b0e0f4e Signed-off-by: Cole Walker --- stx-vault-helm/centos/stx-vault-helm.spec | 18 -- .../helm-charts/_helpers-CA.tpl | 11 - .../helm-charts/vault-certificates.yaml | 64 ------ .../helm-charts/vault-init.yaml | 217 ------------------ vault-helm/centos/build_srpm.data | 2 +- vault-helm/centos/vault-helm.spec | 22 +- .../helm-charts/vault-certificates.yaml | 8 +- .../vault-helm/helm-charts/vault-init.yaml | 28 +-- .../vault-helm/manifests/vault_manifest.yaml | 126 ---------- 9 files changed, 24 insertions(+), 472 deletions(-) delete mode 100644 stx-vault-helm/stx-vault-helm/helm-charts/_helpers-CA.tpl delete mode 100644 stx-vault-helm/stx-vault-helm/helm-charts/vault-certificates.yaml delete mode 100644 stx-vault-helm/stx-vault-helm/helm-charts/vault-init.yaml delete mode 100644 vault-helm/vault-helm/manifests/vault_manifest.yaml diff --git a/stx-vault-helm/centos/stx-vault-helm.spec b/stx-vault-helm/centos/stx-vault-helm.spec index 3dccbac..6a8c144 100644 --- a/stx-vault-helm/centos/stx-vault-helm.spec +++ b/stx-vault-helm/centos/stx-vault-helm.spec @@ -21,14 +21,6 @@ Packager: Wind River URL: unknown Source0: helm-charts-vault-0-6-0.tar.gz -#Source1: repositories.yaml -#Source2: index.yaml -#Source3: Makefile -#Source4: metadata.yaml -#Source5: vault_manifest.yaml -#Source6: vault-init.yaml -#Source7: vault-certificates.yaml -#Source8: _helpers-CA.tpl BuildArch: noarch @@ -69,16 +61,6 @@ helm serve --repo-path . & helm repo rm local helm repo add local http://localhost:8879/charts -# Create the tgz file -#cp %{SOURCE3} ./ -#mkdir ./vault -#cp ./Chart.yaml ./vault -#mv ./values.yaml ./vault -#cp %{SOURCE6} ./templates -#cp %{SOURCE7} ./templates -#cat %{SOURCE8} >> ./templates/_helpers.tpl -#mv ./templates ./vault/templates - cd helm-charts make psp-rolebinding cd - diff --git a/stx-vault-helm/stx-vault-helm/helm-charts/_helpers-CA.tpl b/stx-vault-helm/stx-vault-helm/helm-charts/_helpers-CA.tpl deleted file mode 100644 index d2ef9cd..0000000 --- a/stx-vault-helm/stx-vault-helm/helm-charts/_helpers-CA.tpl +++ /dev/null @@ -1,11 +0,0 @@ - -{{/* -Generate certificates for vault CA -*/}} -{{- define "vault.gen-certs" -}} -{{- $altNames := list ( printf "%s.%s" (include "vault.name" .) .Release.Namespace ) ( printf "%s.%s.svc" (include "vault.name" .) .Release.Namespace ) -}} -{{- $ca := genCA "vault-ca" 365 -}} -{{- $cert := genSignedCert ( include "vault.name" . ) nil $altNames 365 $ca -}} -tls.crt: {{ $ca.Cert | b64enc }} -tls.key: {{ $ca.Key | b64enc }} -{{- end -}} \ No newline at end of file diff --git a/stx-vault-helm/stx-vault-helm/helm-charts/vault-certificates.yaml b/stx-vault-helm/stx-vault-helm/helm-charts/vault-certificates.yaml deleted file mode 100644 index 4f34138..0000000 --- a/stx-vault-helm/stx-vault-helm/helm-charts/vault-certificates.yaml +++ /dev/null @@ -1,64 +0,0 @@ -apiVersion: v1 -kind: Secret -type: kubernetes.io/tls -metadata: - name: {{ template "vault.name" . }}-ca - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "vault.name" . }} - chart: {{ template "vault.chart" . }} - heritage: {{ .Release.Service }} - release: {{ .Release.Name }} - annotations: - "helm.sh/hook": "pre-install" - "helm.sh/hook-delete-policy": "before-hook-creation" -data: -{{ ( include "vault.gen-certs" . ) | indent 2 }} ---- -apiVersion: cert-manager.io/v1alpha2 -kind: Issuer -metadata: - name: ca-issuer - namespace: {{ .Release.Namespace }} -spec: - ca: - secretName: {{ template "vault.name" . }}-ca ---- -apiVersion: cert-manager.io/v1alpha2 -kind: Certificate -metadata: - name: vault-server-tls - namespace: {{ .Release.Namespace }} -spec: - # Secret names are always required. - secretName: vault-server-tls - duration: 2160h # 90d - renewBefore: 360h # 15d - organization: - - stx - isCA: false - keySize: 2048 - keyAlgorithm: rsa - keyEncoding: pkcs1 - usages: - - server auth - - client auth - # At least one of a DNS Name, URI, or IP address is required. - dnsNames: - - sva-{{ template "vault.name" . }} - - '*.sva-{{ template "vault.name" . }}-internal' - - '*.{{ .Release.Namespace }}.pod.cluster.local' - - sva-{{ template "vault.name" . }}.{{ .Release.Namespace }} - - sva-{{ template "vault.name" . }}.{{ .Release.Namespace }}.svc - - sva-{{ template "vault.name" . }}.{{ .Release.Namespace }}.svc.cluster.local - ipAddresses: - - 127.0.0.1 - # Issuer references are always required. - issuerRef: - name: ca-issuer - # We can reference ClusterIssuers by changing the kind here. - # The default value is Issuer (i.e. a locally namespaced Issuer) - kind: Issuer - # This is optional since cert-manager will default to this value however - # if you are using an external issuer, change this to that issuer group. - group: cert-manager.io \ No newline at end of file diff --git a/stx-vault-helm/stx-vault-helm/helm-charts/vault-init.yaml b/stx-vault-helm/stx-vault-helm/helm-charts/vault-init.yaml deleted file mode 100644 index 68b553e..0000000 --- a/stx-vault-helm/stx-vault-helm/helm-charts/vault-init.yaml +++ /dev/null @@ -1,217 +0,0 @@ -apiVersion: v1 -data: - init.sh: | - #!/bin/bash - - CERT=$CA_CERT # Get the CA path from environment vars - CA_ONELINE=$(awk '{printf "%s\\n", $0}' $CERT) # Store cert as a oneliner for curl purposes - DOMAIN={{ .Release.Namespace }}.pod.cluster.local # Set the domain for resolving pod names - WORKDIR=$PVCDIR # PVC location so that keys can be persisted - - # FUNCTIONS - - # Creates a list of all k8s vault pods and stores in text file. - # Converts ips from X.X.X.X to X-X-X-X for use as pod dns names - function getVaultPods { - kubectl get pods -n {{ .Release.Namespace }} -l component=server,app.kubernetes.io/name=vault -o=jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.podIPs[].ip}{"\n"}{end}' > $WORKDIR/pods.txt - sed -i 's/\./-/g' $WORKDIR/pods.txt - } - - # Wait for the vault servers in the stateful set to be created before initializing - function waitForPods { - CURRENT_PODS=$(kubectl get pods -l component=server,app.kubernetes.io/name=vault \ - -o=jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.podIPs[].ip}{"\t"}{.status.phase}{"\n"} \ - {end}' | grep Running | wc -l) - DESIRED_PODS={{ .Values.server.ha.replicas }} - - while [ $CURRENT_PODS != $DESIRED_PODS ]; do - sleep 5 - echo "Waiting for {{ template "vault.fullname" . }} statefulset running pods ($CURRENT_PODS) to equal desired pods ($DESIRED_PODS)" - CURRENT_PODS=$(kubectl get pods -l component=server,app.kubernetes.io/name=vault \ - -o=jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.podIPs[].ip}{"\t"}{.status.phase}{"\n"} \ - {end}' | grep Running | wc -l) - done - } - - # Initializes the first vault pod, only needs to be performed once after deploying the helm chart - # Stores the root token and master key shards in plaintext in working directory as cluster_keys.json - insecure. - function initVault { - V0=$(awk 'NR==1{print $2}' $WORKDIR/pods.txt) - echo "Initializing $V0" - curl -s --cacert $CERT --request POST --data '{"secret_shares": 5, "secret_threshold": 3}' https://$V0.$DOMAIN:8200/v1/sys/init > $WORKDIR/cluster_keys.json - } - - # Uses the master key shards in cluster_keys.json to unseal vault - function unsealVault { - for shard in $(cat $WORKDIR/cluster_keys.json | jq -r .keys_base64[]); do - echo {\"key\": \"$shard\"} | curl -s --cacert $CERT --request POST -d @- https://$VAULT.$DOMAIN:8200/v1/sys/unseal > /dev/null - sleep 3 #Some sleep is required to allow Raft convergence - done - } - - # Takes the address of vault-0 as the cluster leader and joins other nodes to raft - function joinRaft { - CLUSTER_LEAD=$(awk 'NR==1{print $2}' $WORKDIR/pods.txt) - ROOT_TOKEN=$(cat $WORKDIR/cluster_keys.json | jq -r .root_token) - curl -s --cacert $CERT -H "X-Vault-Token: $ROOT_TOKEN" --request POST --data "{\"leader_api_addr\": \"https://$CLUSTER_LEAD.$DOMAIN:8200\", \"leader_ca_cert\": \"$CA_ONELINE\"}" https://$row.$DOMAIN:8200/v1/sys/storage/raft/join - } - - # Simply calls the status check of a vault, used to check if it is initialized, unsealed, or part of raft cluster - function vaultServerStatus { - curl --cacert $CERT -s https://$row.$DOMAIN:8200/v1/sys/health | jq - } - - - # - # LOGIC - # - - # Waiting for vault servers to come up - waitForPods - - echo "" - echo "Putting a list of vault pods and ip in $WORKDIR/pods.txt" - getVaultPods - - echo "" - row=$(awk 'NR==1{print $2}' $WORKDIR/pods.txt) - vaultServerStatus > $WORKDIR/healthcheck.txt - TEMP=$(cat $WORKDIR/healthcheck.txt | jq -r .initialized) - grep $row $WORKDIR/pods.txt & echo "Initialized status is $TEMP" - if [ ! -z $TEMP ] && [ $TEMP = false ]; then - - echo "Initializing the vault on vault-0 and storing keys in $WORKDIR/cluster_keys.json" - initVault - sleep 10 #Some sleep required to allow convergence" - - echo "" - echo "Unsealing vault-0 using the init shards" - for row in $(awk 'NR==1{print $2}' $WORKDIR/pods.txt); do - VAULT=$row - unsealVault - done - - echo "" - echo "Joining other vault servers to the HA Raft cluster" - for row in $(awk 'NR>1{print $2}' $WORKDIR/pods.txt); do - grep $row $WORKDIR/pods.txt - joinRaft - sleep 5 - done - - echo "" - echo "Unsealing the remaining vaults" - for row in $(awk 'NR>1{print $2}' $WORKDIR/pods.txt); do - grep $row $WORKDIR/pods.txt - VAULT=$row - unsealVault - sleep 10 - done - fi - - # Loop forever to check the seal status of vaults and unseal if required - while true; do - sleep 5 - echo "Checking vault pods seal status" - rm $WORKDIR/pods.txt - getVaultPods - for row in $(awk '{print $2}' $WORKDIR/pods.txt); do - vaultServerStatus > $WORKDIR/healthcheck.txt - TEMP=$(cat $WORKDIR/healthcheck.txt | jq -r .sealed) - grep $row $WORKDIR/pods.txt & echo "Sealed status is $TEMP" - if [ ! -z $TEMP ] && [ $TEMP = true ]; then - VAULT=$row - echo "Unsealing $row" - unsealVault - fi - done - done -kind: ConfigMap -metadata: - managedFields: - - apiVersion: v1 - fieldsType: FieldsV1 - fieldsV1: - f:data: - .: {} - f:init.sh: {} - manager: vault-init-unseal - name: vault-init-unseal - namespace: {{ .Release.Namespace }} ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: manager-pvc - namespace: {{ .Release.Namespace }} -spec: - accessModes: - - ReadWriteOnce - storageClassName: general - resources: - requests: - storage: 1Gi ---- -{{- if and (eq (.Values.injector.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") }} -# Deployment for the unsealer -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "vault.fullname" . }}-manager - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/name: {{ include "vault.name" . }}-manager - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - component: webhook -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/instance: {{ .Release.Name }} - component: webhook - template: - metadata: - labels: - app.kubernetes.io/name: {{ template "vault.name" . }}-manager - app.kubernetes.io/instance: {{ .Release.Name }} - component: webhook - spec: - serviceAccountName: "{{ template "vault.fullname" . }}" - containers: - - name: manager - image: cwalops/k8 - imagePullPolicy: "{{ .Values.injector.image.pullPolicy }}" - args: - - bash - - /opt/script/init.sh - env: - - name: PVCDIR - value: /mnt/data - - name: CA_CERT - value: /mnt/data/ca/tls.crt - volumeMounts: - - name: vault-init-unseal - mountPath: /opt/script - readOnly: false - - name: manager-pvc - mountPath: /mnt/data - readOnly: false - - name: vault-ca - mountPath: /mnt/data/ca - readOnly: true - volumes: - - name: vault-init-unseal - configMap: - name: vault-init-unseal - - name: manager-pvc - persistentVolumeClaim: - claimName: manager-pvc - - name: vault-ca - secret: - secretName: vault-ca - {{- if .Values.global.imagePullSecrets }} - imagePullSecrets: - {{- toYaml .Values.global.imagePullSecrets | nindent 8 }} - {{- end }} -{{ end }} \ No newline at end of file diff --git a/vault-helm/centos/build_srpm.data b/vault-helm/centos/build_srpm.data index 83d12d2..d45a4f3 100644 --- a/vault-helm/centos/build_srpm.data +++ b/vault-helm/centos/build_srpm.data @@ -4,7 +4,7 @@ TAR_NAME=helm-charts-vault-0-6-0 VERSION=1.0.0 TAR="$TAR_NAME.tar.gz" -COPY_LIST="${CGCS_BASE}/downloads/$TAR $PKG_BASE/$SRC_DIR/files/* $PKG_BASE/$SRC_DIR/manifests/* $PKG_BASE/$SRC_DIR/helm-charts/*" +COPY_LIST="${CGCS_BASE}/downloads/$TAR $PKG_BASE/$SRC_DIR/files/* $PKG_BASE/$SRC_DIR/helm-charts/*" TIS_PATCH_VER=0 diff --git a/vault-helm/centos/vault-helm.spec b/vault-helm/centos/vault-helm.spec index b13ad81..c6dc6a3 100644 --- a/vault-helm/centos/vault-helm.spec +++ b/vault-helm/centos/vault-helm.spec @@ -25,10 +25,9 @@ Source1: repositories.yaml Source2: index.yaml Source3: Makefile Source4: metadata.yaml -Source5: vault_manifest.yaml -Source6: vault-init.yaml -Source7: vault-certificates.yaml -Source8: _helpers-CA.tpl +Source5: vault-init.yaml +Source6: vault-certificates.yaml +Source7: _helpers-CA.tpl BuildArch: noarch @@ -70,9 +69,9 @@ cp %{SOURCE3} ./ mkdir ./vault cp ./Chart.yaml ./vault mv ./values.yaml ./vault +cp %{SOURCE5} ./templates cp %{SOURCE6} ./templates -cp %{SOURCE7} ./templates -cat %{SOURCE8} >> ./templates/_helpers.tpl +cat %{SOURCE7} >> ./templates/_helpers.tpl mv ./templates ./vault/templates make vault @@ -88,21 +87,10 @@ kill %1 # Setup staging mkdir -p %{app_staging} cp %{SOURCE4} %{app_staging} -cp %{SOURCE5} %{app_staging} mkdir -p %{app_staging}/charts cp ./helm-charts-vault/*.tgz %{app_staging}/charts cd %{app_staging} -# Populate metadata -#sed -i 's/@APP_NAME@/%{app_name}/g' %{app_staging}/metadata.yaml -#sed -i 's/@APP_VERSION@/%{version}-%{tis_patch_ver}/g' %{app_staging}/metadata.yaml -#sed -i 's/@HELM_REPO@/%{helm_repo}/g' %{app_staging}/metadata.yaml - - -# Copy the plugins: installed in the buildroot -#mkdir -p %{app_staging}/plugins -#cp /plugins/%{app_name}/*.whl %{app_staging}/plugins - # package it up find . -type f ! -name '*.md5' -print0 | xargs -0 md5sum > checksum.md5 tar -zcf %{_builddir}/%{app_tarball} -C %{app_staging}/ . diff --git a/vault-helm/vault-helm/helm-charts/vault-certificates.yaml b/vault-helm/vault-helm/helm-charts/vault-certificates.yaml index 4f34138..60d73f9 100644 --- a/vault-helm/vault-helm/helm-charts/vault-certificates.yaml +++ b/vault-helm/vault-helm/helm-charts/vault-certificates.yaml @@ -1,3 +1,8 @@ +{{ $ca := genCA "svc-cat-ca" 3650 }} +{{ $cn := printf "fluentd.%s.svc.cluster.local" .Release.Namespace }} +{{ $server := genSignedCert $cn nil nil 365 $ca }} +{{ $client := genSignedCert "" nil nil 365 $ca }} + apiVersion: v1 kind: Secret type: kubernetes.io/tls @@ -13,7 +18,8 @@ metadata: "helm.sh/hook": "pre-install" "helm.sh/hook-delete-policy": "before-hook-creation" data: -{{ ( include "vault.gen-certs" . ) | indent 2 }} + tls.crt: {{ b64enc $ca.Cert }} + tls.key: {{ b64enc $ca.Key }} --- apiVersion: cert-manager.io/v1alpha2 kind: Issuer diff --git a/vault-helm/vault-helm/helm-charts/vault-init.yaml b/vault-helm/vault-helm/helm-charts/vault-init.yaml index 68b553e..0fd9626 100644 --- a/vault-helm/vault-helm/helm-charts/vault-init.yaml +++ b/vault-helm/vault-helm/helm-charts/vault-init.yaml @@ -82,6 +82,7 @@ data: echo "Initializing the vault on vault-0 and storing keys in $WORKDIR/cluster_keys.json" initVault + cp $WORKDIR/cluster_keys.json $WORKDIR/cluster_init.json sleep 10 #Some sleep required to allow convergence" echo "" @@ -139,23 +140,10 @@ metadata: name: vault-init-unseal namespace: {{ .Release.Namespace }} --- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: manager-pvc - namespace: {{ .Release.Namespace }} -spec: - accessModes: - - ReadWriteOnce - storageClassName: general - resources: - requests: - storage: 1Gi ---- {{- if and (eq (.Values.injector.enabled | toString) "true" ) (eq (.Values.global.enabled | toString) "true") }} # Deployment for the unsealer apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: {{ template "vault.fullname" . }}-manager namespace: {{ .Release.Namespace }} @@ -204,12 +192,18 @@ spec: - name: vault-init-unseal configMap: name: vault-init-unseal - - name: manager-pvc - persistentVolumeClaim: - claimName: manager-pvc - name: vault-ca secret: secretName: vault-ca + volumeClaimTemplates: + - metadata: + name: manager-pvc + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.global.imagePullSecrets | nindent 8 }} diff --git a/vault-helm/vault-helm/manifests/vault_manifest.yaml b/vault-helm/vault-helm/manifests/vault_manifest.yaml deleted file mode 100644 index 914897d..0000000 --- a/vault-helm/vault-helm/manifests/vault_manifest.yaml +++ /dev/null @@ -1,126 +0,0 @@ ---- -schema: armada/Chart/v1 -metadata: - schema: metadata/Document/v1 - name: vault-psp-rolebinding -data: - chart_name: psp-rolebinding - release: vault-psp-rolebinding - namespace: vault - values: - rolebindingNamespace: vault - serviceAccount: vault - source: - location: http://172.17.0.1:8080/helm_charts/stx-platform/psp-rolebinding-0.1.0.tgz - subpath: psp-rolebinding - type: tar - reference: master - upgrade: - no_hooks: false - pre: - delete: - - labels: - release_group: vault-psp-rolebinding - type: job - wait: - labels: - release_group: vault-psp-rolebinding - resources: [] - timeout: 1800 - dependencies: [] ---- -schema: armada/Chart/v1 -metadata: - schema: metadata/Document/v1 - name: vault -data: - chart_name: vault - release: vault - namespace: vault - wait: - timeout: 1800 - labels: - app: vault - install: - no_hooks: false - upgrade: - no_hooks: false - pre: - delete: - - type: job - labels: - app: vault - values: - global: - enabled: true - tlsDisable: false - injector: - enabled: true - image: - repository: hashicorp/vault-k8s - tag: 0.4.0 - agentImage: - repository: vault - tag: 1.4.2 - server: - image: - repository: vault - tag: 1.4.2 - auditStorage: - enabled: true - size: 10Gi - ha: - enabled: true - replicas: 3 - raft: - enabled: true - config: | - ui = true - - listener "tcp" { - tls_disable = 0 - address = "[::]:8200" - cluster_address = "[::]:8201" - tls_cert_file = "/vault/userconfig/vault-server-tls/tls.crt" - tls_key_file = "/vault/userconfig/vault-server-tls/tls.key" - tls_client_ca_file = "/vault/userconfig/vault-server-tls/ca.crt" - } - - storage "raft" { - path = "/vault/data" - } - - service_registration "kubernetes" {} - extraLabels: - app: vault - extraEnvironmentVars: - VAULT_CACERT: /vault/userconfig/vault-server-tls/ca.crt - extraVolumes: - - type: secret - name: vault-server-tls - source: - type: tar - location: http://172.17.0.1/helm_charts/stx-platform/vault-0.6.0.tgz - subpath: vault - reference: master - dependencies: [] ---- -schema: armada/ChartGroup/v1 -metadata: - schema: metadata/Document/v1 - name: vault -data: - description: "Deploy Vault" - sequenced: false - chart_group: - - vault - - vault-psp-rolebinding ---- -schema: armada/Manifest/v1 -metadata: - schema: metadata/Document/v1 - name: vault-manifest -data: - release_prefix: sva - chart_groups: - - vault \ No newline at end of file