openstack/openstack-helm-infra
Code Issues Proposed changes

feat(tls): add tls support to helm-toolkit

Browse Source 
This patch set:

- allows options in the bootstrap job to load the proper TLS secret into
  the  proper envvar so the openstack client can connect properly to
  perform bootstrap;
- adds in certificates to make rally work properly with TLS endpoints;
- adds methods to handle TLS secret volume and volumeMount;
- updates ingress to handle secure backends.

Change-Id: I322cda393f18bfeed0b9f8b1827d101f60d6bdeb
Signed-off-by: Tin Lam <tin@irrational.io>
This commit is contained in:
Tin Lam 2020-06-21 14:35:42 -05:00
parent 5e316a9ba0
commit 7cb3ef69ae
12 changed files with 300 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
helm-toolkit/templates/manifests/_certificates.tpl
View File

@ -19,13 +19,17 @@ examples:
- values: |
endpoints:
dashboard:
certs:
horizon-internal-cert:
secretName: horizon-tls-apache
host_fqdn_override:
default:
host: null
tls:
secretName: keystone-tls-api
issuerRef:
name: ca-issuer
duration: 2160h
organization:
- ACME
commonName: horizon-int.openstack.svc.cluster.local
commonName: keystone-api.openstack.svc.cluster.local
keySize: 2048
usages:
- server auth
@ -34,29 +38,27 @@ examples:
- cluster.local
issuerRef:
name: ca-issuer
kind: Issuer
usage: |
{{- $opts := dict "envAll" . "service" "dashboard" "type" "internal" "certName" "horizon-internal-cert" -}}
{{- $opts := dict "envAll" . "service" "dashboard" "type" "internal" -}}
{{ $opts | include "helm-toolkit.manifests.certificates" }}
return: |
---
apiVersion: cert-manager.io/v1alpha3
kind: Certificate
metadata:
name: horizon_internal_cert
name: keystone-tls-api
namespace: NAMESPACE
spec:
commonName: horizon-int.openstack.svc.cluster.local
commonName: keystone-api.openstack.svc.cluster.local
dnsNames:
- cluster.local
duration: 2160h
issuerRef:
kind: Issuer
name: ca-issuer
keySize: 2048
organization:
- ACME
secretName: horizon-tls-apache
secretName: keystone-tls-api
usages:
- server auth
- client auth
@ -66,37 +68,36 @@ examples:
{{- $envAll := index . "envAll" -}}
{{- $service := index . "service" -}}
{{- $type := index . "type" | default "" -}}
{{- $name := index . "certName" -}}
{{- $slice := index $envAll.Values.endpoints $service "certs" $name -}}
{{- $slice := index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls" -}}
{{/* Put in some sensible default value if one is not provided by values.yaml */}}
{{/* If a dnsNames list is not in the values.yaml, it can be overridden by a passed-in parameter.
This allows user to use other HTK method to determine the URI and pass that into this method.*/}}
{{- if not (hasKey $slice "dnsNames") -}}
{{- $hostName := tuple $service $type $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" -}}
{{- $dnsNames := list $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) -}}
{{- $_ := $dnsNames | set (index $envAll.Values.endpoints $service "certs" $name) "dnsNames" -}}
{{- $_ := $dnsNames | set (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls") "dnsNames" -}}
{{- end -}}
{{/* Default keySize to 4096. This can be overridden. */}}
{{- if not (hasKey $slice "keySize") -}}
{{- $_ := ( printf "%d" 4096 | atoi ) | set (index $envAll.Values.endpoints $service "certs" $name) "keySize" -}}
{{- $_ := ( printf "%d" 4096 | atoi ) | set (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls") "keySize" -}}
{{- end -}}
{{/* Default keySize to 3 months. Note the min is 720h. This can be overridden. */}}
{{- if not (hasKey $slice "duration") -}}
{{- $_ := printf "%s" "2190h" | set (index $envAll.Values.endpoints $service "certs" $name) "duration" -}}
{{- $_ := printf "%s" "2190h" | set (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls") "duration" -}}
{{- end -}}
{{/* Default renewBefore to 15 days. This can be overridden. */}}
{{- if not (hasKey $slice "renewBefore") -}}
{{- $_ := printf "%s" "360h" | set (index $envAll.Values.endpoints $service "certs" $name) "renewBefore" -}}
{{- $_ := printf "%s" "360h" | set (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls") "renewBefore" -}}
{{- end -}}
{{/* Default the usage to server auth and client auth. This can be overridden. */}}
{{- if not (hasKey $slice "usages") -}}
{{- $_ := (list "server auth" "client auth") | set (index $envAll.Values.endpoints $service "certs" $name) "usages" -}}
{{- $_ := (list "server auth" "client auth") | set (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls") "usages" -}}
{{- end -}}
---
apiVersion: cert-manager.io/v1alpha3
kind: Certificate
metadata:
name: {{ $name | replace "_" "-" }}
name: {{ index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls" "secretName" }}
namespace: {{ $envAll.Release.Namespace }}
spec:
{{ $slice | toYaml | indent 2 }}

106
helm-toolkit/templates/manifests/_ingress.tpl
View File

@ -219,6 +219,97 @@ examples:
backend:
serviceName: barbican-api
servicePort: b-api
- values: |
network:
api:
ingress:
public: true
classes:
namespace: "nginx"
cluster: "nginx-cluster"
annotations:
nginx.ingress.kubernetes.io/secure-backends: "true"
nginx.ingress.kubernetes.io/backend-protocol: "https"
secrets:
tls:
key_manager:
api:
public: barbican-tls-public
internal: barbican-tls-api
endpoints:
cluster_domain_suffix: cluster.local
key_manager:
name: barbican
hosts:
default: barbican-api
public:
host: barbican
tls:
crt: |
FOO-CRT
key: |
FOO-KEY
ca: |
FOO-CA_CRT
host_fqdn_override:
default: null
path:
default: /
scheme:
default: http
public: https
port:
api:
default: 9311
public: 80
certs:
barbican_tls_api:
secretName: barbican-tls-api
issuerRef:
name: ca-issuer
kind: Issuer
usage: |
{{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" ) -}}
return: |
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: barbican
annotations:
kubernetes.io/ingress.class: "nginx"
cert-manager.io/issuer: ca-issuer
nginx.ingress.kubernetes.io/backend-protocol: https
nginx.ingress.kubernetes.io/secure-backends: "true"
spec:
tls:
- secretName: barbican-tls-public-certmanager
hosts:
- barbican
- barbican.default
- barbican.default.svc.cluster.local
rules:
- host: barbican
http:
paths:
- path: /
backend:
serviceName: barbican-api
servicePort: b-api
- host: barbican.default
http:
paths:
- path: /
backend:
serviceName: barbican-api
servicePort: b-api
- host: barbican.default.svc.cluster.local
http:
paths:
- path: /
backend:
serviceName: barbican-api
servicePort: b-api
*/}}
{{- define "helm-toolkit.manifests.ingress._host_rules" -}}
@ -240,6 +331,7 @@ examples:
{{- $backendServiceType := index . "backendServiceType" -}}
{{- $backendPort := index . "backendPort" -}}
{{- $endpoint := index . "endpoint" | default "public" -}}
{{- $certIssuer := index . "certIssuer" | default "" -}}
{{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
{{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
{{- $hostName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
@ -251,9 +343,22 @@ metadata:
name: {{ $ingressName }}
annotations:
kubernetes.io/ingress.class: {{ index $envAll.Values.network $backendService "ingress" "classes" "namespace" | quote }}
{{- if $certIssuer }}
cert-manager.io/issuer: {{ $certIssuer }}
{{- end }}
{{ toYaml (index $envAll.Values.network $backendService "ingress" "annotations") | indent 4 }}
spec:
{{- $host := index $envAll.Values.endpoints ( $backendServiceType | replace "-" "_" ) "hosts" }}
{{- if $certIssuer }}
{{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }}
{{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }}
tls:
- secretName: {{ printf "%s-ing" $secretName }}
hosts:
{{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}
- {{ $vHost }}
{{- end }}
{{- else }}
{{- if hasKey $host $endpoint }}
{{- $endpointHost := index $host $endpoint }}
{{- if kindIs "map" $endpointHost }}
@ -270,6 +375,7 @@ spec:
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
rules:
{{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }}

7
helm-toolkit/templates/manifests/_job-bootstrap.tpl
View File

@ -27,6 +27,7 @@ limitations under the License.
{{- $configMapEtc := index . "configMapEtc" | default (printf "%s-%s" $serviceName "etc" ) -}}
{{- $configFile := index . "configFile" | default (printf "/etc/%s/%s.conf" $serviceName $serviceName ) -}}
{{- $logConfigFile := index . "logConfigFile" | default (printf "/etc/%s/logging.conf" $serviceName ) -}}
{{- $tlsSecret := index . "tlsSecret" | default "" -}}
{{- $keystoneUser := index . "keystoneUser" | default $serviceName -}}
{{- $openrc := index . "openrc" | default "true" -}}
{{- $secretBin := index . "secretBin" -}}
@ -66,7 +67,7 @@ spec:
{{ tuple $envAll $envAll.Values.pod.resources.jobs.bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{- if eq $openrc "true" }}
env:
{{- with $env := dict "ksUserSecret" ( index $envAll.Values.secrets.identity $keystoneUser ) }}
{{- with $env := dict "ksUserSecret" ( index $envAll.Values.secrets.identity $keystoneUser ) "useCA" (ne $tlsSecret "") }}
{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }}
{{- end }}
{{- end }}
@ -91,6 +92,7 @@ spec:
mountPath: {{ $logConfigFile | quote }}
subPath: {{ base $logConfigFile | quote }}
readOnly: true
{{ dict "enabled" (ne $tlsSecret "") "name" $tlsSecret | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
{{- if $podVolMounts }}
{{ $podVolMounts | toYaml | indent 12 }}
{{- end }}
@ -112,7 +114,8 @@ spec:
- name: bootstrap-conf
secret:
secretName: {{ $configMapEtc | quote }}
defaultMode: 292
defaultMode: 0444
{{- dict "enabled" (ne $tlsSecret "") "name" $tlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
{{- if $podVols }}
{{ $podVols | toYaml | indent 8 }}
{{- end }}

5
helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
View File

@ -24,6 +24,7 @@ limitations under the License.
{{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}}
{{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}}
{{- $secretBin := index . "secretBin" -}}
{{- $tlsSecret := index . "tlsSecret" | default "" -}}
{{- $backoffLimit := index . "backoffLimit" | default "1000" -}}
{{- $activeDeadlineSeconds := index . "activeDeadlineSeconds" -}}
{{- $serviceNamePretty := $serviceName | replace "_" "-" -}}
@ -71,8 +72,9 @@ spec:
mountPath: /tmp/ks-endpoints.sh
subPath: ks-endpoints.sh
readOnly: true
{{ dict "enabled" true "name" $tlsSecret "ca" true | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
env:
{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.admin }}
{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.admin "useCA" (ne $tlsSecret "") }}
{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }}
{{- end }}
- name: OS_SVC_ENDPOINT
@ -98,4 +100,5 @@ spec:
name: {{ $configMapBin | quote }}
defaultMode: 365
{{- end }}
{{- dict "enabled" true "name" $tlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
{{- end }}

5
helm-toolkit/templates/manifests/_job-ks-service.tpl
View File

@ -24,6 +24,7 @@ limitations under the License.
{{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}}
{{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}}
{{- $secretBin := index . "secretBin" -}}
{{- $tlsSecret := index . "tlsSecret" | default "" -}}
{{- $backoffLimit := index . "backoffLimit" | default "1000" -}}
{{- $activeDeadlineSeconds := index . "activeDeadlineSeconds" -}}
{{- $serviceNamePretty := $serviceName | replace "_" "-" -}}
@ -70,8 +71,9 @@ spec:
mountPath: /tmp/ks-service.sh
subPath: ks-service.sh
readOnly: true
{{ dict "enabled" true "name" $tlsSecret "ca" true | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
env:
{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.admin }}
{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.admin "useCA" (ne $tlsSecret "") }}
{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }}
{{- end }}
- name: OS_SERVICE_NAME
@ -92,4 +94,5 @@ spec:
name: {{ $configMapBin | quote }}
defaultMode: 365
{{- end }}
{{- dict "enabled" true "name" $tlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
{{- end }}

5
helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl
View File

@ -24,6 +24,7 @@ limitations under the License.
{{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}}
{{- $serviceUser := index . "serviceUser" | default $serviceName -}}
{{- $secretBin := index . "secretBin" -}}
{{- $tlsSecret := index . "tlsSecret" | default "" -}}
{{- $backoffLimit := index . "backoffLimit" | default "1000" -}}
{{- $activeDeadlineSeconds := index . "activeDeadlineSeconds" -}}
{{- $serviceUserPretty := $serviceUser | replace "_" "-" -}}
@ -69,8 +70,9 @@ spec:
mountPath: /tmp/ks-user.sh
subPath: ks-user.sh
readOnly: true
{{ dict "enabled" true "name" $tlsSecret "ca" true | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
env:
{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.admin }}
{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.admin "useCA" (ne $tlsSecret "") }}
{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }}
{{- end }}
- name: SERVICE_OS_SERVICE_NAME
@ -98,4 +100,5 @@ spec:
name: {{ $configMapBin | quote }}
defaultMode: 365
{{- end }}
{{- dict "enabled" true "name" $tlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
{{- end -}}

2
helm-toolkit/templates/scripts/_rally_test.sh.tpl
View File

@ -66,7 +66,7 @@ cat > /tmp/rally-config.json << EOF
}
],
"https_insecure": false,
"https_cacert": ""
"https_cacert": "${OS_CACERT}"
}
}
EOF

13
helm-toolkit/templates/snippets/_keystone_openrc_env_vars.tpl
View File

@ -70,9 +70,15 @@ return: |
secretKeyRef:
name: example-keystone-admin
key: OS_PASSWORD
- name: OS_CACERT
valueFrom:
secretKeyRef:
name: example-keystone-admin
key: OS_CACERT
*/}}
{{- define "helm-toolkit.snippets.keystone_openrc_env_vars" }}
{{- $useCA := .useCA -}}
{{- $ksUserSecret := .ksUserSecret }}
- name: OS_IDENTITY_API_VERSION
value: "3"
@ -126,4 +132,11 @@ return: |
secretKeyRef:
name: {{ $ksUserSecret }}
key: OS_DEFAULT_DOMAIN
{{- if $useCA }}
- name: OS_CACERT
valueFrom:
secretKeyRef:
name: {{ $ksUserSecret }}
key: OS_CACERT
{{- end }}
{{- end }}

3
helm-toolkit/templates/snippets/_keystone_secret_openrc.tpl
View File

@ -26,4 +26,7 @@ OS_USER_DOMAIN_NAME: {{ $userContext.user_domain_name | b64enc }}
OS_USERNAME: {{ $userContext.username | b64enc }}
OS_PASSWORD: {{ $userContext.password | b64enc }}
OS_DEFAULT_DOMAIN: {{ $userContext.default_domain_id | default "default" | b64enc }}
{{- if $userContext.cacert }}
OS_CACERT: {{ $userContext.cacert | b64enc }}
{{- end }}
{{- end }}

2
helm-toolkit/templates/snippets/_keystone_user_create_env_vars.tpl
View File

@ -21,7 +21,7 @@ values: |
identity:
service_user: example-keystone-user
usage: |
{{ include "helm-toolkit.snippets.keystone_user_create_env_vars" ( dict "ksUserSecret" .Values.secrets.identity.service_user ) }}
{{ include "helm-toolkit.snippets.keystone_user_create_env_vars" ( dict "ksUserSecret" .Values.secrets.identity.service_user "useCA" true ) }}
return: |
- name: SERVICE_OS_REGION_NAME
valueFrom:

47
helm-toolkit/templates/snippets/_tls_volume.tpl Normal file
View File

@ -0,0 +1,47 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{/*
abstract: |
Renders a secret volume for tls.
Dictionary Parameters:
enabled: boolean check if you want to conditional disable this snippet (optional)
name: name of the volume (required)
secretName: name of a kuberentes/tls secret, if not specified, use the volume name (optional)
values: |
manifests:
certificates: true
usage: |
{{- $opts := dict "enabled" "true" "name" "glance-tls-api" -}}
{{- $opts | include "helm-toolkit.snippets.tls_volume" -}}
return: |
- name: glance-tls-api
secret:
secretName: glance-tls-api
defaultMode: 292
*/}}
{{- define "helm-toolkit.snippets.tls_volume" }}
{{- $enabled := index . "enabled" -}}
{{- $name := index . "name" -}}
{{- $secretName := index . "secretName" | default $name -}}
{{- if and $enabled (ne $name "") }}
- name: {{ $name }}
secret:
secretName: {{ $secretName }}
defaultMode: 292
{{- end }}
{{- end }}

82
helm-toolkit/templates/snippets/_tls_volume_mount.tpl Normal file
View File

@ -0,0 +1,82 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{/*
abstract: |
Renders a volume mount for TLS key, cert and CA.
Dictionary Parameters:
enabled: boolean check if you want to conditional disable this snippet (optional)
name: name that of the volume and should match the volume name (required)
path: path to place tls.crt tls.key ca.crt, do not suffix with '/' (required)
certs: a tuple containing a nonempty subset of {tls.crt, tls.key, ca.crt}.
the default is the full set. (optional)
values: |
manifests:
certificates: true
usage: |
{{- $opts := dict "enabled" .Values.manifests.certificates "name" "glance-tls-api" "path" "/etc/glance/certs" -}}
{{- $opts | include "helm-toolkit.snippets.tls_volume_mount" -}}
return: |
- name: glance-tls-api
mountPath: /etc/glance/certs/tls.crt
subPath: tls.crt
readOnly: true
- name: glance-tls-api
mountPath: /etc/glance/certs/tls.key
subPath: tls.key
readOnly: true
- name: glance-tls-api
mountPath: /etc/glance/certs/ca.crt
subPath: ca.crt
readOnly: true
abstract: |
This mounts a specific issuing CA only for service validation
usage: |
{{- $opts := dict "enabled" .Values.manifests.certificates "name" "glance-tls-api" "ca" true -}}
{{- $opts | include "helm-toolkit.snippets.tls_volume_mount" -}}
return: |
- name: glance-tls-api
mountPath: /etc/ssl/certs/openstack-helm.crt
subPath: ca.crt
readOnly: true
*/}}
{{- define "helm-toolkit.snippets.tls_volume_mount" }}
{{- $enabled := index . "enabled" -}}
{{- $name := index . "name" -}}
{{- $path := index . "path" | default "" -}}
{{- $certs := index . "certs" | default ( tuple "tls.crt" "tls.key" "ca.crt" ) }}
{{- if $enabled }}
{{- if and (eq $path "") (ne $name "") }}
- name: {{ $name }}
mountPath: "/etc/ssl/certs/openstack-helm.crt"
subPath: ca.crt
readOnly: true
{{- else }}
{{- if ne $name "" }}
{{- range $key, $value := $certs }}
- name: {{ $name }}
mountPath: {{ printf "%s/%s" $path $value }}
subPath: {{ $value }}
readOnly: true
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}