diff --git a/Makefile b/Makefile index fafc4d58..81abef2b 100644 --- a/Makefile +++ b/Makefile @@ -12,17 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -BUILD_CTX ?= src/bin -DOCKER_REGISTRY ?= quay.io -IMAGE_PREFIX ?= airshipit -IMAGE_TAG ?= untagged -HELM ?= helm -LABEL ?= commit-id -IMAGE_NAME := airflow shipyard -PROXY ?= http://proxy.foo.com:8000 -NO_PROXY ?= localhost,127.0.0.1,.svc.cluster.local -USE_PROXY ?= false -PUSH_IMAGE ?= false +BUILD_DIR := $(shell mktemp -d) +BUILD_CTX ?= src/bin + +IMAGE_PREFIX ?= airshipit +IMAGE_TAG ?= untagged +IMAGE_NAME := airflow shipyard +LABEL ?= commit-id + +DOCKER_REGISTRY ?= quay.io +PUSH_IMAGE ?= false + +HELM := $(BUILD_DIR)/helm + +PROXY ?= http://proxy.foo.com:8000 +NO_PROXY ?= localhost,127.0.0.1,.svc.cluster.local +USE_PROXY ?= false + IMAGE:=${DOCKER_REGISTRY}/${IMAGE_PREFIX}/$(IMAGE_NAME):${IMAGE_TAG} IMAGE_DIR:=images/$(IMAGE_NAME) @@ -43,8 +49,7 @@ $(IMAGE_NAME): # Create tgz of the chart .PHONY: charts -charts: clean - tools/helm_tk.sh $(HELM) +charts: clean helm-init $(HELM) dep up charts/shipyard $(HELM) package charts/shipyard @@ -54,8 +59,7 @@ lint: pep8 helm_lint build_docs # Dry run templating of chart .PHONY: dry-run -dry-run: clean - tools/helm_tk.sh $(HELM) +dry-run: clean helm-init $(HELM) template charts/shipyard .PHONY: docs @@ -114,6 +118,9 @@ endif .PHONY: clean clean: + rm -rf $(BUILD_DIR)/* + rm -rf build + rm -rf docs/build cd $(BUILD_CTX)/shipyard_airflow; rm -rf build cd $(BUILD_CTX)/shipyard_client; rm -rf build @@ -123,10 +130,19 @@ pep8: cd $(BUILD_CTX)/shipyard_client; tox -e pep8 .PHONY: helm_lint -helm_lint: clean - tools/helm_tk.sh $(HELM) +helm_lint: clean helm-init $(HELM) lint charts/shipyard +# Initialize local helm config +.PHONY: helm-init +helm-init: helm-install + tools/helm_tk.sh $(HELM) + +# Install helm binary +.PHONY: helm-install +helm-install: + tools/helm_install.sh $(HELM) + .PHONY: build_docs build_docs: tox -e docs diff --git a/charts/shipyard/templates/configmap-shipyard-etc.yaml b/charts/shipyard/templates/configmap-shipyard-etc.yaml index 7969db34..f068d259 100644 --- a/charts/shipyard/templates/configmap-shipyard-etc.yaml +++ b/charts/shipyard/templates/configmap-shipyard-etc.yaml @@ -43,7 +43,7 @@ {{- tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" | set .Values.conf.shipyard.keystone_authtoken "memcached_servers" | quote | trunc 0 -}} {{- end -}} -{{- $userIdentity := .Values.endpoints.identity.auth.user -}} +{{- $userIdentity := .Values.endpoints.identity.auth.shipyard -}} {{- if empty .Values.conf.shipyard.keystone_authtoken.project_name -}} {{- set .Values.conf.shipyard.keystone_authtoken "project_name" $userIdentity.project_name | quote | trunc 0 -}} diff --git a/charts/shipyard/templates/deployment-airflow-flower.yaml b/charts/shipyard/templates/deployment-airflow-flower.yaml index 801fb7b6..283ac59e 100644 --- a/charts/shipyard/templates/deployment-airflow-flower.yaml +++ b/charts/shipyard/templates/deployment-airflow-flower.yaml @@ -15,9 +15,8 @@ {{- if .Values.manifests.deployment_airflow_flower }} {{- $envAll := . }} -{{- $dependencies := .Values.dependencies.airflow_server }} {{- $serviceAccountName := "airflow-flower" }} -{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +{{ tuple $envAll "airflow_server" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} {{- $mounts_airflow_flower := .Values.pod.mounts.airflow_flower.airflow_flower }} {{- $mounts_airflow_flower_init := .Values.pod.mounts.airflow_flower.init_container }} --- @@ -38,11 +37,11 @@ spec: spec: serviceAccountName: {{ $serviceAccountName }} nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} restartPolicy: Always terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }} initContainers: -{{ tuple $envAll $dependencies $mounts_airflow_flower_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ tuple $envAll "airflow_server" $mounts_airflow_flower_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - name: airflow-flower image: {{ .Values.images.tags.airflow }} diff --git a/charts/shipyard/templates/deployment-airflow-scheduler.yaml b/charts/shipyard/templates/deployment-airflow-scheduler.yaml index ed88b6ed..636dac11 100644 --- a/charts/shipyard/templates/deployment-airflow-scheduler.yaml +++ b/charts/shipyard/templates/deployment-airflow-scheduler.yaml @@ -15,9 +15,8 @@ {{- if .Values.manifests.deployment_airflow_scheduler }} {{- $envAll := . }} -{{- $dependencies := .Values.dependencies.airflow_server }} {{- $serviceAccountName := "airflow-scheduler" }} -{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +{{ tuple $envAll "airflow_server" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} {{- $mounts_airflow_scheduler := .Values.pod.mounts.airflow_scheduler.airflow_scheduler }} {{- $mounts_airflow_scheduler_init := .Values.pod.mounts.airflow_scheduler.init_container }} --- @@ -38,11 +37,11 @@ spec: spec: serviceAccountName: {{ $serviceAccountName }} nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} restartPolicy: Always terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }} initContainers: -{{ tuple $envAll $dependencies $mounts_airflow_scheduler_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ tuple $envAll "airflow_server" $mounts_airflow_scheduler_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} {{- if .Values.prod_environment }} - name: airflow-shipyard-init image: {{ .Values.images.tags.shipyard }} diff --git a/charts/shipyard/templates/deployment-airflow-web.yaml b/charts/shipyard/templates/deployment-airflow-web.yaml index 05415247..2e96587a 100644 --- a/charts/shipyard/templates/deployment-airflow-web.yaml +++ b/charts/shipyard/templates/deployment-airflow-web.yaml @@ -15,9 +15,8 @@ {{- if .Values.manifests.deployment_airflow_web }} {{- $envAll := . }} -{{- $dependencies := .Values.dependencies.airflow_server }} {{- $serviceAccountName := "airflow-web" }} -{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +{{ tuple $envAll "airflow_server" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} {{- $mounts_airflow_web := .Values.pod.mounts.airflow_web.airflow_web }} {{- $mounts_airflow_web_init := .Values.pod.mounts.airflow_web.init_container }} --- @@ -38,11 +37,11 @@ spec: spec: serviceAccountName: {{ $serviceAccountName }} nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} restartPolicy: Always terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.airflow.timeout | default "30" }} initContainers: -{{ tuple $envAll $dependencies $mounts_airflow_web_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ tuple $envAll "airflow_server" $mounts_airflow_web_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} {{- if .Values.prod_environment }} - name: airflow-shipyard-init image: {{ .Values.images.tags.shipyard }} diff --git a/charts/shipyard/templates/deployment-shipyard.yaml b/charts/shipyard/templates/deployment-shipyard.yaml index 94a14f83..527e5ca2 100644 --- a/charts/shipyard/templates/deployment-shipyard.yaml +++ b/charts/shipyard/templates/deployment-shipyard.yaml @@ -15,11 +15,10 @@ {{- if .Values.manifests.deployment_shipyard }} {{- $envAll := . }} -{{- $dependencies := .Values.dependencies.shipyard }} {{- $mounts_shipyard := .Values.pod.mounts.shipyard.shipyard }} {{- $mounts_shipyard_init := .Values.pod.mounts.shipyard.init_container }} {{- $serviceAccountName := "shipyard" }} -{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +{{ tuple $envAll "shipyard" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- apiVersion: apps/v1beta1 kind: Deployment @@ -38,11 +37,11 @@ spec: spec: serviceAccountName: {{ $serviceAccountName }} nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.shipyard.timeout | default "30" }} restartPolicy: Always initContainers: -{{ tuple $envAll $dependencies $mounts_shipyard_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ tuple $envAll "shipyard" $mounts_shipyard_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - name: shipyard-api env: diff --git a/charts/shipyard/templates/job-airflow-db-init.yaml b/charts/shipyard/templates/job-airflow-db-init.yaml index d4ec109e..709f1ba5 100644 --- a/charts/shipyard/templates/job-airflow-db-init.yaml +++ b/charts/shipyard/templates/job-airflow-db-init.yaml @@ -17,9 +17,8 @@ limitations under the License. {{- if .Values.manifests.job_airflow_db_init }} {{- $envAll := . }} -{{- $dependencies := .Values.dependencies.airflow_db_init }} {{- $serviceAccountName := "airflow-db-init" }} -{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +{{ tuple $envAll "airflow_db_init" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- apiVersion: batch/v1 kind: Job @@ -34,9 +33,9 @@ spec: serviceAccountName: {{ $serviceAccountName }} restartPolicy: OnFailure nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} initContainers: -{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ tuple $envAll "airflow_db_init" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - name: airflow-db-init image: {{ .Values.images.tags.airflow_db_init | quote }} diff --git a/charts/shipyard/templates/job-airflow-db-sync.yaml b/charts/shipyard/templates/job-airflow-db-sync.yaml index ea2653fd..87b60e24 100644 --- a/charts/shipyard/templates/job-airflow-db-sync.yaml +++ b/charts/shipyard/templates/job-airflow-db-sync.yaml @@ -17,9 +17,8 @@ limitations under the License. {{- if .Values.manifests.job_airflow_db_sync }} {{- $envAll := . }} -{{- $dependencies := .Values.dependencies.airflow_db_sync }} {{- $serviceAccountName := "airflow-db-sync" }} -{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +{{ tuple $envAll "airflow_db_sync" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- apiVersion: batch/v1 kind: Job @@ -34,9 +33,9 @@ spec: serviceAccountName: {{ $serviceAccountName }} restartPolicy: OnFailure nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} initContainers: -{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ tuple $envAll "airflow_db_sync" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - name: airflow-db-sync image: {{ .Values.images.tags.airflow_db_sync }} diff --git a/charts/shipyard/templates/job-ks-endpoints.yaml b/charts/shipyard/templates/job-ks-endpoints.yaml index db51f465..e6796c3c 100644 --- a/charts/shipyard/templates/job-ks-endpoints.yaml +++ b/charts/shipyard/templates/job-ks-endpoints.yaml @@ -13,58 +13,6 @@ # limitations under the License. {{- if .Values.manifests.job_ks_endpoints }} -{{- $envAll := . }} -{{- $dependencies := .Values.dependencies.ks_endpoints }} -{{- $serviceAccountName := "shipyard-ks-endpoints" }} -{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: shipyard-ks-endpoints -spec: - template: - metadata: - labels: -{{ tuple $envAll "shipyard" "ks-endpoints" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} - spec: - serviceAccountName: {{ $serviceAccountName }} - restartPolicy: OnFailure - nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} - initContainers: -{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} - containers: -{{- range $key1, $osServiceType := tuple "shipyard" }} -{{- range $key2, $osServiceEndPoint := tuple "admin" "internal" "public" }} - - name: {{ $osServiceType }}-ks-endpoints-{{ $osServiceEndPoint }} - image: {{ $envAll.Values.images.tags.ks_endpoints }} - imagePullPolicy: {{ $envAll.Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.jobs.ks_endpoints | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - command: - - /tmp/ks-endpoints.sh - volumeMounts: - - name: ks-endpoints-sh - mountPath: /tmp/ks-endpoints.sh - subPath: ks-endpoints.sh - readOnly: true - env: -{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.admin }} -{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }} -{{- end }} - - name: OS_SVC_ENDPOINT - value: {{ $osServiceEndPoint }} - - name: OS_SERVICE_NAME - value: {{ tuple $osServiceType $envAll | include "helm-toolkit.endpoints.keystone_endpoint_name_lookup" }} - - name: OS_SERVICE_TYPE - value: {{ $osServiceType }} - - name: OS_SERVICE_ENDPOINT - value: {{ tuple $osServiceType $osServiceEndPoint "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }} -{{- end }} -{{- end }} - volumes: - - name: ks-endpoints-sh - configMap: - name: shipyard-bin - defaultMode: 0555 +{{- $ksServiceJob := dict "envAll" . "serviceName" "shipyard" "serviceTypes" ( tuple "shipyard" ) -}} +{{ $ksServiceJob | include "helm-toolkit.manifests.job_ks_endpoints" }} {{- end -}} diff --git a/charts/shipyard/templates/job-ks-service.yaml b/charts/shipyard/templates/job-ks-service.yaml index 62906b30..6f9c4508 100644 --- a/charts/shipyard/templates/job-ks-service.yaml +++ b/charts/shipyard/templates/job-ks-service.yaml @@ -13,54 +13,6 @@ # limitations under the License. {{- if .Values.manifests.job_ks_service -}} - -{{- $envAll := . }} -{{- $ksAdminSecret := .Values.secrets.identity.admin }} -{{- $dependencies := .Values.dependencies.ks_service }} -{{- $serviceAccountName := "shipyard-ks-service" }} -{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: shipyard-ks-service -spec: - template: - metadata: - labels: -{{ tuple $envAll "shipyard" "ks-service" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} - spec: - serviceAccountName: {{ $serviceAccountName }} - restartPolicy: OnFailure - nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} - initContainers: -{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} - containers: -{{- range $key1, $osServiceType := tuple "shipyard" }} - - name: {{ $osServiceType }}-ks-service-registration - image: {{ $envAll.Values.images.tags.ks_service }} - imagePullPolicy: {{ $envAll.Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.jobs.ks_service | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - command: - - /tmp/ks-service.sh - volumeMounts: - - name: ks-service-sh - mountPath: /tmp/ks-service.sh - subPath: ks-service.sh - readOnly: true - env: -{{- with $env := dict "ksUserSecret" $ksAdminSecret }} -{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }} -{{- end }} - - name: OS_SERVICE_NAME - value: {{ tuple $osServiceType $envAll | include "helm-toolkit.endpoints.keystone_endpoint_name_lookup" }} - - name: OS_SERVICE_TYPE - value: {{ $osServiceType }} -{{- end }} - volumes: - - name: ks-service-sh - configMap: - name: shipyard-bin - defaultMode: 0555 +{{- $ksServiceJob := dict "envAll" . "serviceName" "shipyard" "serviceTypes" ( tuple "shipyard" ) -}} +{{ $ksServiceJob | include "helm-toolkit.manifests.job_ks_service" }} {{- end -}} diff --git a/charts/shipyard/templates/job-ks-user.yaml b/charts/shipyard/templates/job-ks-user.yaml index 3d1cb583..1bb6d12b 100644 --- a/charts/shipyard/templates/job-ks-user.yaml +++ b/charts/shipyard/templates/job-ks-user.yaml @@ -13,58 +13,6 @@ # limitations under the License. {{- if .Values.manifests.job_ks_user }} - -{{- $ksAdminSecret := .Values.secrets.identity.admin }} -{{- $ksUserSecret := .Values.secrets.identity.user }} -{{- $envAll := . }} -{{- $dependencies := .Values.dependencies.ks_user }} -{{- $serviceAccountName := "shipyard-ks-user" }} -{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: shipyard-ks-user -spec: - template: - metadata: - labels: -{{ tuple $envAll "shipyard" "ks-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} - spec: - serviceAccountName: {{ $serviceAccountName }} - restartPolicy: OnFailure - nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} - initContainers: -{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} - containers: - - name: shipyard-ks-user - image: {{ .Values.images.tags.ks_user }} - imagePullPolicy: {{ .Values.images.pull_policy }} -{{ tuple $envAll $envAll.Values.pod.resources.jobs.ks_user | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - command: - - /tmp/ks-user.sh - volumeMounts: - - name: ks-user-sh - mountPath: /tmp/ks-user.sh - subPath: ks-user.sh - readOnly: true - env: -{{- with $env := dict "ksUserSecret" $ksAdminSecret }} -{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }} -{{- end }} - - name: SERVICE_OS_SERVICE_NAME - value: {{ $envAll.Values.endpoints.shipyard.name | quote }} - - name: SERVICE_OS_DOMAIN_NAME - value: {{ $envAll.Values.endpoints.identity.auth.user.project_domain_name | quote }} -{{- with $env := dict "ksUserSecret" $ksUserSecret }} -{{- include "helm-toolkit.snippets.keystone_user_create_env_vars" $env | indent 12 }} -{{- end }} - - name: SERVICE_OS_ROLE - value: {{ $envAll.Values.endpoints.identity.auth.user.role | quote }} - volumes: - - name: ks-user-sh - configMap: - name: shipyard-bin - defaultMode: 0555 +{{- $ksUserJob := dict "envAll" . "serviceName" "shipyard" -}} +{{ $ksUserJob | include "helm-toolkit.manifests.job_ks_user" }} {{- end -}} diff --git a/charts/shipyard/templates/job-shipyard-db-init.yaml b/charts/shipyard/templates/job-shipyard-db-init.yaml index a13a18ee..1c8704f3 100644 --- a/charts/shipyard/templates/job-shipyard-db-init.yaml +++ b/charts/shipyard/templates/job-shipyard-db-init.yaml @@ -17,11 +17,10 @@ limitations under the License. {{- if .Values.manifests.job_shipyard_db_init }} {{- $envAll := . }} -{{- $dependencies := .Values.dependencies.shipyard_db_init }} {{- $mounts_shipyard_db_init := .Values.pod.mounts.shipyard_db_init.shipyard_db_init }} {{- $mounts_shipyard_db_init_init := .Values.pod.mounts.shipyard_db_init.init_container }} {{- $serviceAccountName := "shipyard-db-init" }} -{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +{{ tuple $envAll "shipyard_db_init" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- apiVersion: batch/v1 kind: Job @@ -36,9 +35,9 @@ spec: serviceAccountName: {{ $serviceAccountName }} restartPolicy: OnFailure nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} initContainers: -{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ tuple $envAll "shipyard_db_init" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - name: shipyard-db-init image: {{ .Values.images.tags.shipyard_db_init | quote }} diff --git a/charts/shipyard/templates/job-shipyard-db-sync.yaml b/charts/shipyard/templates/job-shipyard-db-sync.yaml index 45a18987..437bd1b4 100644 --- a/charts/shipyard/templates/job-shipyard-db-sync.yaml +++ b/charts/shipyard/templates/job-shipyard-db-sync.yaml @@ -17,11 +17,10 @@ limitations under the License. {{- if .Values.manifests.job_shipyard_db_sync }} {{- $envAll := . }} -{{- $dependencies := .Values.dependencies.shipyard_db_sync }} {{- $mounts_shipyard_db_sync := .Values.pod.mounts.shipyard_db_sync.shipyard_db_sync }} {{- $mounts_shipyard_db_sync_init := .Values.pod.mounts.shipyard_db_sync.init_container }} {{- $serviceAccountName := "shipyard-db-sync" }} -{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +{{ tuple $envAll "shipyard_db_sync" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- apiVersion: batch/v1 kind: Job @@ -36,9 +35,9 @@ spec: serviceAccountName: {{ $serviceAccountName }} restartPolicy: OnFailure nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} initContainers: -{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ tuple $envAll "shipyard_db_sync" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - name: shipyard-db-sync image: {{ .Values.images.tags.shipyard_db_sync }} diff --git a/charts/shipyard/templates/secret-keystone-env.yaml b/charts/shipyard/templates/secret-keystone-env.yaml index 16dffe39..676d3d9a 100644 --- a/charts/shipyard/templates/secret-keystone-env.yaml +++ b/charts/shipyard/templates/secret-keystone-env.yaml @@ -15,7 +15,7 @@ limitations under the License. */}} {{- if .Values.manifests.secret_keystone }} {{- $envAll := . }} -{{- range $key1, $userClass := tuple "admin" "user" }} +{{- range $key1, $userClass := tuple "admin" "shipyard" }} {{- $secretName := index $envAll.Values.secrets.identity $userClass }} --- apiVersion: v1 diff --git a/charts/shipyard/templates/statefulset-airflow-worker.yaml b/charts/shipyard/templates/statefulset-airflow-worker.yaml index db99110f..bfa59e6f 100644 --- a/charts/shipyard/templates/statefulset-airflow-worker.yaml +++ b/charts/shipyard/templates/statefulset-airflow-worker.yaml @@ -14,10 +14,9 @@ {{- if .Values.manifests.statefulset_airflow_worker }} {{- $envAll := . }} -{{- $dependencies := .Values.dependencies.airflow_server }} {{- $serviceAccountName := "airflow-worker" }} -{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +{{ tuple $envAll "airflow_server" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole @@ -75,9 +74,9 @@ spec: spec: serviceAccountName: {{ $serviceAccountName }} nodeSelector: - {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} initContainers: -{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ tuple $envAll "airflow_server" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} {{- if .Values.prod_environment }} - name: airflow-shipyard-init image: {{ .Values.images.tags.shipyard }} diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index dead851d..fffb50f4 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -19,8 +19,9 @@ prod_environment: true labels: - node_selector_key: ucp-control-plane - node_selector_value: enabled + job: + node_selector_key: ucp-control-plane + node_selector_value: enabled images: tags: @@ -34,7 +35,13 @@ images: ks_user: docker.io/openstackhelm/heat:ocata ks_service: docker.io/openstackhelm/heat:ocata ks_endpoints: docker.io/openstackhelm/heat:ocata + image_repo_sync: docker.io/docker:17.07.0 pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync release_group: null @@ -69,72 +76,73 @@ network: enable_node_port: false dependencies: - shipyard_db_init: - jobs: - - airflow-db-init - - airflow-db-sync - services: - - service: postgresql_shipyard_db - endpoint: internal - - service: airflow_flower - endpoint: internal - - service: airflow_web - endpoint: internal - shipyard_db_sync: - jobs: - - shipyard-db-init - services: - - service: postgresql_shipyard_db - endpoint: internal - airflow_db_init: - services: - - service: postgresql_airflow_db - endpoint: internal - airflow_db_sync: - jobs: - - airflow-db-init - services: - - service: postgresql_airflow_db - endpoint: internal - ks_user: - services: - - service: identity - endpoint: internal - ks_service: - services: - - service: identity - endpoint: internal - ks_endpoints: - jobs: - - shipyard-ks-service - services: - - service: identity - endpoint: internal - shipyard: - jobs: - - shipyard-db-init - - shipyard-db-sync - - shipyard-ks-endpoints - - shipyard-ks-user - - shipyard-ks-endpoints - services: - - service: airflow_flower - endpoint: internal - - service: airflow_web - endpoint: internal - - service: identity - endpoint: internal - - service: postgresql_shipyard_db - endpoint: internal - airflow_server: - jobs: - - airflow-db-init - - airflow-db-sync - services: - - service: postgresql_airflow_db - endpoint: internal - - service: oslo_messaging - endpoint: internal + static: + shipyard_db_init: + jobs: + - airflow-db-init + - airflow-db-sync + services: + - service: postgresql_shipyard_db + endpoint: internal + - service: airflow_flower + endpoint: internal + - service: airflow_web + endpoint: internal + shipyard_db_sync: + jobs: + - shipyard-db-init + services: + - service: postgresql_shipyard_db + endpoint: internal + airflow_db_init: + services: + - service: postgresql_airflow_db + endpoint: internal + airflow_db_sync: + jobs: + - airflow-db-init + services: + - service: postgresql_airflow_db + endpoint: internal + ks_user: + services: + - service: identity + endpoint: internal + ks_service: + services: + - service: identity + endpoint: internal + ks_endpoints: + jobs: + - shipyard-ks-service + services: + - service: identity + endpoint: internal + shipyard: + jobs: + - shipyard-db-init + - shipyard-db-sync + - shipyard-ks-endpoints + - shipyard-ks-user + - shipyard-ks-endpoints + services: + - service: airflow_flower + endpoint: internal + - service: airflow_web + endpoint: internal + - service: identity + endpoint: internal + - service: postgresql_shipyard_db + endpoint: internal + airflow_server: + jobs: + - airflow-db-init + - airflow-db-sync + services: + - service: postgresql_airflow_db + endpoint: internal + - service: oslo_messaging + endpoint: internal volume_worker: class_name: general @@ -151,7 +159,7 @@ endpoints: identity: name: keystone auth: - user: + shipyard: region_name: RegionOne role: admin project_name: service @@ -318,7 +326,7 @@ endpoints: secrets: identity: admin: shipyard-keystone-admin - user: shipyard-keystone-user + shipyard: shipyard-keystone-user postgresql_shipyard_db: admin: shipyard-db-admin user: shipyard-db-user diff --git a/tools/helm_install.sh b/tools/helm_install.sh new file mode 100755 index 00000000..15f1cf9b --- /dev/null +++ b/tools/helm_install.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -x + +HELM=$1 +HELM_ARTIFACT_URL=${HELM_ARTIFACT_URL:-"https://storage.googleapis.com/kubernetes-helm/helm-v2.7.2-linux-amd64.tar.gz"} + + +function install_helm_binary { + if [[ -z "${HELM}" ]] + then + echo "No Helm binary target location." + exit -1 + fi + + if [[ -w "$(dirname ${HELM})" ]] + then + TMP_DIR=${BUILD_DIR:-$(mktemp -d)} + curl -o "${TMP_DIR}/helm.tar.gz" "${HELM_ARTIFACT_URL}" + cd ${TMP_DIR} + tar -xvzf helm.tar.gz + cp "${TMP_DIR}/linux-amd64/helm" "${HELM}" + else + echo "Cannot write to ${HELM}" + exit -1 + fi +} + +install_helm_binary diff --git a/tools/helm_tk.sh b/tools/helm_tk.sh index 4cea01be..b4c11680 100755 --- a/tools/helm_tk.sh +++ b/tools/helm_tk.sh @@ -48,5 +48,11 @@ git clone --depth 1 https://git.openstack.org/openstack/openstack-helm-infra.git cd openstack-helm-infra git pull helm_serve + +if [[ ${HELM} != "helm" ]] +then + export PATH=${PATH}:$(dirname ${HELM}) +fi + make helm-toolkit ${HELM} dep up ../../charts/shipyard