airship/shipyard
Code Issues Proposed changes

[Database] Shipyard DB changes

Browse Source 
- Use helm-toolkit for DB initialization [0]
- Create DB auxiliary Job for shipyard specific
  additonal DB operations
- Refactor Job dependencies

[0] https://review.openstack.org/#/c/635348/

Depends-On: https://review.openstack.org/#/c/635348/

Change-Id: I093671f9bce747b491f22dd8f38f597bd9dae9af
This commit is contained in:
Nishant Kumar 2019-02-10 21:05:00 +00:00
parent 919868e16c
commit d9f145e2a6
12 changed files with 193 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
charts/shipyard/templates/bin/_airflow-db-init.sh.tpl
View File

@ -1,50 +0,0 @@
#!/bin/bash
{{/*
Copyright 2017 The Openstack-Helm Authors.
Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
export HOME=/tmp
# Extract the DB string from shipyard.conf and get the
# value of the DB host and port
db_string=`grep -i sql_alchemy_conn ${AIRFLOW_CONFIG_FILE}`
db_fqdn=`echo ${db_string#*@} | cut -f1 -d"."`
db_port=`echo ${db_string#*@} | grep -o "[0-9]\+"`
pgsql_superuser_cmd () {
DB_COMMAND="$1"
if [[ ! -z $2 ]]; then
EXPORT PGDATABASE=$2
fi
psql \
-h $db_fqdn \
-p $db_port \
-U ${DB_ADMIN_USER} \
--command="${DB_COMMAND}"
}
# Create db
pgsql_superuser_cmd "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'" | grep -q 1 || pgsql_superuser_cmd "CREATE DATABASE $DB_NAME"
# Create db user
pgsql_superuser_cmd "SELECT * FROM pg_roles WHERE rolname = '$DB_SERVICE_USER';" | tail -n +3 | head -n -2 | grep -q 1 || \
pgsql_superuser_cmd "CREATE ROLE ${DB_SERVICE_USER} LOGIN PASSWORD '$DB_SERVICE_PASSWORD'"
# Grant permissions to user
pgsql_superuser_cmd "GRANT ALL PRIVILEGES ON DATABASE $DB_NAME to $DB_SERVICE_USER;"

46
charts/shipyard/templates/bin/_shipyard-db-auxiliary.sh.tpl Normal file
View File

@ -0,0 +1,46 @@
#!/bin/bash
{{/*
Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -e
if [[ ! -v DB_HOST ]]; then
echo "environment variable DB_HOST not set"
exit 1
elif [[ ! -v DB_ADMIN_USER ]]; then
echo "environment variable DB_ADMIN_USER not set"
exit 1
elif [[ ! -v PGPASSWORD ]]; then
echo "environment variable PGPASSWORD not set"
exit 1
elif [[ ! -v USER_DB_USER ]]; then
echo "environment variable USER_DB_USER not set"
exit 1
elif [[ ! -v DB_PORT ]]; then
echo "environment variable USER_DB_USER not set"
exit 1
elif [[ ! -v USER_DB_NAME ]]; then
echo "environment variable USER_DB_NAME not set"
exit 1
else
echo "Got DB connection info"
fi
# Grant permissions to shipyard user to the airflow dataabase tables
# This will allow shipyard user to query airflow database
/usr/bin/psql -h ${DB_HOST} -p ${DB_PORT} -U ${DB_ADMIN_USER} -d ${AIRFLOW_DB_NAME} \
--command="GRANT select, insert, update, delete on all tables in schema public to $USER_DB_USER;"

55
charts/shipyard/templates/bin/_shipyard-db-init.sh.tpl
View File

@ -1,55 +0,0 @@
#!/bin/bash
{{/*
Copyright 2017 The Openstack-Helm Authors.
Copyright (c) 2018 AT&T Intellectual Property. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
set -ex
export HOME=/tmp
# Extract the DB string from shipyard.conf and get the
# value of the DB host and port
db_string=`grep -i postgresql_db ${SHIPYARD_CONFIG_FILE}`
db_fqdn=`echo ${db_string#*@} | cut -f1 -d"."`
db_port=`echo ${db_string#*@} | grep -o "[0-9]\+"`
pgsql_superuser_cmd () {
DB_COMMAND="$1"
if [[ ! -z $2 ]]; then
EXPORT PGDATABASE=$2
fi
psql \
-h $db_fqdn \
-p $db_port \
-U ${DB_ADMIN_USER} \
--command="${DB_COMMAND}"
}
# Create db
pgsql_superuser_cmd "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'" | grep -q 1 || pgsql_superuser_cmd "CREATE DATABASE $DB_NAME"
# Create db user
pgsql_superuser_cmd "SELECT * FROM pg_roles WHERE rolname = '$DB_SERVICE_USER';" | tail -n +3 | head -n -2 | grep -q 1 || \
pgsql_superuser_cmd "CREATE ROLE ${DB_SERVICE_USER} LOGIN PASSWORD '$DB_SERVICE_PASSWORD'"
# Grant permissions to user
pgsql_superuser_cmd "GRANT ALL PRIVILEGES ON DATABASE $DB_NAME to $DB_SERVICE_USER;"
# Grant permissions to shipyard user to the airflow dataabase tables
# This will allow shipyard user to query airflow database
psql -h $db_fqdn -p $db_port -U ${DB_ADMIN_USER} -d ${AIRFLOW_DB_NAME} \
--command="GRANT select, insert, update, delete on all tables in schema public to $DB_SERVICE_USER;"

2
charts/shipyard/templates/configmap-airflow-bin.yaml
View File

@ -24,7 +24,7 @@ data:
rabbit-init.sh: |+
{{- include "helm-toolkit.scripts.rabbit_init" . | indent 4 }}
airflow-db-init.sh: |+
{{ tuple "bin/_airflow-db-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- include "helm-toolkit.scripts.pg_db_init" . | indent 4 }}
airflow-db-sync.sh: |+
{{ tuple "bin/_airflow-db-sync.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}

4
charts/shipyard/templates/configmap-shipyard-bin.yaml
View File

@ -28,7 +28,9 @@ data:
ks-domain-user.sh: |
{{- include "helm-toolkit.scripts.keystone_domain_user" . | indent 4 }}
shipyard-db-init.sh: |+
{{ tuple "bin/_shipyard-db-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- include "helm-toolkit.scripts.pg_db_init" . | indent 4 }}
shipyard-db-sync.sh: |+
{{ tuple "bin/_shipyard-db-sync.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
shipyard-db-auxiliary.sh: |+
{{ tuple "bin/_shipyard-db-auxiliary.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}

17
charts/shipyard/templates/job-airflow-db-init.yaml
View File

@ -44,24 +44,22 @@ spec:
imagePullPolicy: {{ .Values.images.pull_policy | quote }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.airflow_db_init | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
env:
- name: AIRFLOW_DB_URL
- name: DB_HOST
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: AIRFLOW_DATABASE_URI
- name: AIRFLOW_CONFIG_FILE
value: /etc/airflow/airflow.cfg
- name: DB_NAME
key: AIRFLOW_DATABASE_HOST
- name: USER_DB_NAME
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: AIRFLOW_DATABASE_NAME
- name: DB_SERVICE_USER
- name: USER_DB_USER
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: AIRFLOW_DATABASE_USERNAME
- name: DB_SERVICE_PASSWORD
- name: USER_DB_PASS
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
@ -76,6 +74,11 @@ spec:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.admin }}
key: AIRFLOW_DATABASE_PASSWORD
- name: DB_PORT
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: DATABASE_PORT
command:
- /tmp/airflow-db-init.sh
volumeMounts:

99
charts/shipyard/templates/job-shipyard-db-auxiliary.yaml Normal file
View File

@ -0,0 +1,99 @@
{{/*
Copyright (c) 2019 AT&T Intellectual Property. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if .Values.manifests.job_shipyard_db_auxiliary }}
{{- $envAll := . }}
{{- $mounts_shipyard_db_auxiliary := .Values.pod.mounts.shipyard_db_auxiliary.shipyard_db_auxiliary }}
{{- $mounts_shipyard_db_auxiliary_init := .Values.pod.mounts.shipyard_db_auxiliary.init_container }}
{{- $serviceAccountName := "shipyard-db-auxiliary" }}
{{ tuple $envAll "shipyard_db_auxiliary" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
---
apiVersion: batch/v1
kind: Job
metadata:
name: shipyard-db-auxiliary
annotations:
{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
spec:
template:
metadata:
labels:
{{ tuple $envAll "shipyard" "db-auxiliary" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
spec:
serviceAccountName: {{ $serviceAccountName }}
restartPolicy: OnFailure
nodeSelector:
{{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
initContainers:
{{ tuple $envAll "shipyard_db_auxiliary" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers:
- name: shipyard-db-auxiliary
image: {{ .Values.images.tags.shipyard_db_auxiliary | quote }}
imagePullPolicy: {{ .Values.images.pull_policy | quote }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.shipyard_db_auxiliary | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
env:
- name: DB_HOST
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_HOST
- name: USER_DB_NAME
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_NAME
- name: USER_DB_USER
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_USERNAME
- name: DB_ADMIN_USER
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.admin }}
key: SHIPYARD_DATABASE_USERNAME
- name: PGPASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.admin }}
key: SHIPYARD_DATABASE_PASSWORD
- name: AIRFLOW_DB_NAME
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: AIRFLOW_DATABASE_NAME
- name: DB_PORT
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: DATABASE_PORT
- name: SHIPYARD_AUXILIARY_CONFIG
value: "True"
command:
- /tmp/shipyard-db-auxiliary.sh
volumeMounts:
- name: shipyard-bin
mountPath: /tmp/shipyard-db-auxiliary.sh
subPath: shipyard-db-auxiliary.sh
readOnly: true
{{ if $mounts_shipyard_db_auxiliary.volumeMounts }}{{ toYaml $mounts_shipyard_db_auxiliary.volumeMounts | indent 10 }}{{ end }}
volumes:
- name: shipyard-bin
configMap:
name: shipyard-bin
defaultMode: 0555
{{ if $mounts_shipyard_db_auxiliary.volumes }}{{ toYaml $mounts_shipyard_db_auxiliary.volumes | indent 6 }}{{ end }}
{{- end }}

19
charts/shipyard/templates/job-shipyard-db-init.yaml
View File

@ -46,24 +46,22 @@ spec:
imagePullPolicy: {{ .Values.images.pull_policy | quote }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.shipyard_db_init | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
env:
- name: SHIPYARD_DB_URL
- name: DB_HOST
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_URI
- name: SHIPYARD_CONFIG_FILE
value: /etc/shipyard/shipyard.conf
- name: DB_NAME
key: SHIPYARD_DATABASE_HOST
- name: USER_DB_NAME
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_NAME
- name: DB_SERVICE_USER
- name: USER_DB_USER
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_USERNAME
- name: DB_SERVICE_PASSWORD
- name: USER_DB_PASS
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
@ -83,6 +81,13 @@ spec:
secretKeyRef:
name: {{ .Values.secrets.postgresql_airflow_db.user }}
key: AIRFLOW_DATABASE_NAME
- name: DB_PORT
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: DATABASE_PORT
- name: SHIPYARD_AUXILIARY_CONFIG
value: "True"
command:
- /tmp/shipyard-db-init.sh
volumeMounts:

20
charts/shipyard/templates/job-shipyard-db-sync.yaml
View File

@ -46,28 +46,8 @@ spec:
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.shipyard_db_sync | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
env:
- name: SHIPYARD_DB_URL
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_URI
- name: SHIPYARD_CONFIG_FILE
value: /etc/shipyard/shipyard.conf
- name: DB_NAME
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_NAME
- name: DB_SERVICE_USER
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_USERNAME
- name: DB_SERVICE_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.secrets.postgresql_shipyard_db.user }}
key: SHIPYARD_DATABASE_PASSWORD
command:
- /tmp/shipyard-db-sync.sh
volumeMounts:

3
charts/shipyard/templates/secret-airflow-db.yaml
View File

@ -35,6 +35,9 @@ data:
{{ $auth.username | b64enc | indent 4 }}
AIRFLOW_DATABASE_URI: |-
{{ tuple "postgresql_airflow_db" "internal" "user" "postgresql" $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | b64enc | indent 4 }}
AIRFLOW_DATABASE_HOST: |-
{{ tuple "postgresql_airflow_db" "internal" $envAll | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" | b64enc | indent 4 }}
DATABASE_PORT: {{ tuple "postgresql_airflow_db" "internal" "postgresql" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | b64enc }}
...
{{- end }}
{{- end }}

5
charts/shipyard/templates/secret-shipyard-db.yaml
View File

@ -33,8 +33,9 @@ data:
{{ $auth.password | b64enc | indent 4 }}
SHIPYARD_DATABASE_USERNAME: |-
{{ $auth.username | b64enc | indent 4 }}
SHIPYARD_DATABASE_URI: |-
{{ tuple "postgresql_shipyard_db" "internal" "user" "postgresql" $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | b64enc | indent 4 }}
SHIPYARD_DATABASE_HOST: |-
{{ tuple "postgresql_shipyard_db" "internal" $envAll | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" | b64enc | indent 4 }}
DATABASE_PORT: {{ tuple "postgresql_shipyard_db" "internal" "postgresql" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | b64enc }}
...
{{- end }}
{{- end }}

19
charts/shipyard/values.yaml
View File

@ -32,6 +32,7 @@ images:
shipyard: quay.io/airshipit/shipyard:latest
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
shipyard_db_init: docker.io/postgres:9.5
shipyard_db_auxiliary: docker.io/postgres:9.5
shipyard_db_sync: quay.io/airshipit/shipyard:latest
airflow_db_init: docker.io/postgres:9.5
rabbit_init: docker.io/rabbitmq:3.7-management
@ -79,9 +80,12 @@ dependencies:
services:
- service: postgresql_shipyard_db
endpoint: internal
shipyard_db_sync:
shipyard_db_auxiliary:
jobs:
- shipyard-db-init
shipyard_db_sync:
jobs:
- shipyard-db-auxiliary
services:
- service: postgresql_shipyard_db
endpoint: internal
@ -115,7 +119,6 @@ dependencies:
endpoint: internal
shipyard:
jobs:
- shipyard-db-init
- shipyard-db-sync
- shipyard-ks-endpoints
- shipyard-ks-user
@ -128,7 +131,6 @@ dependencies:
airflow_server:
jobs:
- airflow-rabbit-init
- airflow-db-init
- airflow-db-sync
services:
- service: postgresql_airflow_db
@ -691,6 +693,9 @@ pod:
shipyard_db_init:
init_container: null
shipyard_db_init:
shipyard_db_auxiliary:
init_container: null
shipyard_db_auxiliary:
shipyard_db_sync:
init_container: null
shipyard_db_sync:
@ -801,6 +806,13 @@ pod:
requests:
memory: "128Mi"
cpu: "500m"
shipyard_db_auxiliary:
limits:
memory: "128Mi"
cpu: "500m"
requests:
memory: "128Mi"
cpu: "500m"
shipyard_db_sync:
limits:
memory: "128Mi"
@ -829,6 +841,7 @@ manifests:
statefulset_airflow_worker: true
ingress_shipyard_api: true
job_shipyard_db_init: true
job_shipyard_db_auxiliary: true
job_shipyard_db_sync: true
job_rabbit_init: true
job_airflow_db_init: true