From ed9d9450ac4bd8778f7bed8168db4de2db7f9ccc Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Thu, 6 Jul 2017 22:06:39 -0500 Subject: [PATCH] Horizon: Enable Cached DB Session Backend This PS enables the Cached DB session backend, it is required both for seamless operation across multiple Horizon instances and for supporting session data larger than that supported by the memcached session backend - which is required by Murano. Change-Id: I7682198875067bb47d418ec286d9846d2e147cf4 --- horizon/templates/bin/_db-sync.sh.tpl | 19 +++++++ horizon/templates/configmap-bin.yaml | 4 ++ horizon/templates/etc/_horizon.conf.tpl | 21 ++++++-- horizon/templates/etc/_local_settings.tpl | 21 +++++--- horizon/templates/job-db-init.yaml | 62 +++++++++++++++++++++++ horizon/templates/job-db-sync.yaml | 59 +++++++++++++++++++++ horizon/templates/secret-db.yaml | 26 ++++++++++ horizon/values.yaml | 58 ++++++++++++++++++++- 8 files changed, 257 insertions(+), 13 deletions(-) create mode 100644 horizon/templates/bin/_db-sync.sh.tpl create mode 100644 horizon/templates/job-db-init.yaml create mode 100644 horizon/templates/job-db-sync.yaml create mode 100644 horizon/templates/secret-db.yaml diff --git a/horizon/templates/bin/_db-sync.sh.tpl b/horizon/templates/bin/_db-sync.sh.tpl new file mode 100644 index 0000000000..908728df9f --- /dev/null +++ b/horizon/templates/bin/_db-sync.sh.tpl @@ -0,0 +1,19 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +exec /var/lib/kolla/venv/bin/manage.py migrate diff --git a/horizon/templates/configmap-bin.yaml b/horizon/templates/configmap-bin.yaml index c3df219b4d..3aad9326ed 100644 --- a/horizon/templates/configmap-bin.yaml +++ b/horizon/templates/configmap-bin.yaml @@ -19,5 +19,9 @@ kind: ConfigMap metadata: name: horizon-bin data: + db-init.py: | +{{- include "helm-toolkit.scripts.db_init" . | indent 4 }} + db-sync.sh: | +{{ tuple "bin/_db-sync.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} horizon.sh: | {{ tuple "bin/_horizon.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/horizon/templates/etc/_horizon.conf.tpl b/horizon/templates/etc/_horizon.conf.tpl index c212b02534..a454eb5750 100644 --- a/horizon/templates/etc/_horizon.conf.tpl +++ b/horizon/templates/etc/_horizon.conf.tpl @@ -14,11 +14,14 @@ Listen 0.0.0.0:{{ .Values.network.port}} - - LogLevel warn - ErrorLog /var/log/apache2/horizon.log - CustomLog /var/log/apache2/horizon-access.log combined +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined +LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy +SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded +CustomLog /dev/stdout combined env=!forwarded +CustomLog /dev/stdout proxy env=forwarded + + WSGIScriptReloading On WSGIDaemonProcess horizon-http processes=5 threads=1 user=horizon group=horizon display-name=%{GROUP} python-path=/var/lib/kolla/venv/lib/python2.7/site-packages WSGIProcessGroup horizon-http @@ -33,5 +36,13 @@ Listen 0.0.0.0:{{ .Values.network.port}} SetHandler None - + = 2.4> + ErrorLogFormat "%{cu}t %M" + + ErrorLog /dev/stderr + + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout combined env=!forwarded + CustomLog /dev/stdout proxy env=forwarded + diff --git a/horizon/templates/etc/_local_settings.tpl b/horizon/templates/etc/_local_settings.tpl index 45b64ae6f5..f63b4ef860 100644 --- a/horizon/templates/etc/_local_settings.tpl +++ b/horizon/templates/etc/_local_settings.tpl @@ -135,18 +135,25 @@ LOCAL_PATH = '/tmp' # SECRET_KEY for all of them. SECRET_KEY='{{ .Values.local_settings.horizon_secret_key }}' -# Memcached session engine -SESSION_ENGINE = 'django.contrib.sessions.backends.cache' - -# We recommend you use memcached for development; otherwise after every reload -# of the django development server, you will have to login again. To use -# memcached set CACHES to something like CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', - 'LOCATION': '{{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}' + 'LOCATION': '{{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}', } } +DATABASES = { + 'default': { + # Database configuration here + 'ENGINE': 'django.db.backends.mysql', + 'NAME': '{{ .Values.endpoints.oslo_db.path | base }}', + 'USER': '{{ .Values.endpoints.oslo_db.auth.user.username }}', + 'PASSWORD': '{{ .Values.endpoints.oslo_db.auth.user.password }}', + 'HOST': '{{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}', + 'default-character-set': 'utf8', + 'PORT': '{{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}' + } +} +SESSION_ENGINE = 'django.contrib.sessions.backends.cached_db' # Send email to the console by default EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' diff --git a/horizon/templates/job-db-init.yaml b/horizon/templates/job-db-init.yaml new file mode 100644 index 0000000000..8d9a09a73b --- /dev/null +++ b/horizon/templates/job-db-init.yaml @@ -0,0 +1,62 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.db_init }} +{{- $mounts_horizon_db_init := .Values.mounts.horizon_db_init.horizon_db_init }} +{{- $mounts_horizon_db_init_init := .Values.mounts.horizon_db_init.init_container }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: horizon-db-init +spec: + template: + metadata: + spec: + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + initContainers: +{{ tuple $envAll $dependencies $mounts_horizon_db_init_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: horizon-db-init + image: {{ .Values.images.db_init }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.resources.jobs.db_init | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: ROOT_DB_CONNECTION + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.oslo_db.admin }} + key: DB_CONNECTION + - name: DB_CONNECTION + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.oslo_db.user }} + key: DB_CONNECTION + command: + - /tmp/db-init.py + volumeMounts: + - name: horizon-bin + mountPath: /tmp/db-init.py + subPath: db-init.py + readOnly: true +{{- if $mounts_horizon_db_init.volumeMounts }}{{ toYaml $mounts_horizon_db_init.volumeMounts | indent 10 }}{{ end }} + volumes: + - name: horizon-bin + configMap: + name: horizon-bin + defaultMode: 0555 +{{- if $mounts_horizon_db_init.volumes }}{{ toYaml $mounts_horizon_db_init.volumes | indent 6 }}{{ end }} diff --git a/horizon/templates/job-db-sync.yaml b/horizon/templates/job-db-sync.yaml new file mode 100644 index 0000000000..ffb15dd4b1 --- /dev/null +++ b/horizon/templates/job-db-sync.yaml @@ -0,0 +1,59 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.db_sync }} +{{- $mounts_horizon_db_sync := .Values.mounts.horizon_db_sync.horizon_db_sync }} +{{- $mounts_horizon_db_sync_init := .Values.mounts.horizon_db_sync.init_container }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: horizon-db-sync +spec: + template: + metadata: + annotations: + spec: + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + initContainers: +{{ tuple $envAll $dependencies $mounts_horizon_db_sync_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: horizon-db-sync + image: {{ .Values.images.db_sync }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.resources.jobs.db_sync | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/db-sync.sh + volumeMounts: + - name: horizon-etc + mountPath: /etc/openstack-dashboard/local_settings + subPath: local_settings + readOnly: true + - name: horizon-bin + mountPath: /tmp/db-sync.sh + subPath: db-sync.sh + readOnly: true +{{- if $mounts_horizon_db_sync.volumeMounts }}{{ toYaml $mounts_horizon_db_sync.volumeMounts | indent 10 }}{{ end }} + volumes: + - name: horizon-etc + configMap: + name: horizon-etc + - name: horizon-bin + configMap: + name: horizon-bin + defaultMode: 0555 +{{- if $mounts_horizon_db_sync.volumes }}{{ toYaml $mounts_horizon_db_sync.volumes | indent 6 }}{{ end }} diff --git a/horizon/templates/secret-db.yaml b/horizon/templates/secret-db.yaml new file mode 100644 index 0000000000..4b0456f499 --- /dev/null +++ b/horizon/templates/secret-db.yaml @@ -0,0 +1,26 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- $envAll := . }} +{{- range $key1, $userClass := tuple "admin" "user" }} +{{- $secretName := index $envAll.Values.secrets.oslo_db $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + DB_CONNECTION: {{ tuple "oslo_db" "internal" $userClass "mysql" $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | b64enc -}} +{{- end }} diff --git a/horizon/values.yaml b/horizon/values.yaml index c33a012210..dff08d6718 100644 --- a/horizon/values.yaml +++ b/horizon/values.yaml @@ -20,6 +20,8 @@ replicas: 1 images: + db_init: docker.io/kolla/ubuntu-source-horizon:4.0.0 + db_sync: docker.io/kolla/ubuntu-source-horizon:4.0.0 horizon: docker.io/kolla/ubuntu-source-horizon:4.0.0 dep_check: docker.io/kolla/ubuntu-source-kubernetes-entrypoint:4.0.0 pull_policy: "IfNotPresent" @@ -57,10 +59,22 @@ local_settings: dependencies: dashboard: + jobs: + - horizon-db-sync services: - service: oslo_cache endpoint: internal - - service: identity + - service: oslo_db + endpoint: internal + db_init: + services: + - service: oslo_db + endpoint: internal + db_sync: + jobs: + - horizon-db-init + services: + - service: oslo_db endpoint: internal resources: @@ -72,6 +86,27 @@ resources: limits: memory: "1204Mi" cpu: "2000m" + jobs: + db_init: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + db_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +# Names of secrets used by bootstrap and environmental checks +secrets: + oslo_db: + admin: horizon-db-admin + user: horizon-db-user # typically overriden by environmental # values, but should include all endpoints @@ -109,8 +144,29 @@ endpoints: port: web: default: 80 + oslo_db: + auth: + admin: + username: root + password: password + user: + username: horizon + password: password + hosts: + default: mariadb + path: /horizon + scheme: mysql+pymysql + port: + mysql: + default: 3306 mounts: + horizon_db_init: + init_container: null + horizon_db_init: + horizon_db_sync: + init_container: null + horizon_db_sync: horizon: init_container: null horizon: