From 45ba95a2de6f9ba524f527c0b0398eee8848dfa2 Mon Sep 17 00:00:00 2001 From: Steve Wilkerson Date: Tue, 12 Dec 2017 16:54:23 -0600 Subject: [PATCH] Elasticsearch: Add curator snapshot action and PVC for fs repo This provides an example action in the curator config for taking snapshots of the elasticsearch indexes. As the snapshot action requires a repository registered with Elasticsearch, this also adds a PVC for a filesystem repository backed with NFS and a job for registering the repository with Elasticsearch. Change-Id: I26b788c58f52844e997bde5002459bddc1bb685e --- .../_register-repository.sh.tpl} | 17 ++-- elasticsearch/templates/configmap-bin.yaml | 2 + elasticsearch/templates/cron-job-curator.yaml | 4 +- .../templates/deployment-client.yaml | 13 ++++ .../templates/deployment-master.yaml | 13 ++++ .../templates/etc/_elasticsearch.yml.tpl | 1 + .../templates/job-image-repo-sync.yaml | 6 +- .../job-register-snapshot-repository.yaml | 65 ++++++++++++++++ elasticsearch/templates/pvc-snapshots.yaml | 33 ++++++++ elasticsearch/templates/statefulset-data.yaml | 21 ++++- elasticsearch/values.yaml | 78 ++++++++++++++++--- tools/gate/chart-deploys/default.yaml | 5 +- 12 files changed, 231 insertions(+), 27 deletions(-) rename elasticsearch/templates/{serviceaccount.yaml => bin/_register-repository.sh.tpl} (70%) create mode 100644 elasticsearch/templates/job-register-snapshot-repository.yaml create mode 100644 elasticsearch/templates/pvc-snapshots.yaml diff --git a/elasticsearch/templates/serviceaccount.yaml b/elasticsearch/templates/bin/_register-repository.sh.tpl similarity index 70% rename from elasticsearch/templates/serviceaccount.yaml rename to elasticsearch/templates/bin/_register-repository.sh.tpl index 1579d19b4..5c19083ff 100644 --- a/elasticsearch/templates/serviceaccount.yaml +++ b/elasticsearch/templates/bin/_register-repository.sh.tpl @@ -1,3 +1,4 @@ +#!/bin/bash {{/* Copyright 2017 The Openstack-Helm Authors. @@ -14,9 +15,13 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if .Values.manifests.serviceaccount }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: elasticsearch -{{- end }} +set -ex + +exec curl -X PUT "${ELASTICSEARCH_ENDPOINT}/_snapshot/${REPO_NAME}" -H 'Content-Type: application/json' -d' +{ + "type": "'"$REPO_TYPE"'", + "settings": { + "location": "'"$REPO_LOCATION"'", + "compress": true + } +}' diff --git a/elasticsearch/templates/configmap-bin.yaml b/elasticsearch/templates/configmap-bin.yaml index 22b2a6cd1..25a6c6d31 100644 --- a/elasticsearch/templates/configmap-bin.yaml +++ b/elasticsearch/templates/configmap-bin.yaml @@ -26,6 +26,8 @@ data: {{ tuple "bin/_elasticsearch.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} helm-tests.sh: | {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + register-repository.sh: | +{{ tuple "bin/_register-repository.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} curator.sh: | {{ tuple "bin/_curator.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} image-repo-sync.sh: |+ diff --git a/elasticsearch/templates/cron-job-curator.yaml b/elasticsearch/templates/cron-job-curator.yaml index eedf397e3..ea5931ac1 100644 --- a/elasticsearch/templates/cron-job-curator.yaml +++ b/elasticsearch/templates/cron-job-curator.yaml @@ -19,13 +19,13 @@ limitations under the License. {{- $envAll := . }} {{- $_ := set .Values "pod_dependency" .Values.dependencies.curator -}} -{{- $serviceAccountName := "curator"}} +{{- $serviceAccountName := "elastic-curator"}} {{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- apiVersion: batch/v2alpha1 kind: CronJob metadata: - name: curator + name: elastic-curator spec: schedule: {{ .Values.conf.curator.schedule | quote }} jobTemplate: diff --git a/elasticsearch/templates/deployment-client.yaml b/elasticsearch/templates/deployment-client.yaml index c871e2280..f5de19bd3 100644 --- a/elasticsearch/templates/deployment-client.yaml +++ b/elasticsearch/templates/deployment-client.yaml @@ -22,6 +22,8 @@ limitations under the License. {{- $_ := set .Values "pod_dependency" .Values.dependencies.elasticsearch_client -}} {{- end -}} +{{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }} + {{- $serviceAccountName := "elasticsearch-client"}} {{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- @@ -126,6 +128,11 @@ spec: readOnly: true - name: storage mountPath: {{ .Values.conf.elasticsearch.path.data }} + {{ if .Values.storage.filesystem_repository.enabled }} + - name: snapshots + mountPath: {{ .Values.conf.elasticsearch.path.repo }} + {{ end }} +{{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }} volumes: - name: elastic-logs emptyDir: {} @@ -141,4 +148,10 @@ spec: defaultMode: 0444 - name: storage emptyDir: {} + {{ if .Values.storage.filesystem_repository.enabled }} + - name: snapshots + persistentVolumeClaim: + claimName: {{ .Values.storage.filesystem_repository.pvc.name }} + {{ end }} +{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }} {{- end }} diff --git a/elasticsearch/templates/deployment-master.yaml b/elasticsearch/templates/deployment-master.yaml index a67abfcd8..30afc5ed7 100644 --- a/elasticsearch/templates/deployment-master.yaml +++ b/elasticsearch/templates/deployment-master.yaml @@ -22,6 +22,8 @@ limitations under the License. {{- $_ := set .Values "pod_dependency" .Values.dependencies.elasticsearch_master -}} {{- end -}} +{{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }} + {{- $serviceAccountName := "elasticsearch-master"}} {{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- @@ -120,6 +122,11 @@ spec: readOnly: true - name: storage mountPath: {{ .Values.conf.elasticsearch.path.data }} + {{ if .Values.storage.filesystem_repository.enabled }} + - name: snapshots + mountPath: {{ .Values.conf.elasticsearch.path.repo }} + {{ end }} +{{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }} volumes: - name: elastic-logs emptyDir: {} @@ -135,4 +142,10 @@ spec: defaultMode: 0444 - name: storage emptyDir: {} + {{ if .Values.storage.filesystem_repository.enabled }} + - name: snapshots + persistentVolumeClaim: + claimName: {{ .Values.storage.filesystem_repository.pvc.name }} + {{ end }} +{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }} {{- end }} diff --git a/elasticsearch/templates/etc/_elasticsearch.yml.tpl b/elasticsearch/templates/etc/_elasticsearch.yml.tpl index b150e5b6d..640eab60a 100644 --- a/elasticsearch/templates/etc/_elasticsearch.yml.tpl +++ b/elasticsearch/templates/etc/_elasticsearch.yml.tpl @@ -28,6 +28,7 @@ network.host: {{ .Values.conf.elasticsearch.network.host }} path: data: {{ .Values.conf.elasticsearch.path.data }} logs: {{ .Values.conf.elasticsearch.path.logs }} + repo: {{ .Values.conf.elasticsearch.path.repo }} bootstrap: memory_lock: {{ .Values.conf.elasticsearch.bootstrap.memory_lock }} diff --git a/elasticsearch/templates/job-image-repo-sync.yaml b/elasticsearch/templates/job-image-repo-sync.yaml index 0d75d6d19..2c2c04411 100644 --- a/elasticsearch/templates/job-image-repo-sync.yaml +++ b/elasticsearch/templates/job-image-repo-sync.yaml @@ -50,16 +50,16 @@ spec: command: - /tmp/image-repo-sync.sh volumeMounts: - - name: elasticsearch-bin + - name: elastic-bin mountPath: /tmp/image-repo-sync.sh subPath: image-repo-sync.sh readOnly: true - name: docker-socket mountPath: /var/run/docker.sock volumes: - - name: elasticsearch-bin + - name: elastic-bin configMap: - name: elasticsearch-bin + name: elastic-bin defaultMode: 0555 - name: docker-socket hostPath: diff --git a/elasticsearch/templates/job-register-snapshot-repository.yaml b/elasticsearch/templates/job-register-snapshot-repository.yaml new file mode 100644 index 000000000..e6a8b19b0 --- /dev/null +++ b/elasticsearch/templates/job-register-snapshot-repository.yaml @@ -0,0 +1,65 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_snapshot_repository }} +{{- $envAll := . }} +{{- $_ := set .Values "pod_dependency" .Values.dependencies.snapshot_repository -}} + +{{- $serviceAccountName := "elasticsearch-register-snapshot-repository" }} +{{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: elasticsearch-register-snapshot-repository +spec: + template: + metadata: + labels: +{{ tuple $envAll "elasticsearch" "snapshot-repository" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }} + initContainers: +{{ tuple $envAll .Values.pod_dependency list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: register-snapshot-repository +{{ tuple $envAll "snapshot_repository" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.snapshot_repository | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: ELASTICSEARCH_ENDPOINT + value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + - name: REPO_NAME + value: {{ .Values.conf.elasticsearch.repository.name | quote }} + - name: REPO_TYPE + value: {{ .Values.conf.elasticsearch.repository.type | quote }} + - name: REPO_LOCATION + value: {{ .Values.conf.elasticsearch.path.repo | quote }} + command: + - /tmp/register-repository.sh + volumeMounts: + - name: elastic-bin + mountPath: /tmp/register-repository.sh + subPath: register-repository.sh + readOnly: true + volumes: + - name: elastic-bin + configMap: + name: elastic-bin + defaultMode: 0555 +{{- end }} diff --git a/elasticsearch/templates/pvc-snapshots.yaml b/elasticsearch/templates/pvc-snapshots.yaml new file mode 100644 index 000000000..4dd5028cc --- /dev/null +++ b/elasticsearch/templates/pvc-snapshots.yaml @@ -0,0 +1,33 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.pvc_snapshots }} +{{- if .Values.storage.filesystem_repository.enabled }} +{{- $envAll := . }} +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ .Values.storage.filesystem_repository.pvc.name }} +spec: + accessModes: + - {{ .Values.storage.filesystem_repository.pvc.access_mode }} + resources: + requests: + storage: {{ .Values.storage.filesystem_repository.requests.storage }} + storageClassName: {{ .Values.storage.filesystem_repository.storage_class }} +{{- end }} +{{- end }} diff --git a/elasticsearch/templates/statefulset-data.yaml b/elasticsearch/templates/statefulset-data.yaml index c998ff3ef..fda2f479a 100644 --- a/elasticsearch/templates/statefulset-data.yaml +++ b/elasticsearch/templates/statefulset-data.yaml @@ -22,6 +22,8 @@ limitations under the License. {{- $_ := set .Values "pod_dependency" .Values.dependencies.elasticsearch_data -}} {{- end -}} +{{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }} + {{- $serviceAccountName := "elasticsearch-data"}} {{ tuple $envAll $envAll.Values.pod_dependency $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- @@ -115,8 +117,13 @@ spec: mountPath: /usr/share/elasticsearch/config/log4j2.properties subPath: log4j2.properties readOnly: true + {{ if .Values.storage.filesystem_repository.enabled }} + - name: snapshots + mountPath: {{ .Values.conf.elasticsearch.path.repo }} + {{ end }} - name: storage mountPath: {{ .Values.conf.elasticsearch.path.data }} +{{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }} volumes: - name: elastic-logs emptyDir: {} @@ -130,7 +137,13 @@ spec: configMap: name: elastic-etc defaultMode: 0444 -{{- if not .Values.storage.enabled }} + {{ if .Values.storage.filesystem_repository.enabled }} + - name: snapshots + persistentVolumeClaim: + claimName: {{ .Values.storage.filesystem_repository.pvc.name }} + {{ end }} +{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }} +{{- if not .Values.storage.elasticsearch.enabled }} - name: storage emptyDir: {} {{- else }} @@ -138,10 +151,10 @@ spec: - metadata: name: storage spec: - accessModes: {{ .Values.storage.pvc.access_mode }} + accessModes: {{ .Values.storage.elasticsearch.pvc.access_mode }} resources: requests: - storage: {{ .Values.storage.requests.storage }} - storageClassName: {{ .Values.storage.storage_class }} + storage: {{ .Values.storage.elasticsearch.requests.storage }} + storageClassName: {{ .Values.storage.elasticsearch.storage_class }} {{- end }} {{- end }} diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index a0a1349ab..fb03ef42c 100644 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -18,11 +18,12 @@ images: tags: - memory_init: docker.io/kolla/ubuntu-source-kolla-toolbox:4.0.0 + memory_init: docker.io/kolla/ubuntu-source-heat-engine:3.0.3 curator: docker.io/bobrik/curator:5.2.0 elasticsearch: docker.io/elasticsearch:5.4.2 - helm_tests: docker.io/kolla/ubuntu-source-kolla-toolbox:3.0.3 + helm_tests: docker.io/kolla/ubuntu-source-heat-engine:3.0.3 dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 + snapshot_repository: docker.io/kolla/ubuntu-source-heat-engine:3.0.3 image_repo_sync: docker.io/docker:17.07.0 pull_policy: "IfNotPresent" local_registry: @@ -48,6 +49,10 @@ dependencies: services: - service: local_image_registry endpoint: internal + snapshot_repository: + services: + - service: elasticsearch + endpoint: internal conditional_dependencies: local_image_registry: @@ -83,6 +88,9 @@ pod: timeout: 600 client: timeout: 600 + mounts: + elasticsearch: + elasticsearch: resources: enabled: false client: @@ -121,6 +129,13 @@ pod: limits: memory: "1024Mi" cpu: "2000m" + snapshot_repository: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" tests: requests: memory: "128Mi" @@ -149,7 +164,33 @@ conf: options: timeout_override: continue_if_exception: False - disable_action: False + ignore_empty_list: True + disable_action: True + filters: + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: 30 + field: + stats_result: + epoch: + exclude: False + 2: + action: snapshot + description: "Snapshot indices and send to configured repository" + options: + repository: default_repo + # Leaving this blank results in the default name format + name: + wait_for_completion: True + max_wait: 3600 + wait_interval: 10 + timeout_override: + ignore_empty_list: True + continue_if_exception: False + disable_action: True filters: - filtertype: age source: name @@ -195,6 +236,10 @@ conf: path: data: /usr/share/elasticsearch/data logs: /usr/share/elasticsearch/logs + repo: /usr/share/elasticsearch/repo + repository: + name: default_repo + type: fs zen: min_masters: 2 env: @@ -244,13 +289,23 @@ network: port: 30931 storage: - enabled: true - pvc: - name: pvc-elastic - access_mode: [ "ReadWriteOnce" ] - requests: - storage: 5Gi - storage_class: general + elasticsearch: + enabled: true + pvc: + name: pvc-elastic + access_mode: [ "ReadWriteOnce" ] + requests: + storage: 5Gi + storage_class: general + filesystem_repository: + enabled: true + pvc: + name: pvc-snapshots + access_mode: ReadWriteMany + requests: + storage: 5Gi + storage_class: general + manifests: clusterrole: true @@ -262,8 +317,9 @@ manifests: deployment_client: true deployment_master: true job_image_repo_sync: true + job_snapshot_repository: true helm_tests: true - serviceaccount: true + pvc_snapshots: true service_data: true service_discovery: true service_logging: true diff --git a/tools/gate/chart-deploys/default.yaml b/tools/gate/chart-deploys/default.yaml index a4afac007..717e6a114 100644 --- a/tools/gate/chart-deploys/default.yaml +++ b/tools/gate/chart-deploys/default.yaml @@ -141,7 +141,10 @@ charts: output: false values: storage: - enabled: false + elasticsearch: + storage_class: openstack-helm-bootstrap + filesystem_repository: + storage_class: openstack-helm-bootstrap fluent_logging: chart_name: fluent-logging