From 2d3c9575ff6d7419ce5460e2beadddd6655c33e3 Mon Sep 17 00:00:00 2001 From: Steve Wilkerson Date: Thu, 14 Nov 2019 14:45:46 -0600 Subject: [PATCH] Elasticsearch/Kibana: Update version to 7.1.0 This updates the Elasticsearch and Kibana charts to deploy version 7.1.0. This move required significant changes to both charts, including: changing elasticsearch masters to a statefulset to utilize reliable dns names for the discovery process, config updates to reflect deprecated/updated/removed values, use the kibana saved objects api for managing index patterns and setting the default index, and updating the elasticsearch entrypoint scripts to reflect the use of elastic-keystore for storing s3 credentials instead of defining them in the configuration file Change-Id: I270d905f266fc15492e47d8376714ba80603e66d Signed-off-by: Steve Wilkerson --- .../templates/bin/_elasticsearch.sh.tpl | 37 +++- .../templates/bin/_register-repository.sh.tpl | 4 +- .../configmap-etc-elasticsearch.yaml | 20 +-- .../templates/deployment-client.yaml | 24 ++- elasticsearch/templates/statefulset-data.yaml | 44 ++++- ...nt-master.yaml => statefulset-master.yaml} | 58 ++++++- elasticsearch/values.yaml | 158 ++++++++++-------- .../bin/_create_kibana_index_patterns.sh.tpl | 14 +- kibana/templates/bin/_kibana.sh.tpl | 10 +- kibana/templates/deployment.yaml | 24 ++- .../job-register-kibana-indexes.yaml | 2 + kibana/values.yaml | 14 +- .../armada/manifests/armada-lma.yaml | 111 ++++++------ .../deployment/multinode/120-elasticsearch.sh | 2 +- .../osh-infra-logging/050-elasticsearch.sh | 2 +- 15 files changed, 342 insertions(+), 182 deletions(-) rename elasticsearch/templates/{deployment-master.yaml => statefulset-master.yaml} (79%) diff --git a/elasticsearch/templates/bin/_elasticsearch.sh.tpl b/elasticsearch/templates/bin/_elasticsearch.sh.tpl index ecd619cd8..11d0608fb 100644 --- a/elasticsearch/templates/bin/_elasticsearch.sh.tpl +++ b/elasticsearch/templates/bin/_elasticsearch.sh.tpl @@ -15,12 +15,21 @@ See the License for the specific language governing permissions and limitations under the License. */}} -set -ex +{{- $envAll := . }} + +set -e COMMAND="${@:-start}" +function initiate_keystore () { + bin/elasticsearch-keystore create + echo ${S3_ACCESS_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.default.access_key + echo ${S3_SECRET_KEY} | /usr/share/elasticsearch/bin/elasticsearch-keystore add -xf s3.client.default.secret_key +} + function start () { ulimit -l unlimited - exec /docker-entrypoint.sh elasticsearch + initiate_keystore + exec /usr/local/bin/docker-entrypoint.sh elasticsearch } function stop () { @@ -42,10 +51,32 @@ function allocate_data_node () { echo "Node ${NODE_NAME} is ready to be used" } +function start_master_node () { + ulimit -l unlimited + initiate_keystore + if [ ! -f {{ $envAll.Values.conf.elasticsearch.config.path.data }}/cluster-bootstrap.txt ]; + then + {{ if empty $envAll.Values.conf.elasticsearch.config.cluster.initial_master_nodes -}} + {{- $_ := set $envAll.Values "__eligible_masters" ( list ) }} + {{- range $podInt := until ( atoi (print $envAll.Values.pod.replicas.master ) ) }} + {{- $eligibleMaster := printf "elasticsearch-master-%s" (toString $podInt) }} + {{- $__eligible_masters := append $envAll.Values.__eligible_masters $eligibleMaster }} + {{- $_ := set $envAll.Values "__eligible_masters" $__eligible_masters }} + {{- end -}} + {{- $masters := include "helm-toolkit.utils.joinListWithComma" $envAll.Values.__eligible_masters -}} + echo {{$masters}} >> {{ $envAll.Values.conf.elasticsearch.config.path.data }}/cluster-bootstrap.txt + exec /usr/local/bin/docker-entrypoint.sh elasticsearch -Ecluster.initial_master_nodes={{$masters}} + {{- end }} + else + exec /usr/local/bin/docker-entrypoint.sh elasticsearch + fi +} + function start_data_node () { ulimit -l unlimited + initiate_keystore allocate_data_node & - /docker-entrypoint.sh elasticsearch & + /usr/local/bin/docker-entrypoint.sh elasticsearch & function drain_data_node () { echo "Prepare to migrate data off node ${NODE_NAME}" echo "Move all data from node ${NODE_NAME}" diff --git a/elasticsearch/templates/bin/_register-repository.sh.tpl b/elasticsearch/templates/bin/_register-repository.sh.tpl index decb2bc86..0a011b9cc 100644 --- a/elasticsearch/templates/bin/_register-repository.sh.tpl +++ b/elasticsearch/templates/bin/_register-repository.sh.tpl @@ -32,9 +32,7 @@ function register_snapshot_repository() { "settings": { "endpoint": "'"$RGW_HOST"'", "protocol": "http", - "bucket": "'"$S3_BUCKET"'", - "access_key": "'"$S3_ACCESS_KEY"'", - "secret_key": "'"$S3_SECRET_KEY"'" + "bucket": "'"$S3_BUCKET"'" } }' | python -c "import sys, json; print(json.load(sys.stdin)['acknowledged'])") if [ "$result" == "True" ]; diff --git a/elasticsearch/templates/configmap-etc-elasticsearch.yaml b/elasticsearch/templates/configmap-etc-elasticsearch.yaml index eebeb303e..a71224b90 100644 --- a/elasticsearch/templates/configmap-etc-elasticsearch.yaml +++ b/elasticsearch/templates/configmap-etc-elasticsearch.yaml @@ -17,23 +17,18 @@ limitations under the License. {{- if .Values.manifests.configmap_etc_elasticsearch }} {{- $envAll := . }} -{{- if empty .Values.conf.elasticsearch.config.cloud.aws.access_key -}} -{{- set .Values.conf.elasticsearch.config.cloud.aws "access_key" .Values.endpoints.ceph_object_store.auth.elasticsearch.access_key -}} -{{- end -}} - -{{- if empty .Values.conf.elasticsearch.config.cloud.aws.secret_key -}} -{{- set .Values.conf.elasticsearch.config.cloud.aws "secret_key" .Values.endpoints.ceph_object_store.auth.elasticsearch.secret_key -}} -{{- end -}} - {{- if empty .Values.endpoints.ceph_object_store.path.default -}} {{- set .Values.endpoints.ceph_object_store.path "default" .Values.conf.elasticsearch.snapshots.bucket -}} {{- end -}} -{{- if empty .Values.conf.elasticsearch.config.cloud.aws.s3.endpoint -}} +{{- if empty .Values.conf.elasticsearch.config.s3.client.default.endpoint -}} {{- $radosgw_host := tuple "ceph_object_store" "internal" "api" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} -{{- $bucket_path := tuple "ceph_object_store" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_path_lookup" -}} -{{- $s3_endpoint := printf "%s/%s" $radosgw_host $bucket_path -}} -{{- set .Values.conf.elasticsearch.config.cloud.aws.s3 "endpoint" $s3_endpoint -}} +{{- set .Values.conf.elasticsearch.config.s3.client.default "endpoint" $radosgw_host -}} +{{- end -}} + +{{- if empty .Values.conf.elasticsearch.config.discovery.seed_hosts -}} +{{- $discovery_svc := tuple "elasticsearch" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" -}} +{{- set .Values.conf.elasticsearch.config.discovery "seed_hosts" $discovery_svc -}} {{- end -}} --- apiVersion: v1 @@ -46,4 +41,5 @@ data: #NOTE(portdirect): this must be last, to work round helm ~2.7 bug. {{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.httpd "key" "httpd.conf" "format" "Secret") | indent 2 }} {{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.log4j2 "key" "log4j2.properties" "format" "Secret") | indent 2 }} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.jvm_options "key" "jvm.options" "format" "Secret") | indent 2 }} {{- end }} diff --git a/elasticsearch/templates/deployment-client.yaml b/elasticsearch/templates/deployment-client.yaml index 9b158ba15..a327157ff 100644 --- a/elasticsearch/templates/deployment-client.yaml +++ b/elasticsearch/templates/deployment-client.yaml @@ -18,6 +18,7 @@ limitations under the License. {{- $envAll := . }} {{- $esUserSecret := .Values.secrets.elasticsearch.user }} +{{- $s3UserSecret := .Values.secrets.rgw.elasticsearch }} {{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }} @@ -177,7 +178,7 @@ spec: - name: NODE_MASTER value: "false" - name: NODE_INGEST - value: "false" + value: "true" - name: NODE_DATA value: "false" - name: HTTP_ENABLE @@ -186,6 +187,19 @@ spec: value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} - name: ES_JAVA_OPTS value: "{{ .Values.conf.elasticsearch.env.java_opts.client }}" + - name: S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ $s3UserSecret }} + key: S3_ACCESS_KEY + - name: S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ $s3UserSecret }} + key: S3_SECRET_KEY +{{- if .Values.pod.env.client }} +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.client | indent 12 }} +{{- end }} volumeMounts: - name: pod-tmp mountPath: /tmp @@ -195,8 +209,6 @@ spec: mountPath: /tmp/elasticsearch.sh subPath: elasticsearch.sh readOnly: true - - name: elasticsearch-config - mountPath: /usr/share/elasticsearch/config - name: elasticsearch-etc mountPath: /usr/share/elasticsearch/config/elasticsearch.yml subPath: elasticsearch.yml @@ -205,14 +217,16 @@ spec: mountPath: /usr/share/elasticsearch/config/log4j2.properties subPath: log4j2.properties readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/jvm.options + subPath: jvm.options + readOnly: true - name: storage mountPath: {{ .Values.conf.elasticsearch.config.path.data }} {{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }} volumes: - name: pod-tmp emptyDir: {} - - name: elasticsearch-config - emptyDir: {} - name: elasticsearch-logs emptyDir: {} - name: elasticsearch-bin diff --git a/elasticsearch/templates/statefulset-data.yaml b/elasticsearch/templates/statefulset-data.yaml index a585266c6..c68fe0399 100644 --- a/elasticsearch/templates/statefulset-data.yaml +++ b/elasticsearch/templates/statefulset-data.yaml @@ -18,6 +18,7 @@ limitations under the License. {{- $envAll := . }} {{- $esUserSecret := .Values.secrets.elasticsearch.user }} +{{- $s3UserSecret := .Values.secrets.rgw.elasticsearch }} {{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }} @@ -70,6 +71,7 @@ metadata: spec: {{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_statefulset" | indent 2 }} serviceName: {{ tuple "elasticsearch" "data" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + podManagementPolicy: "Parallel" replicas: {{ .Values.pod.replicas.data }} selector: matchLabels: @@ -101,6 +103,19 @@ spec: - sysctl - -w - vm.max_map_count={{ .Values.conf.init.max_map_count }} + - name: elasticsearch-perms +{{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.prometheus | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + securityContext: + runAsUser: 0 + command: + - chown + - -R + - "elasticsearch:" + - {{ .Values.conf.elasticsearch.config.path.data }} + volumeMounts: + - name: storage + mountPath: {{ .Values.conf.elasticsearch.config.path.data }} containers: - name: elasticsearch-data {{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }} @@ -148,8 +163,21 @@ spec: value: "false" - name: ES_JAVA_OPTS value: "{{ .Values.conf.elasticsearch.env.java_opts.data }}" + - name: S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ $s3UserSecret }} + key: S3_ACCESS_KEY + - name: S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ $s3UserSecret }} + key: S3_SECRET_KEY - name: DISCOVERY_SERVICE value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +{{- if .Values.pod.env.data }} +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.data | indent 12 }} +{{- end }} volumeMounts: - name: pod-tmp mountPath: /tmp @@ -159,8 +187,6 @@ spec: mountPath: /tmp/elasticsearch.sh subPath: elasticsearch.sh readOnly: true - - name: elasticsearch-config - mountPath: /usr/share/elasticsearch/config - name: elasticsearch-etc mountPath: /usr/share/elasticsearch/config/elasticsearch.yml subPath: elasticsearch.yml @@ -169,6 +195,10 @@ spec: mountPath: /usr/share/elasticsearch/config/log4j2.properties subPath: log4j2.properties readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/jvm.options + subPath: jvm.options + readOnly: true - name: storage mountPath: {{ .Values.conf.elasticsearch.config.path.data }} {{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }} @@ -177,8 +207,6 @@ spec: emptyDir: {} - name: elasticsearch-logs emptyDir: {} - - name: elasticsearch-config - emptyDir: {} - name: elasticsearch-bin configMap: name: elasticsearch-bin @@ -188,7 +216,7 @@ spec: secretName: elasticsearch-etc defaultMode: 0444 {{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }} -{{- if not .Values.storage.enabled }} +{{- if not .Values.storage.data.enabled }} - name: storage emptyDir: {} {{- else }} @@ -196,10 +224,10 @@ spec: - metadata: name: storage spec: - accessModes: {{ .Values.storage.pvc.access_mode }} + accessModes: {{ .Values.storage.data.pvc.access_mode }} resources: requests: - storage: {{ .Values.storage.requests.storage }} - storageClassName: {{ .Values.storage.storage_class }} + storage: {{ .Values.storage.data.requests.storage }} + storageClassName: {{ .Values.storage.data.storage_class }} {{- end }} {{- end }} diff --git a/elasticsearch/templates/deployment-master.yaml b/elasticsearch/templates/statefulset-master.yaml similarity index 79% rename from elasticsearch/templates/deployment-master.yaml rename to elasticsearch/templates/statefulset-master.yaml index 5a47d2ceb..e257c1ea6 100644 --- a/elasticsearch/templates/deployment-master.yaml +++ b/elasticsearch/templates/statefulset-master.yaml @@ -14,9 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if .Values.manifests.deployment_master }} +{{- if .Values.manifests.statefulset_master }} {{- $envAll := . }} +{{- $s3UserSecret := .Values.secrets.rgw.elasticsearch }} + {{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }} {{- $serviceAccountName := "elasticsearch-master" }} @@ -58,7 +60,7 @@ rules: - get --- apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: elasticsearch-master annotations: @@ -66,6 +68,8 @@ metadata: labels: {{ tuple $envAll "elasticsearch" "master" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} spec: + serviceName: {{ tuple "elasticsearch" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + podManagementPolicy: "Parallel" replicas: {{ .Values.pod.replicas.master }} selector: matchLabels: @@ -98,6 +102,19 @@ spec: - sysctl - -w - vm.max_map_count={{ .Values.conf.init.max_map_count }} + - name: elasticsearch-perms +{{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.prometheus | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + securityContext: + runAsUser: 0 + command: + - chown + - -R + - "elasticsearch:" + - {{ .Values.conf.elasticsearch.config.path.data }} + volumeMounts: + - name: storage + mountPath: {{ .Values.conf.elasticsearch.config.path.data }} containers: - name: elasticsearch-master {{ dict "envAll" $envAll "application" "master" "container" "elasticsearch_master" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} @@ -105,7 +122,7 @@ spec: {{ tuple $envAll $envAll.Values.pod.resources.master | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} command: - /tmp/elasticsearch.sh - - start + - start_master_node lifecycle: preStop: exec: @@ -141,6 +158,19 @@ spec: value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} - name: ES_JAVA_OPTS value: "{{ .Values.conf.elasticsearch.env.java_opts.master }}" + - name: S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ $s3UserSecret }} + key: S3_ACCESS_KEY + - name: S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ $s3UserSecret }} + key: S3_SECRET_KEY +{{- if .Values.pod.env.master }} +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.master | indent 12 }} +{{- end }} volumeMounts: - name: pod-tmp mountPath: /tmp @@ -150,8 +180,6 @@ spec: mountPath: /tmp/elasticsearch.sh subPath: elasticsearch.sh readOnly: true - - name: elasticsearch-config - mountPath: /usr/share/elasticsearch/config - name: elasticsearch-etc mountPath: /usr/share/elasticsearch/config/elasticsearch.yml subPath: elasticsearch.yml @@ -160,6 +188,10 @@ spec: mountPath: /usr/share/elasticsearch/config/log4j2.properties subPath: log4j2.properties readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/jvm.options + subPath: jvm.options + readOnly: true - name: storage mountPath: {{ .Values.conf.elasticsearch.config.path.data }} {{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }} @@ -168,8 +200,6 @@ spec: emptyDir: {} - name: elasticsearch-logs emptyDir: {} - - name: elasticsearch-config - emptyDir: {} - name: elasticsearch-bin configMap: name: elasticsearch-bin @@ -178,7 +208,19 @@ spec: secret: secretName: elasticsearch-etc defaultMode: 0444 +{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }} +{{- if not .Values.storage.master.enabled }} - name: storage emptyDir: {} -{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }} +{{- else }} + volumeClaimTemplates: + - metadata: + name: storage + spec: + accessModes: {{ .Values.storage.master.pvc.access_mode }} + resources: + requests: + storage: {{ .Values.storage.master.requests.storage }} + storageClassName: {{ .Values.storage.master.storage_class }} +{{- end }} {{- end }} diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 69732b988..ab02fcbcf 100644 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -20,8 +20,8 @@ images: tags: apache_proxy: docker.io/httpd:2.4 memory_init: docker.io/openstackhelm/heat:newton-ubuntu_xenial + elasticsearch: docker.io/openstackhelm/elasticsearch-s3:7_1_0-20191119 curator: docker.io/bobrik/curator:5.6.0 - elasticsearch: docker.io/openstackhelm/elasticsearch-s3:5_6_4-20191119 ceph_key_placement: docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119 s3_bucket: docker.io/openstackhelm/ceph-daemon:ubuntu_xenial-20191119 s3_user: docker.io/openstackhelm/ceph-config-helper:ubuntu_xenial-20191119 @@ -123,6 +123,10 @@ dependencies: - elasticsearch-register-snapshot-repository pod: + env: + client: null + data: null + master: null mandatory_access_control: type: apparmor elasticsearch-master: @@ -154,7 +158,7 @@ pod: add: - IPC_LOCK - SYS_RESOURCE - readOnlyRootFilesystem: true + readOnlyRootFilesystem: false master: pod: runAsUser: 0 @@ -168,7 +172,7 @@ pod: add: - IPC_LOCK - SYS_RESOURCE - readOnlyRootFilesystem: true + readOnlyRootFilesystem: false es_cluster_wait: pod: runAsUser: 0 @@ -418,6 +422,7 @@ conf: LoadModule unixd_module modules/mod_unixd.so LoadModule status_module modules/mod_status.so LoadModule autoindex_module modules/mod_autoindex.so + LoadModule rewrite_module modules/mod_rewrite.so User daemon @@ -480,10 +485,6 @@ conf: Require valid-user - # Restrict access to the Elasticsearch Update API endpoint to prevent modification of indexed documents - - Require all denied - # Restrict access to the Elasticsearch Update By Query API Endpoint to prevent modification of indexed documents Require all denied @@ -492,30 +493,49 @@ conf: Require all denied - - # Prohibit DELETE methods on the document API endpoint - - AllowMethods GET POST OPTIONS - ProxyPass http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/ - ProxyPassReverse http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/ - AuthName "Elasticsearch" - AuthType Basic - AuthBasicProvider file ldap - AuthUserFile /usr/local/apache2/conf/.htpasswd - AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }} - AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }} - AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} - Require valid-user - log4j2: | status = error appender.console.type = Console appender.console.name = console appender.console.layout.type = PatternLayout - appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n + appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker%m%n rootLogger.level = info rootLogger.appenderRef.console.ref = console + jvm_options: | + -Xms1g + -Xmx1g + -XX:+UseConcMarkSweepGC + -XX:CMSInitiatingOccupancyFraction=75 + -XX:+UseCMSInitiatingOccupancyOnly + -Des.networkaddress.cache.ttl=60 + -Des.networkaddress.cache.negative.ttl=10 + -XX:+AlwaysPreTouch + -Xss1m + -Djava.awt.headless=true + -Dfile.encoding=UTF-8 + -Djna.nosys=true + -XX:-OmitStackTraceInFastThrow + -Dio.netty.noUnsafe=true + -Dio.netty.noKeySetOptimization=true + -Dio.netty.recycler.maxCapacityPerThread=0 + -Dlog4j.shutdownHookEnabled=false + -Dlog4j2.disable.jmx=true + -Djava.io.tmpdir=${ES_TMPDIR} + -XX:+HeapDumpOnOutOfMemoryError + -XX:HeapDumpPath=data + -XX:ErrorFile=logs/hs_err_pid%p.log + 8:-XX:+PrintGCDetails + 8:-XX:+PrintGCDateStamps + 8:-XX:+PrintTenuringDistribution + 8:-XX:+PrintGCApplicationStoppedTime + 8:-Xloggc:logs/gc.log + 8:-XX:+UseGCLogFileRotation + 8:-XX:NumberOfGCLogFiles=32 + 8:-XX:GCLogFileSize=64m + 9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m + 9-:-Djava.locale.providers=COMPAT + 10-:-XX:UseAVX=2 init: max_map_count: 262144 ceph: @@ -636,21 +656,17 @@ conf: cluster: name: elasticsearch discovery: - zen: - ping.unicast.hosts: ${DISCOVERY_SERVICE} - minimum_master_nodes: 2 - http: - enabled: ${HTTP_ENABLE} - compression: true + # NOTE(srwilkers): This gets configured dynamically via endpoint lookups + seed_hosts: null network: host: 0.0.0.0 - cloud: - aws: - protocol: http - s3: + s3: + client: + default: # NOTE(srwilkers): This gets configured dynamically via endpoint # lookups endpoint: null + protocol: http node: ingest: ${NODE_INGEST} master: ${NODE_MASTER} @@ -658,8 +674,8 @@ conf: name: ${NODE_NAME} max_local_storage_nodes: 3 path: - data: /usr/share/elasticsearch/data - logs: /usr/share/elasticsearch/logs + data: /data + logs: /logs snapshots: enabled: false # NOTE(srwilkers): The path for the radosgw s3 endpoint gets populated @@ -680,33 +696,32 @@ conf: timeout: 20s templates: fluent: - template: "logstash-*" index_patterns: "logstash-*" settings: - number_of_shards: 1 + index: + number_of_shards: 1 mappings: - fluent: - properties: - kubernetes: - properties: - container_name: - type: keyword - index: false - docker_id: - type: keyword - index: false - host: - type: keyword - index: false - namespace_name: - type: keyword - index: false - pod_id: - type: keyword - index: false - pod_name: - type: keyword - index: false + properties: + kubernetes: + properties: + container_name: + type: keyword + index: false + docker_id: + type: keyword + index: false + host: + type: keyword + index: false + namespace_name: + type: keyword + index: false + pod_id: + type: keyword + index: false + pod_name: + type: keyword + index: false endpoints: cluster_domain_suffix: cluster.local @@ -829,13 +844,22 @@ network: port: 30920 storage: - enabled: true - pvc: - name: pvc-elastic - access_mode: [ "ReadWriteOnce" ] - requests: - storage: 5Gi - storage_class: general + data: + enabled: true + pvc: + name: pvc-elastic + access_mode: [ "ReadWriteOnce" ] + requests: + storage: 5Gi + storage_class: general + master: + enabled: true + pvc: + name: pvc-elastic + access_mode: [ "ReadWriteOnce" ] + requests: + storage: 1Gi + storage_class: general manifests: @@ -847,7 +871,6 @@ manifests: cron_curator: true cron_verify_repositories: true deployment_client: true - deployment_master: true ingress: true job_cluster_wait: true job_elasticsearch_templates: true @@ -870,3 +893,4 @@ manifests: service_ingress: true service_logging: true statefulset_data: true + statefulset_master: true diff --git a/kibana/templates/bin/_create_kibana_index_patterns.sh.tpl b/kibana/templates/bin/_create_kibana_index_patterns.sh.tpl index c31f23fd3..eebf5023e 100644 --- a/kibana/templates/bin/_create_kibana_index_patterns.sh.tpl +++ b/kibana/templates/bin/_create_kibana_index_patterns.sh.tpl @@ -18,9 +18,13 @@ set -ex {{- range .Values.conf.create_kibana_indexes.indexes }} curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ - -XPOST "${ELASTICSEARCH_ENDPOINT}/.kibana/index-pattern/{{ . }}-*" -H 'Content-Type: application/json' \ - -d '{"title":"{{ . }}-*","timeFieldName":"@timestamp","notExpandable":true}' -{{- end }} + -XPOST "${KIBANA_ENDPOINT}/api/saved_objects/index-pattern/{{ . }}*" -H 'kbn-xsrf: true' \ + -H 'Content-Type: application/json' -d \ + '{"attributes":{"title":"{{ . }}-*","timeFieldName":"@timestamp"}}' + +{{ end }} + curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ - -XPOST "${ELASTICSEARCH_ENDPOINT}/.kibana/config/5.6.4" -H 'Content-Type: application/json' \ - -d '{"defaultIndex" : "{{ .Values.conf.create_kibana_indexes.default_index }}-*"}' + -XPOST "${KIBANA_ENDPOINT}/api/kibana/settings/defaultIndex" -H 'kbn-xsrf: true' \ + -H 'Content-Type: application/json' -d \ + '{"value" : "{{ .Values.conf.create_kibana_indexes.default_index }}*"}' diff --git a/kibana/templates/bin/_kibana.sh.tpl b/kibana/templates/bin/_kibana.sh.tpl index 7021ac0dd..1bf3bd7eb 100644 --- a/kibana/templates/bin/_kibana.sh.tpl +++ b/kibana/templates/bin/_kibana.sh.tpl @@ -15,14 +15,14 @@ See the License for the specific language governing permissions and limitations under the License. */}} -set -ex +set -e COMMAND="${@:-start}" function start () { - exec kibana \ - --elasticsearch.url="$ELASTICSEARCH_URL" \ - --elasticsearch.username="$ELASTICSEARCH_USERNAME" \ - --elasticsearch.password="$ELASTICSEARCH_PASSWORD" + exec /usr/share/kibana/bin/kibana \ + --elasticsearch.hosts="${ELASTICSEARCH_HOSTS}" \ + --elasticsearch.username="${ELASTICSEARCH_USERNAME}" \ + --elasticsearch.password="${ELASTICSEARCH_PASSWORD}" } function stop () { diff --git a/kibana/templates/deployment.yaml b/kibana/templates/deployment.yaml index 298afbdd6..14a50e9f2 100644 --- a/kibana/templates/deployment.yaml +++ b/kibana/templates/deployment.yaml @@ -18,6 +18,13 @@ limitations under the License. {{- $envAll := . }} {{- $esUserSecret := .Values.secrets.elasticsearch.user }} +{{- $esUser := .Values.endpoints.elasticsearch.auth.admin.username }} +{{- $esPass := .Values.endpoints.elasticsearch.auth.admin.password }} +{{- $authHeader := printf "%s:%s" $esUser $esPass | b64enc }} + +{{- $esScheme := tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} +{{- $esSvc := tuple "elasticsearch" "default" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +{{- $esHosts := printf "%s://%s" $esScheme $esSvc }} {{- $serviceAccountName := "kibana" }} {{ tuple $envAll "kibana" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} @@ -101,9 +108,18 @@ spec: ports: - name: kibana containerPort: {{ tuple "kibana" "internal" "kibana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + readinessProbe: + httpGet: + path: /status + port: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + httpHeaders: + - name: Authorization + value: Basic {{ $authHeader }} + initialDelaySeconds: 20 + periodSeconds: 10 env: - - name: ELASTICSEARCH_URL - value: {{ tuple "elasticsearch" "default" "client" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }} + - name: ELASTICSEARCH_HOSTS + value: {{ $esHosts }} - name: ELASTICSEARCH_USERNAME valueFrom: secretKeyRef: @@ -124,6 +140,8 @@ spec: subPath: kibana.sh readOnly: true - name: pod-etc-kibana + mountPath: /usr/share/kibana/config + - name: pod-optimize-kibana mountPath: /usr/share/kibana/optimize - name: kibana-etc mountPath: /usr/share/kibana/config/kibana.yml @@ -137,6 +155,8 @@ spec: medium: "Memory" - name: pod-etc-kibana emptyDir: {} + - name: pod-optimize-kibana + emptyDir: {} - name: kibana-bin configMap: name: kibana-bin diff --git a/kibana/templates/job-register-kibana-indexes.yaml b/kibana/templates/job-register-kibana-indexes.yaml index d5b351c44..4a5de4fbf 100644 --- a/kibana/templates/job-register-kibana-indexes.yaml +++ b/kibana/templates/job-register-kibana-indexes.yaml @@ -53,6 +53,8 @@ spec: secretKeyRef: name: {{ $esUserSecret }} key: ELASTICSEARCH_PASSWORD + - name: KIBANA_ENDPOINT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} - name: ELASTICSEARCH_ENDPOINT value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} command: diff --git a/kibana/values.yaml b/kibana/values.yaml index 1c23f9fcf..4378e320b 100644 --- a/kibana/values.yaml +++ b/kibana/values.yaml @@ -23,7 +23,7 @@ labels: images: tags: apache_proxy: docker.io/httpd:2.4 - kibana: docker.io/kibana:5.6.4 + kibana: docker.elastic.co/kibana/kibana-oss:7.1.0 dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 image_repo_sync: docker.io/docker:17.07.0 register_kibana_indexes: docker.io/openstackhelm/heat:newton-ubuntu_xenial @@ -38,17 +38,19 @@ pod: security_context: dashboard: pod: - runAsUser: 999 + runAsUser: 1000 container: apache_proxy: runAsUser: 0 readOnlyRootFilesystem: false kibana: + fsGroup: 1000 + runAsNonRoot: true allowPrivilegeEscalation: false - readOnlyRootFilesystem: true + readOnlyRootFilesystem: false register_kibana_indexes: pod: - runAsUser: 999 + runAsUser: 1000 container: register_kibana_indexes: allowPrivilegeEscalation: false @@ -250,7 +252,6 @@ conf: startupTimeout: 5000 kibana: defaultAppId: discover - index: .kibana logging: quiet: false silent: false @@ -258,13 +259,16 @@ conf: ops: interval: 5000 server: + rewriteBasePath: false host: localhost + name: kibana maxPayloadBytes: 1048576 port: 5601 ssl: enabled: false create_kibana_indexes: enabled: true + version: 7.1.0 indexes: - logstash - openstack diff --git a/tools/deployment/armada/manifests/armada-lma.yaml b/tools/deployment/armada/manifests/armada-lma.yaml index fe9e78a4d..081f33abe 100644 --- a/tools/deployment/armada/manifests/armada-lma.yaml +++ b/tools/deployment/armada/manifests/armada-lma.yaml @@ -316,7 +316,7 @@ data: secret_key: ${RADOSGW_S3_ELASTICSEARCH_SECRET_KEY} pod: replicas: - data: 1 + data: 2 master: 2 labels: elasticsearch: @@ -341,74 +341,71 @@ data: settings: number_of_shards: 1 mappings: - syslog: - properties: - cluster: - type: keyword - app: - type: keyword - pid: - type: integer - host: - type: keyword - log: - type: text + properties: + cluster: + type: keyword + app: + type: keyword + pid: + type: integer + host: + type: keyword + log: + type: text oslo_openstack_fluentd: template: "openstack-*" index_patterns: "openstack-*" settings: number_of_shards: 1 mappings: - oslo_openstack_fluentd: - properties: - extra: - properties: - project: - type: text - norms: false - version: - type: text - norms: false - filename: - type: text - norms: false - funcname: - type: text - norms: false - message: - type: text - norms: false - process_name: - type: keyword - index: false + properties: + extra: + properties: + project: + type: text + norms: false + version: + type: text + norms: false + filename: + type: text + norms: false + funcname: + type: text + norms: false + message: + type: text + norms: false + process_name: + type: keyword + index: false docker_fluentd: template: "logstash-*" index_patterns: "logstash-*" settings: number_of_shards: 1 mappings: - docker_fluentd: - properties: - kubernetes: - properties: - container_name: - type: keyword - index: false - docker_id: - type: keyword - index: false - host: - type: keyword - index: false - namespace_name: - type: keyword - index: false - pod_id: - type: keyword - index: false - pod_name: - type: keyword - index: false + properties: + kubernetes: + properties: + container_name: + type: keyword + index: false + docker_id: + type: keyword + index: false + host: + type: keyword + index: false + namespace_name: + type: keyword + index: false + pod_id: + type: keyword + index: false + pod_name: + type: keyword + index: false curator: action_file: actions: diff --git a/tools/deployment/multinode/120-elasticsearch.sh b/tools/deployment/multinode/120-elasticsearch.sh index 2f3b45fe2..c64180996 100755 --- a/tools/deployment/multinode/120-elasticsearch.sh +++ b/tools/deployment/multinode/120-elasticsearch.sh @@ -26,7 +26,7 @@ jobs: cron: "*/3 * * * *" pod: replicas: - data: 1 + data: 2 master: 2 conf: elasticsearch: diff --git a/tools/deployment/osh-infra-logging/050-elasticsearch.sh b/tools/deployment/osh-infra-logging/050-elasticsearch.sh index ed5c3dbd4..b84b6beb3 100755 --- a/tools/deployment/osh-infra-logging/050-elasticsearch.sh +++ b/tools/deployment/osh-infra-logging/050-elasticsearch.sh @@ -29,7 +29,7 @@ monitoring: enabled: true pod: replicas: - data: 1 + data: 2 master: 2 conf: elasticsearch: