diff --git a/elasticsearch/Chart.yaml b/elasticsearch/Chart.yaml index 7c1d6c7c8..600dd5c8f 100644 --- a/elasticsearch/Chart.yaml +++ b/elasticsearch/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v7.6.2 description: OpenStack-Helm ElasticSearch name: elasticsearch -version: 0.2.0 +version: 0.2.1 home: https://www.elastic.co/ sources: - https://github.com/elastic/elasticsearch diff --git a/elasticsearch/templates/bin/_create_s3_buckets.sh.tpl b/elasticsearch/templates/bin/_create_s3_buckets.sh.tpl new file mode 100644 index 000000000..e1563a69d --- /dev/null +++ b/elasticsearch/templates/bin/_create_s3_buckets.sh.tpl @@ -0,0 +1,63 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +#!/bin/bash + +set -e + +function check_rgw_s3_bucket () { + echo "Checking if bucket exists" + s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET +} + +function create_rgw_s3_bucket () { + echo "Creating bucket" + s3cmd $CONNECTION_ARGS $S3_BUCKET_OPTS $USER_AUTH_ARGS mb s3://$S3_BUCKET +} + +function modify_bucket_acl () { + echo "Updating bucket ACL" + s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME +} + +ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY" + +{{- $envAll := . }} +{{- range $bucket := .Values.storage.s3.buckets }} + +S3_BUCKET={{ $bucket.name }} +S3_BUCKET_OPTS={{ $bucket.options | default nil | include "helm-toolkit.utils.joinListWithSpace" }} + +S3_USERNAME=${{ printf "%s_S3_USERNAME" ( $bucket.client | replace "-" "_" | upper) }} +S3_ACCESS_KEY=${{ printf "%s_S3_ACCESS_KEY" ( $bucket.client | replace "-" "_" | upper) }} +S3_SECRET_KEY=${{ printf "%s_S3_SECRET_KEY" ( $bucket.client | replace "-" "_" | upper) }} + +{{- with $client := index $envAll.Values.storage.s3.clients $bucket.client }} + +RGW_HOST={{ $client.settings.endpoint | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup") }} +RGW_PROTO={{ $client.settings.protocol | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") }} + +{{- end }} + +CONNECTION_ARGS="--host=$RGW_HOST --host-bucket=$RGW_HOST" +if [ "$RGW_PROTO" = "http" ]; then + CONNECTION_ARGS+=" --no-ssl" +fi + +USER_AUTH_ARGS=" --access_key=$S3_ACCESS_KEY --secret_key=$S3_SECRET_KEY" + +echo "Creating Bucket $S3_BUCKET at $RGW_HOST" +check_rgw_s3_bucket || ( create_rgw_s3_bucket && modify_bucket_acl ) + +{{- end }} diff --git a/elasticsearch/templates/bin/_create_template.sh.tpl b/elasticsearch/templates/bin/_create_template.sh.tpl index d90dd05ba..c61bb868a 100644 --- a/elasticsearch/templates/bin/_create_template.sh.tpl +++ b/elasticsearch/templates/bin/_create_template.sh.tpl @@ -13,11 +13,31 @@ See the License for the specific language governing permissions and limitations under the License. */}} -set -ex +set -e -{{ range $object := .Values.conf.api_objects }} -curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ +NUM_ERRORS=0 + +{{ range $name, $object := .Values.conf.api_objects }} +{{ if not (empty $object) }} + +echo "creating {{$name}}" +error=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ -X{{ $object.method | default "PUT" | upper }} \ "${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/{{ $object.endpoint }}" \ - -H 'Content-Type: application/json' -d '{{ $object.body | toJson }}' + -H 'Content-Type: application/json' -d '{{ $object.body | toJson }}' | jq -r '.error') + +if [ $error == "null" ]; then + echo "Object {{$name}} was created." +else + echo "Error when creating object {{$name}}: $(echo $error | jq -r)" + NUM_ERRORS=$(($NUM_ERRORS+1)) +fi + {{ end }} +{{ end }} + +if [ $NUM_ERRORS -gt 0 ]; then + exit 1 +else + echo "leaving normally" +fi diff --git a/elasticsearch/templates/bin/_helm-tests.sh.tpl b/elasticsearch/templates/bin/_helm-tests.sh.tpl index e6a7d2d08..79381733a 100644 --- a/elasticsearch/templates/bin/_helm-tests.sh.tpl +++ b/elasticsearch/templates/bin/_helm-tests.sh.tpl @@ -36,17 +36,30 @@ function create_test_index () { fi } -{{ if .Values.conf.elasticsearch.snapshots.enabled }} -function check_snapshot_repositories_registered () { - total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ - "${ELASTICSEARCH_ENDPOINT}/_snapshot" | jq length) - if [ "$total_hits" -gt 0 ]; then - echo "PASS: $total_hits Snapshot repositories have been registered!" +{{ if not (empty .Values.conf.api_objects) }} + +function test_api_object_creation () { + NUM_ERRORS=0 + {{ range $object, $config := .Values.conf.api_objects }} + error=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -XGET "${ELASTICSEARCH_ENDPOINT}/{{ $config.endpoint }}" | jq -r '.error') + + if [ $error == "null" ]; then + echo "PASS: {{ $object }} is verified." + else + echo "FAIL: Error for {{ $object }}: $(echo $error | jq -r)" + NUM_ERRORS=$(($NUM_ERRORS+1)) + fi + {{ end }} + + if [ $NUM_ERRORS -gt 0 ]; then + echo "FAIL: Some API Objects were not created!" + exit 1 else - echo "FAIL: No snapshot repositories found! Exiting"; - exit 1; + echo "PASS: API Objects are verified!" fi } + {{ end }} {{ if .Values.conf.elasticsearch.snapshots.enabled }} @@ -70,21 +83,6 @@ function check_snapshot_repositories_verified () { } {{ end }} -{{ if .Values.manifests.job_elasticsearch_templates }} -# Tests whether elasticsearch has successfully generated the elasticsearch index mapping -# templates defined by values.yaml -function check_templates () { - total_hits=$(curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ - -XGET "${ELASTICSEARCH_ENDPOINT}/_template" | jq length) - if [ "$total_hits" -gt 0 ]; then - echo "PASS: Successful hits on templates!" - else - echo "FAIL: No hits on query for templates! Exiting"; - exit 1; - fi -} -{{ end }} - function remove_test_index () { echo "Deleting index created for service testing" curl -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ @@ -93,9 +91,8 @@ function remove_test_index () { remove_test_index || true create_test_index +remove_test_index +test_api_object_creation {{ if .Values.conf.elasticsearch.snapshots.enabled }} -check_snapshot_repositories_registered check_snapshot_repositories_verified {{ end }} -check_templates -remove_test_index diff --git a/elasticsearch/templates/configmap-bin-elasticsearch.yaml b/elasticsearch/templates/configmap-bin-elasticsearch.yaml index afaa06534..645f16d7d 100644 --- a/elasticsearch/templates/configmap-bin-elasticsearch.yaml +++ b/elasticsearch/templates/configmap-bin-elasticsearch.yaml @@ -29,7 +29,7 @@ data: ceph-admin-keyring.sh: | {{ tuple "bin/_ceph-admin-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} create-s3-bucket.sh: | -{{- include "helm-toolkit.scripts.create_s3_bucket" . | indent 4 }} +{{ tuple "bin/_create_s3_buckets.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} create-s3-user.sh: | {{ tuple "bin/_create_s3_users.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} create_template.sh: | diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index e88253a04..250b75239 100644 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -736,43 +736,23 @@ conf: ca: null client_private_key: null client_cert: null - api_objects: - - endpoint: _template/fluent - body: - index_patterns: "logstash-*" - settings: - index: - number_of_shards: 1 - mappings: - properties: - kubernetes: - properties: - container_name: - type: keyword - index: false - docker_id: - type: keyword - index: false - host: - type: keyword - index: false - namespace_name: - type: keyword - index: false - pod_id: - type: keyword - index: false - pod_name: - type: keyword - index: false - - endpoint: _ilm/policy/delete_all_indexes - body: - policy: - phases: - delete: - min_age: 14d - actions: - delete: {} + + api_objects: {} + # Fill this map with API objects to create once Elasticsearch is deployed + # name: # This name can be completely arbitrary + # method: # Defaults to PUT + # endpoint: # Path for the request + # body: # Body of the request in yaml (Converted to Json in Template) + # Example: ILM Policy + # ilm_policy: + # endpoint: _ilm/policy/delete_all_indexes + # body: + # policy: + # phases: + # delete: + # min_age: 14d + # actions: + # delete: {} endpoints: cluster_domain_suffix: cluster.local diff --git a/helm-toolkit/Chart.yaml b/helm-toolkit/Chart.yaml index 20994d77e..5b4c7b6ed 100644 --- a/helm-toolkit/Chart.yaml +++ b/helm-toolkit/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v1.0.0 description: OpenStack-Helm Helm-Toolkit name: helm-toolkit -version: 0.2.10 +version: 0.2.11 home: https://docs.openstack.org/openstack-helm icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png sources: diff --git a/helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl b/helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl index 22b1f57b5..bf1465b23 100644 --- a/helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl +++ b/helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl @@ -11,56 +11,25 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */}} - {{- define "helm-toolkit.scripts.create_s3_bucket" }} #!/bin/bash - set -e - -function check_rgw_s3_bucket () { - echo "Checking if bucket exists" - s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET -} - -function create_rgw_s3_bucket () { - echo "Creating bucket" - s3cmd $CONNECTION_ARGS $S3_BUCKET_OPTS $USER_AUTH_ARGS mb s3://$S3_BUCKET -} - -function modify_bucket_acl () { - echo "Updating bucket ACL" - s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME -} - -ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY" - -{{- $envAll := . }} -{{- range $bucket := .Values.storage.s3.buckets }} - -S3_BUCKET={{ $bucket.name }} -S3_BUCKET_OPTS={{ $bucket.options | default nil | include "helm-toolkit.utils.joinListWithSpace" }} - -S3_USERNAME=${{ printf "%s_S3_USERNAME" ( $bucket.client | replace "-" "_" | upper) }} -S3_ACCESS_KEY=${{ printf "%s_S3_ACCESS_KEY" ( $bucket.client | replace "-" "_" | upper) }} -S3_SECRET_KEY=${{ printf "%s_S3_SECRET_KEY" ( $bucket.client | replace "-" "_" | upper) }} - -{{- with $client := index $envAll.Values.storage.s3.clients $bucket.client }} - -RGW_HOST={{ $client.settings.endpoint | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup") }} -RGW_PROTO={{ $client.settings.protocool | tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} - -{{- end }} - CONNECTION_ARGS="--host=$RGW_HOST --host-bucket=$RGW_HOST" if [ "$RGW_PROTO" = "http" ]; then CONNECTION_ARGS+=" --no-ssl" else CONNECTION_ARGS+=" --no-check-certificate" fi +ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY" USER_AUTH_ARGS=" --access_key=$S3_ACCESS_KEY --secret_key=$S3_SECRET_KEY" - -echo "Creating Bucket $S3_BUCKET at $RGW_HOST" +function check_rgw_s3_bucket () { + s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET +} +function create_rgw_s3_bucket () { + s3cmd $CONNECTION_ARGS $ADMIN_AUTH_ARGS mb s3://$S3_BUCKET +} +function modify_bucket_acl () { + s3cmd $CONNECTION_ARGS $ADMIN_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME +} check_rgw_s3_bucket || ( create_rgw_s3_bucket && modify_bucket_acl ) - -{{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm-toolkit/templates/scripts/_create-s3-user.sh.tpl b/helm-toolkit/templates/scripts/_create-s3-user.sh.tpl index c2d9ded15..08796d29c 100644 --- a/helm-toolkit/templates/scripts/_create-s3-user.sh.tpl +++ b/helm-toolkit/templates/scripts/_create-s3-user.sh.tpl @@ -11,12 +11,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */}} - {{- define "helm-toolkit.scripts.create_s3_user" }} #!/bin/bash - set -e - function create_s3_user () { echo "Creating s3 user and key pair" radosgw-admin user create \ @@ -26,7 +23,6 @@ function create_s3_user () { --access-key ${S3_ACCESS_KEY} \ --secret-key ${S3_SECRET_KEY} } - function update_s3_user () { # Retrieve old access keys, if they exist old_access_keys=$(radosgw-admin user info --uid=${S3_USERNAME} \ @@ -60,12 +56,10 @@ function update_s3_user () { --secret-key ${S3_SECRET_KEY} fi } - user_exists=$(radosgw-admin user info --uid=${S3_USERNAME} || true) if [[ -z ${user_exists} ]]; then create_s3_user else update_s3_user fi - -{{- end }} +{{- end }} \ No newline at end of file diff --git a/releasenotes/notes/elasticsearch.yaml b/releasenotes/notes/elasticsearch.yaml index 0cc8c6649..811773096 100644 --- a/releasenotes/notes/elasticsearch.yaml +++ b/releasenotes/notes/elasticsearch.yaml @@ -10,4 +10,5 @@ elasticsearch: - 0.1.7 Pin Java options to specific versions - 0.1.8 Disable Curator in Gate & Chart Defaults - 0.2.0 Add more S3 configuration options + - 0.2.1 Make templates job more robust & allow overrides ... diff --git a/releasenotes/notes/helm-toolkit.yaml b/releasenotes/notes/helm-toolkit.yaml index a511bf40f..1103cf8b5 100644 --- a/releasenotes/notes/helm-toolkit.yaml +++ b/releasenotes/notes/helm-toolkit.yaml @@ -17,4 +17,5 @@ helm-toolkit: - 0.2.8 Override the expiry of Ingress TLS certificate - 0.2.9 Jobs; put labels only in the template spec - 0.2.10 Add more S3 configuration options + - 0.2.11 Revert S3 User & Bucket job scripts to v0.2.9 ... diff --git a/tools/deployment/osh-infra-logging/050-elasticsearch.sh b/tools/deployment/osh-infra-logging/050-elasticsearch.sh index 8fa950b32..5e62ef05b 100755 --- a/tools/deployment/osh-infra-logging/050-elasticsearch.sh +++ b/tools/deployment/osh-infra-logging/050-elasticsearch.sh @@ -35,36 +35,34 @@ conf: snapshots: enabled: true api_objects: - - endpoint: _snapshot/ceph-rgw + snapshot_repo: + endpoint: _snapshot/ceph-rgw body: type: s3 settings: client: default bucket: elasticsearch-bucket - - endpoint: _snapshot/backup - body: - type: s3 - settings: - client: backup - bucket: backup-bucket - - endpoint: _slm/policy/rgw-snapshots + slm_policy: + endpoint: _slm/policy/snapshots body: schedule: "0 */3 * * * ?" name: "" repository: ceph-rgw config: - indices: ["*"] + indices: + - "<*-{now/d}>" retention: expire_after: 30d - - endpoint: _slm/policy/backup-snapshots + ilm_policy: + endpoint: _ilm/policy/cleanup body: - schedule: "0 */3 * * * ?" - name: "" - repository: backup - config: - indices: ["*"] - retention: - expire_after: 180d + policy: + phases: + delete: + min_age: 5d + actions: + delete: {} + test_empty: {} storage: s3: clients: