Update elasticsearch chart to work with Rook Ceph

When using Rook for managing Ceph we can use Rook CRDs to create S3 buckets and users. This PR adds bucket claim template to the elasticsearch chart. Rook creates a bucket for a bucket claim and also creates a secret containing the credentials to get access to this bucket. So we also add a snippet to expose these credentials via environment variables to containers where they are needed. Change-Id: Ic5cd35a5c64a914af97d2b3cfec21dbe399c0f14
2023-11-09 13:16:49 -06:00 · 2023-11-09 13:16:49 -06:00 · 7f783dba51
commit 7f783dba51
parent 145e9df9b7
16 changed files with 190 additions and 26 deletions
--- a/elasticsearch/Chart.yaml
+++ b/elasticsearch/Chart.yaml
@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v8.9.0
 description: OpenStack-Helm ElasticSearch
 name: elasticsearch
-version: 0.2.27
+version: 0.2.28
 home: https://www.elastic.co/
 sources:
  - https://github.com/elastic/elasticsearch
--- a/elasticsearch/templates/deployment-client.yaml
+++ b/elasticsearch/templates/deployment-client.yaml
@ -177,8 +177,12 @@ spec:
                  key: ELASTICSEARCH_PASSWORD
 {{- end }}
 {{- if .Values.conf.elasticsearch.snapshots.enabled }}
+{{- if .Values.manifests.object_bucket_claim }}
+{{- include "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" . | indent 12 }}
+{{- else }}
 {{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
 {{- end }}
+{{- end }}
 {{- if .Values.pod.env.client }}
 {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.client | indent 12 }}
 {{- end }}
--- a/elasticsearch/templates/deployment-gateway.yaml
+++ b/elasticsearch/templates/deployment-gateway.yaml
@ -119,8 +119,12 @@ spec:
                  key: ELASTICSEARCH_PASSWORD
 {{- end }}
 {{- if .Values.conf.elasticsearch.snapshots.enabled }}
+{{- if .Values.manifests.object_bucket_claim }}
+{{- include "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" . | indent 12 }}
+{{- else }}
 {{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
 {{- end }}
+{{- end }}
 {{- if .Values.pod.env.gateway }}
 {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.gateway | indent 12 }}
 {{- end }}
--- a/elasticsearch/templates/object-bucket-claim.yaml
+++ b/elasticsearch/templates/object-bucket-claim.yaml
@ -0,0 +1,29 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and (.Values.manifests.object_bucket_claim) (.Values.conf.elasticsearch.snapshots.enabled) }}
+{{- range $bucket := .Values.storage.s3.buckets }}
+# When using this Rook CRD, not only bucket will be created,
+# but also a secret containing the credentials to access the bucket.
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: {{ $bucket.name }}
+spec:
+  bucketName: {{ $bucket.name }}
+  storageClassName: {{ $bucket.storage_class }}
+...
+{{- end -}}
+{{- end -}}
--- a/elasticsearch/templates/statefulset-data.yaml
+++ b/elasticsearch/templates/statefulset-data.yaml
@ -133,8 +133,12 @@ spec:
            - name: DISCOVERY_SERVICE
              value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
 {{- if .Values.conf.elasticsearch.snapshots.enabled }}
+{{- if .Values.manifests.object_bucket_claim }}
+{{- include "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" . | indent 12 }}
+{{- else }}
 {{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
 {{- end }}
+{{- end }}
 {{- if .Values.pod.env.data }}
 {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.data | indent 12 }}
 {{- end }}
--- a/elasticsearch/templates/statefulset-master.yaml
+++ b/elasticsearch/templates/statefulset-master.yaml
@ -127,8 +127,12 @@ spec:
                  key: ELASTICSEARCH_PASSWORD
 {{- end }}
 {{- if .Values.conf.elasticsearch.snapshots.enabled }}
+{{- if .Values.manifests.object_bucket_claim }}
+{{- include "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" . | indent 12 }}
+{{- else }}
 {{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
 {{- end }}
+{{- end }}
 {{- if .Values.pod.env.master }}
 {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.master | indent 12 }}
 {{- end }}
--- a/elasticsearch/values.yaml
+++ b/elasticsearch/values.yaml
@ -978,4 +978,5 @@ manifests:
  service_logging: true
  statefulset_data: true
  statefulset_master: true
+  object_bucket_claim: false
 ...
--- a/helm-toolkit/Chart.yaml
+++ b/helm-toolkit/Chart.yaml
@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Helm-Toolkit
 name: helm-toolkit
-version: 0.2.55
+version: 0.2.56
 home: https://docs.openstack.org/openstack-helm
 icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png
 sources:
--- a/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
+++ b/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl
@ -0,0 +1,28 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
+{{- range $s3Bucket := .Values.storage.s3.buckets }}
+- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_ACCESS_KEY_ID
+- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ $s3Bucket.name }}
+      key: AWS_SECRET_ACCESS_KEY
+{{- end }}
+{{- end }}
--- a/releasenotes/notes/elasticsearch.yaml
+++ b/releasenotes/notes/elasticsearch.yaml
@ -37,4 +37,5 @@ elasticsearch:
  - 0.2.25 Update ElasticSearch to 8.9.0
  - 0.2.26 Add 2023.1 Ubuntu Focal overrides
  - 0.2.27 Update Rook to 1.12.5 and Ceph to 18.2.0
+  - 0.2.28 Utilize bucket claim CRD when using with Rook
 ...
--- a/releasenotes/notes/helm-toolkit.yaml
+++ b/releasenotes/notes/helm-toolkit.yaml
@ -62,4 +62,5 @@ helm-toolkit:
  - 0.2.53 Update create db user queries
  - 0.2.54 Fix dependency resolver to ignore non-existing dependencyKey when dependencyMixinParam is a slice
  - 0.2.55 Updated deprecated IngressClass annotation
+  - 0.2.56 Expose S3 credentials from Rook bucket CRD secret
 ...
--- a/roles/deploy-env/tasks/control-plane.yaml
+++ b/roles/deploy-env/tasks/control-plane.yaml
@ -19,7 +19,7 @@
  shell: |
    mkdir -p /home/{{ kubectl.user }}/.kube
    cp -i /etc/kubernetes/admin.conf /home/{{ kubectl.user }}/.kube/config
-    chown {{ kubectl.user }}:{{ kubectl.group }} /home/{{ kubectl.user }}/.kube/config
+    chown -R {{ kubectl.user }}:{{ kubectl.group }} /home/{{ kubectl.user }}/.kube
  args:
    executable: /bin/bash

--- a/tools/deployment/ceph/ceph-rook.sh
+++ b/tools/deployment/ceph/ceph-rook.sh
@ -600,6 +600,8 @@ cephObjectStores:
  - name: default
    namespace: ceph
    spec:
+      allowUsersInNamespaces:
+        - "*"
      metadataPool:
        failureDomain: host
        replicated:
@ -693,18 +695,40 @@ bootstrap:
 manifests:
  daemonset_mon: false
  daemonset_osd: false
-  deployment_checkdns: true
  deployment_mds: false
  deployment_mgr: false
  deployment_mgr_sa: false
  deployment_moncheck: false
  helm_tests: false
  job_bootstrap: false
-  job_storage_admin_keys: true
  service_mgr: false
  service_mon: false
  service_mon_discovery: true
+  job_storage_admin_keys: true
+  job_keyring: true
 EOF

 helm upgrade --install ceph-mon ./ceph-mon --namespace=ceph --values=/tmp/ceph-supplemental.yaml
 ./tools/deployment/common/wait-for-pods.sh ceph
+
+# credentials for this object store user will be placed
+# to the rook-ceph-object-user-default-s3-admin secret
+# AccessKey is the secret field where the access key is stored
+# SecretKey is the secret field where the secret key is stored
+# cat > /tmp/s3_admin.yaml <<EOF
+# apiVersion: ceph.rook.io/v1
+# kind: CephObjectStoreUser
+# metadata:
+#   name: s3-admin
+#   namespace: osh-infra
+# spec:
+#   store: default
+#   clusterNamespace: ceph
+#   # this is what is passed to radosgw-admin as uid argument
+#   displayName: s3_admin
+#   capabilities:
+#     user: "*"
+#     bucket: "*"
+# EOF
+
+# kubectl apply -f /tmp/s3_admin.yaml
--- a/tools/deployment/osh-infra-logging/050-elasticsearch.sh
+++ b/tools/deployment/osh-infra-logging/050-elasticsearch.sh
@ -68,38 +68,103 @@ storage:
    clients:
      # These values configure the s3 clients section of elasticsearch.yml, with access_key and secret_key being saved to the keystore
      default:
-        auth:
-          username: elasticsearch
-          access_key: "elastic_access_key"
-          secret_key: "elastic_secret_key"
+        # not needed when using Rook Ceph CRDs
+        # auth:
+        #   username: elasticsearch
+        #   access_key: "elastic_access_key"
+        #   secret_key: "elastic_secret_key"
        settings:
          # endpoint: Defaults to the ceph-rgw endpoint
          # protocol: Defaults to http
          path_style_access: true # Required for ceph-rgw S3 API
        create_user: true # Attempt to create the user at the ceph_object_store endpoint, authenticating using the secret named at .Values.secrets.rgw.admin
      backup: # Change this as you'd like
-        auth:
-          username: backup
-          access_key: "backup_access_key"
-          secret_key: "backup_secret_key"
+        # not needed when using Rook Ceph CRDs
+        # auth:
+        #   username: backup
+        #   access_key: "backup_access_key"
+        #   secret_key: "backup_secret_key"
        settings:
-          endpoint: radosgw.osh-infra.svc.cluster.local # Using the ingress here to test the endpoint override
+          # endpoint: rook-ceph-rgw-default.ceph.svc.cluster.local # Using the ingress here to test the endpoint override
          path_style_access: true
        create_user: true
    buckets: # List of buckets to create (if required).
      - name: elasticsearch-bucket
        client: default
-        options: # list of extra options for s3cmd
-          - --region="default:osh-infra"
+        storage_class: ceph-bucket # this is valid when using Rook CRDs
+        # not needed when using Rook Ceph CRDs
+        # options: # list of extra options for s3cmd
+        #   - --region="default:osh-infra"
      - name: backup-bucket
        client: backup
-        options: # list of extra options for s3cmd
-          - --region="default:backup"
+        storage_class: ceph-bucket # this is valid when using Rook CRDs
+        # not needed when using Rook Ceph CRDs
+        # options: # list of extra options for s3cmd
+        #   - --region="default:backup"
+endpoints:
+  ceph_object_store:
+    name: radosgw
+    namespace: ceph
+    hosts:
+      default: rook-ceph-rgw-default
+      public: radosgw
+    host_fqdn_override:
+      default: null
+    path:
+      default: null
+    scheme:
+      default: http
+    port:
+      api:
+        default: 8080
+        public: 80
 network:
  elasticsearch:
    ingress:
      classes:
        namespace: nginx-osh-infra
+dependencies:
+  static:
+    elasticsearch_templates:
+      services:
+        - endpoint: internal
+          service: elasticsearch
+      jobs: null
+      custom_resources:
+        - apiVersion: objectbucket.io/v1alpha1
+          kind: ObjectBucket
+          name: obc-osh-infra-elasticsearch-bucket
+          fields:
+            - key: "status.phase"
+              value: "Bound"
+        - apiVersion: objectbucket.io/v1alpha1
+          kind: ObjectBucket
+          name: obc-osh-infra-backup-bucket
+          fields:
+            - key: "status.phase"
+              value: "Bound"
+    snapshot_repository:
+      services:
+        - endpoint: internal
+          service: elasticsearch
+      jobs: null
+      custom_resources:
+        - apiVersion: objectbucket.io/v1alpha1
+          kind: ObjectBucket
+          name: obc-osh-infra-elasticsearch-bucket
+          fields:
+            - key: "status.phase"
+              value: "Bound"
+        - apiVersion: objectbucket.io/v1alpha1
+          kind: ObjectBucket
+          name: obc-osh-infra-backup-bucket
+          fields:
+            - key: "status.phase"
+              value: "Bound"
+manifests:
+  job_s3_user: false
+  job_s3_bucket: false
+  object_bucket_claim: true
 EOF

 : ${OSH_INFRA_EXTRA_HELM_ARGS_ELASTICSEARCH:="$(./tools/deployment/common/get-values-overrides.sh elasticsearch)"}
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@ -75,7 +75,7 @@
      - ^.*\.rst$
      - ^doc/.*$
      - ^releasenotes/.*$
-    timeout: 7200
+    timeout: 10800
    pre-run:
      - playbooks/prepare-hosts.yaml
      - playbooks/mount-volumes.yaml
@ -122,9 +122,8 @@
      gate_scripts:
        - ./tools/deployment/osh-infra-logging/000-prepare-k8s.sh
        - ./tools/deployment/osh-infra-logging/010-ingress.sh
-        - ./tools/deployment/ceph/ceph.sh
+        - ./tools/deployment/ceph/ceph-rook.sh
        - ./tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh
-        - ./tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh
        - ./tools/deployment/osh-infra-logging/040-ldap.sh
        - ./tools/deployment/osh-infra-logging/050-elasticsearch.sh
        - ./tools/deployment/osh-infra-logging/060-fluentd.sh
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@ -23,11 +23,11 @@
        - openstack-helm-lint-osh
        - openstack-helm-infra-bandit
        - openstack-helm-infra-logging
-        - openstack-helm-infra-monitoring
-        - openstack-helm-infra-openstack-support
-        - openstack-helm-infra-openstack-support-rook
-        - openstack-helm-infra-openstack-support-ssl
-        - openstack-helm-infra-metacontroller
+        # - openstack-helm-infra-monitoring
+        # - openstack-helm-infra-openstack-support
+        # - openstack-helm-infra-openstack-support-rook
+        # - openstack-helm-infra-openstack-support-ssl
+        # - openstack-helm-infra-metacontroller
    gate:
      jobs:
        - openstack-helm-lint