openstack
/
openstack-helm-infra
Code Issues Proposed changes

Update elasticsearch chart to work with Rook Ceph 

When using Rook for managing Ceph we can use
Rook CRDs to create S3 buckets and users.

This PR adds bucket claim template to the
elasticsearch chart. Rook creates a bucket for
a bucket claim and also creates a secret
containing the credentials to get access to this
bucket. So we also add a snippet to expose
these credentials via environment variables to
containers where they are needed.

Change-Id: Ic5cd35a5c64a914af97d2b3cfec21dbe399c0f14
This commit is contained in:
Vladimir Kozhukalov 2023-11-09 13:16:49 -06:00
parent 145e9df9b7
commit 7f783dba51
16 changed files with 190 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
elasticsearch/Chart.yaml
View File

@ -15,7 +15,7 @@ apiVersion: v1
appVersion: v8.9.0
description: OpenStack-Helm ElasticSearch
name: elasticsearch
version: 0.2.27
version: 0.2.28
home: https://www.elastic.co/
sources:
- https://github.com/elastic/elasticsearch

4
elasticsearch/templates/deployment-client.yaml
View File

@ -177,8 +177,12 @@ spec:
key: ELASTICSEARCH_PASSWORD
{{- end }}
{{- if .Values.conf.elasticsearch.snapshots.enabled }}
{{- if .Values.manifests.object_bucket_claim }}
{{- include "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" . | indent 12 }}
{{- else }}
{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
{{- end }}
{{- end }}
{{- if .Values.pod.env.client }}
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.client | indent 12 }}
{{- end }}

4
elasticsearch/templates/deployment-gateway.yaml
View File

@ -119,8 +119,12 @@ spec:
key: ELASTICSEARCH_PASSWORD
{{- end }}
{{- if .Values.conf.elasticsearch.snapshots.enabled }}
{{- if .Values.manifests.object_bucket_claim }}
{{- include "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" . | indent 12 }}
{{- else }}
{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
{{- end }}
{{- end }}
{{- if .Values.pod.env.gateway }}
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.gateway | indent 12 }}
{{- end }}

29
elasticsearch/templates/object-bucket-claim.yaml Normal file
View File

@ -0,0 +1,29 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- if and (.Values.manifests.object_bucket_claim) (.Values.conf.elasticsearch.snapshots.enabled) }}
{{- range $bucket := .Values.storage.s3.buckets }}
# When using this Rook CRD, not only bucket will be created,
# but also a secret containing the credentials to access the bucket.
---
apiVersion: objectbucket.io/v1alpha1
kind: ObjectBucketClaim
metadata:
name: {{ $bucket.name }}
spec:
bucketName: {{ $bucket.name }}
storageClassName: {{ $bucket.storage_class }}
...
{{- end -}}
{{- end -}}

4
elasticsearch/templates/statefulset-data.yaml
View File

@ -133,8 +133,12 @@ spec:
- name: DISCOVERY_SERVICE
value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
{{- if .Values.conf.elasticsearch.snapshots.enabled }}
{{- if .Values.manifests.object_bucket_claim }}
{{- include "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" . | indent 12 }}
{{- else }}
{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
{{- end }}
{{- end }}
{{- if .Values.pod.env.data }}
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.data | indent 12 }}
{{- end }}

4
elasticsearch/templates/statefulset-master.yaml
View File

@ -127,8 +127,12 @@ spec:
key: ELASTICSEARCH_PASSWORD
{{- end }}
{{- if .Values.conf.elasticsearch.snapshots.enabled }}
{{- if .Values.manifests.object_bucket_claim }}
{{- include "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" . | indent 12 }}
{{- else }}
{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }}
{{- end }}
{{- end }}
{{- if .Values.pod.env.master }}
{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.master | indent 12 }}
{{- end }}

1
elasticsearch/values.yaml
View File

@ -978,4 +978,5 @@ manifests:
service_logging: true
statefulset_data: true
statefulset_master: true
object_bucket_claim: false
...

2
helm-toolkit/Chart.yaml
View File

@ -15,7 +15,7 @@ apiVersion: v1
appVersion: v1.0.0
description: OpenStack-Helm Helm-Toolkit
name: helm-toolkit
version: 0.2.55
version: 0.2.56
home: https://docs.openstack.org/openstack-helm
icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png
sources:

28
helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl Normal file
View File

@ -0,0 +1,28 @@
{{/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }}
{{- range $s3Bucket := .Values.storage.s3.buckets }}
- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
valueFrom:
secretKeyRef:
name: {{ $s3Bucket.name }}
key: AWS_ACCESS_KEY_ID
- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }}
valueFrom:
secretKeyRef:
name: {{ $s3Bucket.name }}
key: AWS_SECRET_ACCESS_KEY
{{- end }}
{{- end }}

1
releasenotes/notes/elasticsearch.yaml
View File

@ -37,4 +37,5 @@ elasticsearch:
- 0.2.25 Update ElasticSearch to 8.9.0
- 0.2.26 Add 2023.1 Ubuntu Focal overrides
- 0.2.27 Update Rook to 1.12.5 and Ceph to 18.2.0
- 0.2.28 Utilize bucket claim CRD when using with Rook
...

1
releasenotes/notes/helm-toolkit.yaml
View File

@ -62,4 +62,5 @@ helm-toolkit:
- 0.2.53 Update create db user queries
- 0.2.54 Fix dependency resolver to ignore non-existing dependencyKey when dependencyMixinParam is a slice
- 0.2.55 Updated deprecated IngressClass annotation
- 0.2.56 Expose S3 credentials from Rook bucket CRD secret
...

2
roles/deploy-env/tasks/control-plane.yaml
View File

@ -19,7 +19,7 @@
shell: |
mkdir -p /home/{{ kubectl.user }}/.kube
cp -i /etc/kubernetes/admin.conf /home/{{ kubectl.user }}/.kube/config
chown {{ kubectl.user }}:{{ kubectl.group }} /home/{{ kubectl.user }}/.kube/config
chown -R {{ kubectl.user }}:{{ kubectl.group }} /home/{{ kubectl.user }}/.kube
args:
executable: /bin/bash

28
tools/deployment/ceph/ceph-rook.sh
View File

@ -600,6 +600,8 @@ cephObjectStores:
- name: default
namespace: ceph
spec:
allowUsersInNamespaces:
- "*"
metadataPool:
failureDomain: host
replicated:
@ -693,18 +695,40 @@ bootstrap:
manifests:
daemonset_mon: false
daemonset_osd: false
deployment_checkdns: true
deployment_mds: false
deployment_mgr: false
deployment_mgr_sa: false
deployment_moncheck: false
helm_tests: false
job_bootstrap: false
job_storage_admin_keys: true
service_mgr: false
service_mon: false
service_mon_discovery: true
job_storage_admin_keys: true
job_keyring: true
EOF
helm upgrade --install ceph-mon ./ceph-mon --namespace=ceph --values=/tmp/ceph-supplemental.yaml
./tools/deployment/common/wait-for-pods.sh ceph
# credentials for this object store user will be placed
# to the rook-ceph-object-user-default-s3-admin secret
# AccessKey is the secret field where the access key is stored
# SecretKey is the secret field where the secret key is stored
# cat > /tmp/s3_admin.yaml <<EOF
# apiVersion: ceph.rook.io/v1
# kind: CephObjectStoreUser
# metadata:
# name: s3-admin
# namespace: osh-infra
# spec:
# store: default
# clusterNamespace: ceph
# # this is what is passed to radosgw-admin as uid argument
# displayName: s3_admin
# capabilities:
# user: "*"
# bucket: "*"
# EOF
# kubectl apply -f /tmp/s3_admin.yaml

91
tools/deployment/osh-infra-logging/050-elasticsearch.sh
View File

@ -68,38 +68,103 @@ storage:
clients:
# These values configure the s3 clients section of elasticsearch.yml, with access_key and secret_key being saved to the keystore
default:
auth:
username: elasticsearch
access_key: "elastic_access_key"
secret_key: "elastic_secret_key"
# not needed when using Rook Ceph CRDs
# auth:
# username: elasticsearch
# access_key: "elastic_access_key"
# secret_key: "elastic_secret_key"
settings:
# endpoint: Defaults to the ceph-rgw endpoint
# protocol: Defaults to http
path_style_access: true # Required for ceph-rgw S3 API
create_user: true # Attempt to create the user at the ceph_object_store endpoint, authenticating using the secret named at .Values.secrets.rgw.admin
backup: # Change this as you'd like
auth:
username: backup
access_key: "backup_access_key"
secret_key: "backup_secret_key"
# not needed when using Rook Ceph CRDs
# auth:
# username: backup
# access_key: "backup_access_key"
# secret_key: "backup_secret_key"
settings:
endpoint: radosgw.osh-infra.svc.cluster.local # Using the ingress here to test the endpoint override
# endpoint: rook-ceph-rgw-default.ceph.svc.cluster.local # Using the ingress here to test the endpoint override
path_style_access: true
create_user: true
buckets: # List of buckets to create (if required).
- name: elasticsearch-bucket
client: default
options: # list of extra options for s3cmd
- --region="default:osh-infra"
storage_class: ceph-bucket # this is valid when using Rook CRDs
# not needed when using Rook Ceph CRDs
# options: # list of extra options for s3cmd
# - --region="default:osh-infra"
- name: backup-bucket
client: backup
options: # list of extra options for s3cmd
- --region="default:backup"
storage_class: ceph-bucket # this is valid when using Rook CRDs
# not needed when using Rook Ceph CRDs
# options: # list of extra options for s3cmd
# - --region="default:backup"
endpoints:
ceph_object_store:
name: radosgw
namespace: ceph
hosts:
default: rook-ceph-rgw-default
public: radosgw
host_fqdn_override:
default: null
path:
default: null
scheme:
default: http
port:
api:
default: 8080
public: 80
network:
elasticsearch:
ingress:
classes:
namespace: nginx-osh-infra
dependencies:
static:
elasticsearch_templates:
services:
- endpoint: internal
service: elasticsearch
jobs: null
custom_resources:
- apiVersion: objectbucket.io/v1alpha1
kind: ObjectBucket
name: obc-osh-infra-elasticsearch-bucket
fields:
- key: "status.phase"
value: "Bound"
- apiVersion: objectbucket.io/v1alpha1
kind: ObjectBucket
name: obc-osh-infra-backup-bucket
fields:
- key: "status.phase"
value: "Bound"
snapshot_repository:
services:
- endpoint: internal
service: elasticsearch
jobs: null
custom_resources:
- apiVersion: objectbucket.io/v1alpha1
kind: ObjectBucket
name: obc-osh-infra-elasticsearch-bucket
fields:
- key: "status.phase"
value: "Bound"
- apiVersion: objectbucket.io/v1alpha1
kind: ObjectBucket
name: obc-osh-infra-backup-bucket
fields:
- key: "status.phase"
value: "Bound"
manifests:
job_s3_user: false
job_s3_bucket: false
object_bucket_claim: true
EOF
: ${OSH_INFRA_EXTRA_HELM_ARGS_ELASTICSEARCH:="$(./tools/deployment/common/get-values-overrides.sh elasticsearch)"}

5
zuul.d/jobs.yaml
View File

@ -75,7 +75,7 @@
- ^.*\.rst$
- ^doc/.*$
- ^releasenotes/.*$
timeout: 7200
timeout: 10800
pre-run:
- playbooks/prepare-hosts.yaml
- playbooks/mount-volumes.yaml
@ -122,9 +122,8 @@
gate_scripts:
- ./tools/deployment/osh-infra-logging/000-prepare-k8s.sh
- ./tools/deployment/osh-infra-logging/010-ingress.sh
- ./tools/deployment/ceph/ceph.sh
- ./tools/deployment/ceph/ceph-rook.sh
- ./tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh
- ./tools/deployment/osh-infra-logging/030-radosgw-osh-infra.sh
- ./tools/deployment/osh-infra-logging/040-ldap.sh
- ./tools/deployment/osh-infra-logging/050-elasticsearch.sh
- ./tools/deployment/osh-infra-logging/060-fluentd.sh

10
zuul.d/project.yaml
View File

@ -23,11 +23,11 @@
- openstack-helm-lint-osh
- openstack-helm-infra-bandit
- openstack-helm-infra-logging
- openstack-helm-infra-monitoring
- openstack-helm-infra-openstack-support
- openstack-helm-infra-openstack-support-rook
- openstack-helm-infra-openstack-support-ssl
- openstack-helm-infra-metacontroller
# - openstack-helm-infra-monitoring
# - openstack-helm-infra-openstack-support
# - openstack-helm-infra-openstack-support-rook
# - openstack-helm-infra-openstack-support-ssl
# - openstack-helm-infra-metacontroller
gate:
jobs:
- openstack-helm-lint