From a3f444299e2f0a73fc039235b0fc955568aec9f2 Mon Sep 17 00:00:00 2001 From: Steve Wilkerson Date: Wed, 1 Aug 2018 15:05:56 -0500 Subject: [PATCH] HTK: Add s3 user/bucket scripts, snippets, manifests This proposes adding the following: Snippets for the environment variables for the s3 admin user and service users for using rgw's s3 api Scripts for creating s3 users for use by a particular service and for creating and linking buckets to those users Manifest templates for the jobs for creating the s3 users and for creating and linking the buckets to those users Change-Id: Ibd5ed0aac49d172c56faffdacd44bdd487978570 --- .../manifests/_job-s3-bucket.yaml.tpl | 104 +++++++++++++++ .../templates/manifests/_job-s3-user.yaml.tpl | 123 ++++++++++++++++++ .../scripts/_create-s3-bucket.py.tpl | 94 +++++++++++++ .../templates/scripts/_create-s3-user.sh.tpl | 55 ++++++++ .../snippets/_rgw_s3_admin_env_vars.tpl | 34 +++++ .../snippets/_rgw_s3_secret_creds.tpl | 24 ++++ .../snippets/_rgw_s3_user_env_vars.tpl | 34 +++++ 7 files changed, 468 insertions(+) create mode 100644 helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl create mode 100644 helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl create mode 100644 helm-toolkit/templates/scripts/_create-s3-bucket.py.tpl create mode 100644 helm-toolkit/templates/scripts/_create-s3-user.sh.tpl create mode 100644 helm-toolkit/templates/snippets/_rgw_s3_admin_env_vars.tpl create mode 100644 helm-toolkit/templates/snippets/_rgw_s3_secret_creds.tpl create mode 100644 helm-toolkit/templates/snippets/_rgw_s3_user_env_vars.tpl diff --git a/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl new file mode 100644 index 000000000..e3d3e67e0 --- /dev/null +++ b/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl @@ -0,0 +1,104 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This function creates a manifest for linking an s3 bucket to an s3 user. +# It can be used in charts dict created similar to the following: +# {- $s3BucketJob := dict "envAll" . "serviceName" "elasticsearch" } +# { $s3BucketJob | include "helm-toolkit.manifests.job_s3_bucket" } + +{{- define "helm-toolkit.manifests.job_s3_bucket" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +{{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}} +{{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}} +{{- $configMapCeph := index . "configMapCeph" | default (printf "ceph-etc" ) -}} +{{- $serviceNamePretty := $serviceName | replace "_" "-" -}} +{{- $s3UserSecret := index $envAll.Values.secrets.rgw $serviceName -}} +{{- $s3Bucket := index . "s3Bucket" | default $serviceName }} + +{{- $serviceAccountName := printf "%s-%s" $serviceNamePretty "s3-bucket" }} +{{ tuple $envAll "s3_bucket" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-%s" $serviceNamePretty "s3-bucket" | quote }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll $serviceName "s3-bucket" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName | quote }} + restartPolicy: OnFailure + nodeSelector: +{{ toYaml $nodeSelector | indent 8 }} + initContainers: +{{ tuple $envAll "s3_bucket" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: s3-bucket + image: {{ $envAll.Values.images.tags.s3_bucket }} + imagePullPolicy: {{ $envAll.Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.s3_bucket | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/create-s3-bucket.py + env: +{{- with $env := dict "s3AdminSecret" $envAll.Values.secrets.rgw.admin }} +{{- include "helm-toolkit.snippets.rgw_s3_admin_env_vars" $env | indent 12 }} +{{- end }} +{{- with $env := dict "s3UserSecret" $s3UserSecret }} +{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" $env | indent 12 }} +{{- end }} + - name: S3_BUCKET + value: {{ $s3Bucket }} + - name: RGW_HOST + value: {{ tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + volumeMounts: + - name: s3-bucket-py + mountPath: /tmp/create-s3-bucket.py + subPath: create-s3-bucket.py + readOnly: true + - name: etcceph + mountPath: /etc/ceph + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + {{- if empty $envAll.Values.conf.ceph.admin_keyring }} + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{ end }} + volumes: + - name: s3-bucket-py + configMap: + name: {{ $configMapBin | quote }} + defaultMode: 0555 + - name: etcceph + emptyDir: {} + - name: ceph-etc + configMap: + name: {{ $configMapCeph | quote }} + defaultMode: 0444 + {{- if empty $envAll.Values.conf.ceph.admin_keyring }} + - name: ceph-keyring + secret: + secretName: pvc-ceph-client-key + {{ end }} +{{- end -}} diff --git a/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl new file mode 100644 index 000000000..6d2378ed4 --- /dev/null +++ b/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl @@ -0,0 +1,123 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This function creates a manifest for s3 user management. +# It can be used in charts dict created similar to the following: +# {- $s3UserJob := dict "envAll" . "serviceName" "elasticsearch" } +# { $s3UserJob | include "helm-toolkit.manifests.job_s3_user" } + +{{- define "helm-toolkit.manifests.job_s3_user" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +{{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}} +{{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}} +{{- $configMapCeph := index . "configMapCeph" | default (printf "ceph-etc" ) -}} +{{- $serviceNamePretty := $serviceName | replace "_" "-" -}} +{{- $s3UserSecret := index $envAll.Values.secrets.rgw $serviceName -}} + +{{- $serviceAccountName := printf "%s-%s" $serviceNamePretty "s3-user" }} +{{ tuple $envAll "s3_user" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-%s" $serviceNamePretty "s3-user" | quote }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll $serviceName "s3-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName | quote }} + restartPolicy: OnFailure + nodeSelector: +{{ toYaml $nodeSelector | indent 8 }} + initContainers: +{{ tuple $envAll "s3_user" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: ceph-keyring-placement + image: {{ $envAll.Values.images.tags.ceph_key_placement }} + imagePullPolicy: {{ $envAll.Values.images.pull_policy }} + command: + - /tmp/ceph-admin-keyring.sh + volumeMounts: + - name: etcceph + mountPath: /etc/ceph + - name: ceph-keyring-sh + mountPath: /tmp/ceph-admin-keyring.sh + subPath: ceph-admin-keyring.sh + readOnly: true + {{- if empty $envAll.Values.conf.ceph.admin_keyring }} + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{ end }} + containers: + - name: s3-user + image: {{ $envAll.Values.images.tags.s3_user }} + imagePullPolicy: {{ $envAll.Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.s3_user | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/create-s3-user.sh + env: +{{- with $env := dict "s3AdminSecret" $envAll.Values.secrets.rgw.admin }} +{{- include "helm-toolkit.snippets.rgw_s3_admin_env_vars" $env | indent 12 }} +{{- end }} +{{- with $env := dict "s3UserSecret" $s3UserSecret }} +{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" $env | indent 12 }} +{{- end }} + - name: RGW_HOST + value: {{ tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + volumeMounts: + - name: create-s3-user-sh + mountPath: /tmp/create-s3-user.sh + subPath: create-s3-user.sh + readOnly: true + - name: etcceph + mountPath: /etc/ceph + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + {{- if empty $envAll.Values.conf.ceph.admin_keyring }} + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{ end }} + volumes: + - name: create-s3-user-sh + configMap: + name: {{ $configMapBin | quote }} + defaultMode: 0555 + - name: ceph-keyring-sh + configMap: + name: {{ $configMapBin | quote }} + defaultMode: 0555 + - name: etcceph + emptyDir: {} + - name: ceph-etc + configMap: + name: {{ $configMapCeph | quote }} + defaultMode: 0444 + {{- if empty $envAll.Values.conf.ceph.admin_keyring }} + - name: ceph-keyring + secret: + secretName: pvc-ceph-client-key + {{ end }} +{{- end -}} diff --git a/helm-toolkit/templates/scripts/_create-s3-bucket.py.tpl b/helm-toolkit/templates/scripts/_create-s3-bucket.py.tpl new file mode 100644 index 000000000..643fe9160 --- /dev/null +++ b/helm-toolkit/templates/scripts/_create-s3-bucket.py.tpl @@ -0,0 +1,94 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.scripts.create_s3_bucket" }} +#!/usr/bin/env python + +import os +import sys +import logging +import rgwadmin +import rgwadmin.exceptions + +# Create logger, console handler and formatter +logger = logging.getLogger('OpenStack-Helm S3 Bucket') +logger.setLevel(logging.DEBUG) +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + +# Set the formatter and add the handler +ch.setFormatter(formatter) +logger.addHandler(ch) + +# Get S3 admin user's access key +if "S3_ADMIN_ACCESS_KEY" in os.environ: + access_key = os.environ['S3_ADMIN_ACCESS_KEY'] + logger.info('Found S3 admin access key') +else: + logger.critical('S3 admin access key environment variable not set') + sys.exit(1) + +# Get S3 admin user's secret key +if "S3_ADMIN_SECRET_KEY" in os.environ: + secret_key = os.environ['S3_ADMIN_SECRET_KEY'] + logger.info('Found S3 admin secret key') +else: + logger.critical('S3 admin secret key environment variable not set') + sys.exit(1) + +# Get RGW S3 host endpoint +if "RGW_HOST" in os.environ: + server = os.environ['RGW_HOST'] + logger.info('Found RGW S3 host endpoint') +else: + logger.critical('RGW S3 host endpoint environment variable not set') + sys.exit(1) + +# Get name of S3 user to link to bucket +if "S3_USERNAME" in os.environ: + s3_user = os.environ['S3_USERNAME'] + logger.info('Found S3 user name') +else: + logger.critical('S3 user name environment variable not set') + sys.exit(1) + +# Get name of bucket to create for user link +if "S3_BUCKET" in os.environ: + s3_bucket = os.environ['S3_BUCKET'] + logger.info('Found S3 bucket name') +else: + logger.critical('S3 bucket name environment variable not set') + sys.exit(1) + +try: + rgw_admin = rgwadmin.RGWAdmin(access_key, secret_key, server, secure=False) + try: + rgw_admin.get_bucket(bucket=s3_bucket,uid=s3_user) + except (rgwadmin.exceptions.NoSuchBucket, rgwadmin.exceptions.NoSuchKey), e: + rgw_admin.create_bucket(bucket=s3_bucket) + bucket = rgw_admin.get_bucket(bucket=s3_bucket) + bucket_id = bucket['id'] + rgw_admin.link_bucket(bucket=s3_bucket, bucket_id=bucket_id, uid=s3_user) + logger.info("Created bucket {} and linked it to user {}".format(s3_bucket, s3_user)) + sys.exit(0) + else: + logger.info("The bucket {} exists for user {}! Exiting without creating a new bucket!".format(s3_bucket, s3_user)) +except rgwadmin.exceptions.InvalidArgument: + logger.critical("Invalid arguments supplied for rgwadmin connection. Please check your s3 keys and endpoint") + sys.exit(1) + +{{- end }} diff --git a/helm-toolkit/templates/scripts/_create-s3-user.sh.tpl b/helm-toolkit/templates/scripts/_create-s3-user.sh.tpl new file mode 100644 index 000000000..d1e0ea448 --- /dev/null +++ b/helm-toolkit/templates/scripts/_create-s3-user.sh.tpl @@ -0,0 +1,55 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.scripts.create_s3_user" }} +#!/bin/bash + +set -ex + +function create_admin_user () { + radosgw-admin user create \ + --uid=${S3_ADMIN_USERNAME} \ + --display-name=${S3_ADMIN_USERNAME} + + radosgw-admin caps add \ + --uid=${S3_ADMIN_USERNAME} \ + --caps={{ .Values.conf.ceph.radosgw.s3_admin_caps | quote }} + + radosgw-admin key create \ + --uid=${S3_ADMIN_USERNAME} \ + --key-type=s3 \ + --access-key ${S3_ADMIN_ACCESS_KEY} \ + --secret-key ${S3_ADMIN_SECRET_KEY} +} + +function create_s3_user () { + radosgw-admin user create \ + --uid=${S3_USERNAME} \ + --display-name=${S3_USERNAME} + + radosgw-admin key create \ + --uid=${S3_USERNAME} \ + --key-type=s3 \ + --access-key ${S3_ACCESS_KEY} \ + --secret-key ${S3_SECRET_KEY} +} + +radosgw-admin user stats --uid=${S3_ADMIN_USERNAME} || \ + create_admin_user + +radosgw-admin user stats --uid=${S3_USERNAME} || \ + create_s3_user +{{- end }} diff --git a/helm-toolkit/templates/snippets/_rgw_s3_admin_env_vars.tpl b/helm-toolkit/templates/snippets/_rgw_s3_admin_env_vars.tpl new file mode 100644 index 000000000..3ecbadeeb --- /dev/null +++ b/helm-toolkit/templates/snippets/_rgw_s3_admin_env_vars.tpl @@ -0,0 +1,34 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.snippets.rgw_s3_admin_env_vars" }} +{{- $s3AdminSecret := .s3AdminSecret }} +- name: S3_ADMIN_USERNAME + valueFrom: + secretKeyRef: + name: {{ $s3AdminSecret }} + key: S3_ADMIN_USERNAME +- name: S3_ADMIN_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ $s3AdminSecret }} + key: S3_ADMIN_ACCESS_KEY +- name: S3_ADMIN_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ $s3AdminSecret }} + key: S3_ADMIN_SECRET_KEY +{{- end }} diff --git a/helm-toolkit/templates/snippets/_rgw_s3_secret_creds.tpl b/helm-toolkit/templates/snippets/_rgw_s3_secret_creds.tpl new file mode 100644 index 000000000..688bf388e --- /dev/null +++ b/helm-toolkit/templates/snippets/_rgw_s3_secret_creds.tpl @@ -0,0 +1,24 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.snippets.rgw_s3_secret_creds" }} +{{- $userClass := index . 0 -}} +{{- $context := index . 1 -}} +{{- $userContext := index $context.Values.endpoints.ceph_object_store.auth $userClass }} +S3_USERNAME: {{ $userContext.username | b64enc }} +S3_ACCESS_KEY: {{ $userContext.access_key | b64enc }} +S3_SECRET_KEY: {{ $userContext.secret_key | b64enc }} +{{- end }} diff --git a/helm-toolkit/templates/snippets/_rgw_s3_user_env_vars.tpl b/helm-toolkit/templates/snippets/_rgw_s3_user_env_vars.tpl new file mode 100644 index 000000000..1bcd868b5 --- /dev/null +++ b/helm-toolkit/templates/snippets/_rgw_s3_user_env_vars.tpl @@ -0,0 +1,34 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.snippets.rgw_s3_user_env_vars" }} +{{- $s3UserSecret := .s3UserSecret }} +- name: S3_USERNAME + valueFrom: + secretKeyRef: + name: {{ $s3UserSecret }} + key: S3_USERNAME +- name: S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ $s3UserSecret }} + key: S3_ACCESS_KEY +- name: S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ $s3UserSecret }} + key: S3_SECRET_KEY +{{- end }}