Helm-toolkit: Update job for creating s3 buckets

This updates the helm-toolkit manifest template and scipts for
creating an S3 bucket and linking it to a user. This moves away
from the previous python implementation that used rgwadmin, and
instead uses s3cmd for a cleaner approach that can support more
recent versions of ceph

Change-Id: I305062a5daa063bfe21a12448d7a3957bca00bf4
This commit is contained in:
Steve Wilkerson 2019-01-04 16:25:57 -06:00
parent 0b66795342
commit 8180635733
5 changed files with 52 additions and 101 deletions

View File

@ -30,7 +30,7 @@ data:
{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
ceph-admin-keyring.sh: | ceph-admin-keyring.sh: |
{{ tuple "bin/_ceph-admin-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{ tuple "bin/_ceph-admin-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
create-s3-bucket.py: | create-s3-bucket.sh: |
{{- include "helm-toolkit.scripts.create_s3_bucket" . | indent 4 }} {{- include "helm-toolkit.scripts.create_s3_bucket" . | indent 4 }}
create-s3-user.sh: | create-s3-user.sh: |
{{- include "helm-toolkit.scripts.create_s3_user" . | indent 4 }} {{- include "helm-toolkit.scripts.create_s3_user" . | indent 4 }}

View File

@ -23,7 +23,7 @@ images:
curator: docker.io/bobrik/curator:5.2.0 curator: docker.io/bobrik/curator:5.2.0
elasticsearch: docker.io/srwilkers/elasticsearch-s3:v0.1.0 elasticsearch: docker.io/srwilkers/elasticsearch-s3:v0.1.0
ceph_key_placement: docker.io/port/ceph-config-helper:v1.10.3 ceph_key_placement: docker.io/port/ceph-config-helper:v1.10.3
s3_bucket: docker.io/port/ceph-config-helper:v1.10.3 s3_bucket: docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04
s3_user: docker.io/port/ceph-config-helper:v1.10.3 s3_user: docker.io/port/ceph-config-helper:v1.10.3
helm_tests: docker.io/openstackhelm/heat:newton helm_tests: docker.io/openstackhelm/heat:newton
prometheus_elasticsearch_exporter: docker.io/justwatch/elasticsearch_exporter:1.0.1 prometheus_elasticsearch_exporter: docker.io/justwatch/elasticsearch_exporter:1.0.1

View File

@ -56,7 +56,7 @@ spec:
imagePullPolicy: {{ $envAll.Values.images.pull_policy }} imagePullPolicy: {{ $envAll.Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.jobs.s3_bucket | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} {{ tuple $envAll $envAll.Values.pod.resources.jobs.s3_bucket | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
command: command:
- /tmp/create-s3-bucket.py - /tmp/create-s3-bucket.sh
env: env:
{{- with $env := dict "s3AdminSecret" $envAll.Values.secrets.rgw.admin }} {{- with $env := dict "s3AdminSecret" $envAll.Values.secrets.rgw.admin }}
{{- include "helm-toolkit.snippets.rgw_s3_admin_env_vars" $env | indent 12 }} {{- include "helm-toolkit.snippets.rgw_s3_admin_env_vars" $env | indent 12 }}
@ -69,9 +69,9 @@ spec:
- name: RGW_HOST - name: RGW_HOST
value: {{ tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} value: {{ tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
volumeMounts: volumeMounts:
- name: s3-bucket-py - name: s3-bucket-sh
mountPath: /tmp/create-s3-bucket.py mountPath: /tmp/create-s3-bucket.sh
subPath: create-s3-bucket.py subPath: create-s3-bucket.sh
readOnly: true readOnly: true
- name: etcceph - name: etcceph
mountPath: /etc/ceph mountPath: /etc/ceph
@ -86,7 +86,7 @@ spec:
readOnly: true readOnly: true
{{ end }} {{ end }}
volumes: volumes:
- name: s3-bucket-py - name: s3-bucket-sh
configMap: configMap:
name: {{ $configMapBin | quote }} name: {{ $configMapBin | quote }}
defaultMode: 0555 defaultMode: 0555

View File

@ -1,94 +0,0 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- define "helm-toolkit.scripts.create_s3_bucket" }}
#!/usr/bin/env python
import os
import sys
import logging
import rgwadmin
import rgwadmin.exceptions
# Create logger, console handler and formatter
logger = logging.getLogger('OpenStack-Helm S3 Bucket')
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# Set the formatter and add the handler
ch.setFormatter(formatter)
logger.addHandler(ch)
# Get S3 admin user's access key
if "S3_ADMIN_ACCESS_KEY" in os.environ:
access_key = os.environ['S3_ADMIN_ACCESS_KEY']
logger.info('Found S3 admin access key')
else:
logger.critical('S3 admin access key environment variable not set')
sys.exit(1)
# Get S3 admin user's secret key
if "S3_ADMIN_SECRET_KEY" in os.environ:
secret_key = os.environ['S3_ADMIN_SECRET_KEY']
logger.info('Found S3 admin secret key')
else:
logger.critical('S3 admin secret key environment variable not set')
sys.exit(1)
# Get RGW S3 host endpoint
if "RGW_HOST" in os.environ:
server = os.environ['RGW_HOST']
logger.info('Found RGW S3 host endpoint')
else:
logger.critical('RGW S3 host endpoint environment variable not set')
sys.exit(1)
# Get name of S3 user to link to bucket
if "S3_USERNAME" in os.environ:
s3_user = os.environ['S3_USERNAME']
logger.info('Found S3 user name')
else:
logger.critical('S3 user name environment variable not set')
sys.exit(1)
# Get name of bucket to create for user link
if "S3_BUCKET" in os.environ:
s3_bucket = os.environ['S3_BUCKET']
logger.info('Found S3 bucket name')
else:
logger.critical('S3 bucket name environment variable not set')
sys.exit(1)
try:
rgw_admin = rgwadmin.RGWAdmin(access_key, secret_key, server, secure=False)
try:
rgw_admin.get_bucket(bucket=s3_bucket,uid=s3_user)
except (rgwadmin.exceptions.NoSuchBucket, rgwadmin.exceptions.NoSuchKey), e:
rgw_admin.create_bucket(bucket=s3_bucket)
bucket = rgw_admin.get_bucket(bucket=s3_bucket)
bucket_id = bucket['id']
rgw_admin.link_bucket(bucket=s3_bucket, bucket_id=bucket_id, uid=s3_user)
logger.info("Created bucket {} and linked it to user {}".format(s3_bucket, s3_user))
sys.exit(0)
else:
logger.info("The bucket {} exists for user {}! Exiting without creating a new bucket!".format(s3_bucket, s3_user))
except rgwadmin.exceptions.InvalidArgument:
logger.critical("Invalid arguments supplied for rgwadmin connection. Please check your s3 keys and endpoint")
sys.exit(1)
{{- end }}

View File

@ -0,0 +1,45 @@
{{/*
Copyright 2017 The Openstack-Helm Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- define "helm-toolkit.scripts.create_s3_bucket" }}
#!/bin/bash
function create_rgw_s3_bucket ()
{
create_bucket=$(s3cmd mb s3://$S3_BUCKET --host=$RGW_HOST --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY --no-encrypt --no-check-certificate)
if [ $? -eq 0 ]; then
echo "Bucket $S3_BUCKET created"
else
echo "Error trying to create bucket $S3_BUCKET"
exit 1
fi
}
function modify_bucket_acl ()
{
modify_acl=$(s3cmd setacl s3://$S3_BUCKET --host=$RGW_HOST --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY --no-encrypt --no-check-certificate --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME)
if [ $? -eq 0 ]; then
echo "Bucket $S3_BUCKET ACL updated"
else
echo "Error trying to update bucket $S3_BUCKET ACL"
exit 1
fi
}
create_rgw_s3_bucket
modify_bucket_acl
{{- end }}