From 19c16fdc2407e829c178ee6fa080029f5394721f Mon Sep 17 00:00:00 2001 From: okozachenko Date: Sat, 15 Aug 2020 20:30:21 +0300 Subject: [PATCH] Implement cinder Depends-On: https://review.opendev.org/747030 Change-Id: If8e45498efa9c7acfb8962e1fcd40accacf13a7a --- Makefile | 1 + chart/test-values.yaml | 1 + config/samples/operator-config.yaml | 1 + devstack/lib/cinder | 136 ++++++++++++++++++ devstack/lib/common | 7 +- devstack/plugin.sh | 9 +- devstack/settings | 1 + images/cinder/Dockerfile | 33 +++++ images/cinder/bindep.txt | 9 ++ images/cinder/build-requirements.txt | 1 + images/cinder/cinder-scheduler | 30 ++++ images/cinder/cinder-volume | 30 ++++ images/cinder/cinder-wsgi | 29 ++++ images/cinder/constraints.txt | 1 + images/cinder/requirements.txt | 8 ++ images/cinder/setup-repos.sh | 55 +++++++ openstack_operator/cinder.py | 86 +++++++++++ openstack_operator/identity.py | 10 +- openstack_operator/operator.py | 4 + .../templates/cinder/daemonset.yml.j2 | 120 ++++++++++++++++ .../templates/cinder/ingress.yml.j2 | 55 +++++++ .../templates/cinder/rabbitmq.yml.j2 | 27 ++++ .../templates/cinder/secret-rabbitmq.yml.j2 | 9 ++ .../templates/cinder/service.yml.j2 | 28 ++++ zuul.d/cinder-jobs.yaml | 51 +++++++ zuul.d/functional-jobs.yaml | 73 +++++----- 26 files changed, 775 insertions(+), 40 deletions(-) create mode 100644 devstack/lib/cinder create mode 100644 images/cinder/Dockerfile create mode 100644 images/cinder/bindep.txt create mode 100644 images/cinder/build-requirements.txt create mode 100755 images/cinder/cinder-scheduler create mode 100755 images/cinder/cinder-volume create mode 100755 images/cinder/cinder-wsgi create mode 100644 images/cinder/constraints.txt create mode 100644 images/cinder/requirements.txt create mode 100755 images/cinder/setup-repos.sh create mode 100644 openstack_operator/cinder.py create mode 100644 openstack_operator/templates/cinder/daemonset.yml.j2 create mode 100644 openstack_operator/templates/cinder/ingress.yml.j2 create mode 100644 openstack_operator/templates/cinder/rabbitmq.yml.j2 create mode 100644 openstack_operator/templates/cinder/secret-rabbitmq.yml.j2 create mode 100644 openstack_operator/templates/cinder/service.yml.j2 create mode 100644 zuul.d/cinder-jobs.yaml diff --git a/Makefile b/Makefile index 078b24b4..71a0291c 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ images: docker build images/horizon -t vexxhost/horizon:latest docker build images/keystone -t vexxhost/keystone:latest docker build images/ceilometer --target ceilometer-agent-notification -t vexxhost/ceilometer-agent-notification:latest + docker build images/cinder --target cinder-api -t vexxhost/cinder-api:latest docker build images/heat --target heat-api -t vexxhost/heat-api:latest docker build images/heat --target heat-api-cfn -t vexxhost/heat-api-cfn:latest docker build images/heat --target heat-engine -t vexxhost/heat-engine:latest diff --git a/chart/test-values.yaml b/chart/test-values.yaml index ca339ea5..45afddb8 100644 --- a/chart/test-values.yaml +++ b/chart/test-values.yaml @@ -4,6 +4,7 @@ configMap: barbican: {} ceilometer: dbUri: "sqlite:///:memory:" + cinder: {} glance: {} placement: {} neutron: {} diff --git a/config/samples/operator-config.yaml b/config/samples/operator-config.yaml index 8bb2fe9b..c82f9a48 100644 --- a/config/samples/operator-config.yaml +++ b/config/samples/operator-config.yaml @@ -5,6 +5,7 @@ metadata: data: operator-config.yaml: | barbican: {} + cinder: {} ceilometer: dbUri: "sqlite:///:memory:" horizon: diff --git a/devstack/lib/cinder b/devstack/lib/cinder new file mode 100644 index 00000000..c4f9ae45 --- /dev/null +++ b/devstack/lib/cinder @@ -0,0 +1,136 @@ +#!/bin/bash +# +# Copyright 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +CINDER_STATE_PATH=/var/lib/cinder + +# configure_cinder() - Set config files, create data dirs, etc +function configure_cinder { + sudo install -d -o $STACK_USER -m 755 $CINDER_CONF_DIR + + rm -f $CINDER_CONF + + configure_rootwrap cinder + sudo sed -e "s:^filters_path=.*$:filters_path=/usr/local/etc/cinder/rootwrap.d:" -i $CINDER_CONF_DIR/rootwrap.conf + cp -p "$CINDER_DIR/etc/cinder/resource_filters.json" "$CINDER_CONF_DIR/resource_filters.json" + cp $CINDER_DIR/etc/cinder/api-paste.ini $CINDER_API_PASTE_INI + + kubernetes_ensure_resource secret/cinder-application-credential + CINDER_APPLICATION_CREDENTIAL_SECRET=$(get_data_from_secret cinder-application-credential openstack secret) + CINDER_APPLICATION_CREDENTIAL_ID=$(get_data_from_secret cinder-application-credential openstack id) + iniset $CINDER_CONF keystone_authtoken auth_url $KEYSTONE_AUTH_URI_V3 + iniset $CINDER_CONF keystone_authtoken auth_type v3applicationcredential + iniset $CINDER_CONF keystone_authtoken application_credential_id $CINDER_APPLICATION_CREDENTIAL_ID + iniset $CINDER_CONF keystone_authtoken application_credential_secret $CINDER_APPLICATION_CREDENTIAL_SECRET + + iniset $CINDER_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL + + # NOTE(Alex): operator stuff + kubernetes_ensure_resource secret/cinder-mysql + CINDER_DATABASE_USER=$(get_data_from_secret cinder-mysql openstack USER) + CINDER_DATABASE_PASSWORD=$(get_data_from_secret cinder-mysql openstack PASSWORD) + CINDER_DATABASE_NAME=$(get_data_from_secret cinder-mysql openstack DATABASE) + iniset $CINDER_CONF database connection "mysql+pymysql://$CINDER_DATABASE_USER:$CINDER_DATABASE_PASSWORD@cinder-mysql-master/$CINDER_DATABASE_NAME?charset=utf8" + + iniset $CINDER_CONF DEFAULT api_paste_config $CINDER_API_PASTE_INI + iniset $CINDER_CONF DEFAULT rootwrap_config "$CINDER_CONF_DIR/rootwrap.conf" + iniset $CINDER_CONF DEFAULT osapi_volume_extension cinder.api.contrib.standard_extensions + iniset $CINDER_CONF DEFAULT osapi_volume_listen $CINDER_SERVICE_LISTEN_ADDRESS + iniset $CINDER_CONF DEFAULT state_path $CINDER_STATE_PATH + iniset $CINDER_CONF DEFAULT my_ip "$HOST_IP" + iniset $CINDER_CONF key_manager backend cinder.keymgr.conf_key_mgr.ConfKeyManager + iniset $CINDER_CONF key_manager fixed_key $(openssl rand -hex 16) + + configure_cinder_backend_ceph ceph + iniset $CINDER_CONF ceph volume_clear $CINDER_VOLUME_CLEAR + iniset $CINDER_CONF DEFAULT enabled_backends ceph + iniset $CINDER_CONF DEFAULT default_volume_type ceph + configure_cinder_image_volume_cache + + iniset $CINDER_CONF DEFAULT backup_swift_url "$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:$SWIFT_DEFAULT_BIND_PORT/v1/AUTH_" + iniset $CINDER_CONF oslo_messaging_notifications driver "messagingv2" + + # Get rabbitmq password + kubernetes_ensure_resource secret/cinder-rabbitmq + CINDER_RABBITMQ_PASSWORD=$(get_data_from_secret cinder-rabbitmq openstack password) + CINDER_RABBITMQ_USERNAME=$(get_data_from_secret cinder-rabbitmq openstack username) + iniset_k8s_rpc_backend cinder $CINDER_CONF DEFAULT "rabbit://$CINDER_RABBITMQ_USERNAME:$CINDER_RABBITMQ_PASSWORD@rabbitmq-cinder:5672/" + + if [[ -r $CINDER_PLUGINS/$CINDER_DRIVER ]]; then + configure_cinder_driver + fi + + iniset $CINDER_CONF DEFAULT osapi_volume_workers "$API_WORKERS" + + iniset $CINDER_CONF DEFAULT glance_api_servers "http://glance" + + # Set nova credentials (used for os-assisted-snapshots) + + iniset $CINDER_CONF nova auth_type password + iniset $CINDER_CONF nova auth_url $KEYSTONE_SERVICE_URI + iniset $CINDER_CONF nova username nova + iniset $CINDER_CONF nova password $SERVICE_PASSWORD + iniset $CINDER_CONF nova user_domain_name "$SERVICE_DOMAIN_NAME" + iniset $CINDER_CONF nova project_name $SERVICE_PROJECT_NAME + iniset $CINDER_CONF nova project_domain_name "$SERVICE_DOMAIN_NAME" + + iniset $CINDER_CONF nova region_name "$REGION_NAME" + iniset $CINDER_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT" + +} +export -f configure_cinder + + +# init_cinder() - Initialize database and volume group +function init_cinder { + echo noop +} + +# install_cinder() - Collect source and prepare +function install_cinder { + echo noop +} + +# install_cinderclient() - Collect source and prepare +# NOTE(Alex): I am not sure this function is not overrided if the content is empty. +# So I remove this function for now in this override lib file. +# function install_cinderclient { +# echo noop +# } + +function cleanup_cinder { + echo noop +} + +# start_cinder() - Start running processes +function start_cinder { + + kubernetes_rollout_restart daemonset/cinder-api + kubernetes_rollout_restart daemonset/cinder-scheduler + kubernetes_rollout_restart daemonset/cinder-volume + + kubernetes_rollout_status daemonset/cinder-api + kubernetes_rollout_status daemonset/cinder-scheduler + kubernetes_rollout_status daemonset/cinder-volume + + proxy_pass_to_kubernetes /volume cinder cinder-wsgi + + sleep 10 +} + +# stop_cinder() - Stop running processes +function stop_cinder { + echo noop +} diff --git a/devstack/lib/common b/devstack/lib/common index 60aa6a34..61d2d29d 100644 --- a/devstack/lib/common +++ b/devstack/lib/common @@ -31,7 +31,7 @@ function kubernetes_rollout_status { kubectl get $resource && break || sleep 1; done - kubectl rollout status --timeout=60s $resource + kubectl rollout status --timeout=300s $resource } function kubernetes_rollout_restart { @@ -46,8 +46,9 @@ function kubernetes_rollout_restart { function kubernetes_ensure_resource { local resource="$1" - for i in {1..60}; do - kubectl get $resource && break || sleep 3; + kubectl logs deployment/openstack-operator -n default + for i in {1..120}; do + kubectl get $resource && break || sleep 5; done } diff --git a/devstack/plugin.sh b/devstack/plugin.sh index c7772297..6302be67 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -41,11 +41,18 @@ elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then --from-file=/etc/glance/glance-api.conf \ --from-file=/etc/glance/glance-api-paste.ini + kubectl create secret generic cinder-config -n openstack \ + --from-file=/etc/cinder/cinder.conf \ + --from-file=/etc/cinder/api-paste.ini \ + --from-file=/etc/cinder/rootwrap.conf \ + --from-file=/etc/cinder/resource_filters.json + # NOTE(Alex): Permissions here are bad but it's temporary so we don't care as much. sudo chmod -Rv 777 /etc/ceph kubectl create secret generic ceph-config -n openstack \ --from-file=/etc/ceph/ceph.conf \ - --from-file=/etc/ceph/ceph.client.glance.keyring + --from-file=/etc/ceph/ceph.client.glance.keyring \ + --from-file=/etc/ceph/ceph.client.cinder.keyring elif [[ "$1" == "stack" && "$2" == "extra" ]]; then : diff --git a/devstack/settings b/devstack/settings index 74a5545c..714a3f11 100644 --- a/devstack/settings +++ b/devstack/settings @@ -18,6 +18,7 @@ define_plugin openstack-operator source $DEST/openstack-operator/devstack/lib/common source $DEST/openstack-operator/devstack/lib/barbican +source $DEST/openstack-operator/devstack/lib/cinder source $DEST/openstack-operator/devstack/lib/glance source $DEST/openstack-operator/devstack/lib/horizon source $DEST/openstack-operator/devstack/lib/keystone diff --git a/images/cinder/Dockerfile b/images/cinder/Dockerfile new file mode 100644 index 00000000..4cf3b2c0 --- /dev/null +++ b/images/cinder/Dockerfile @@ -0,0 +1,33 @@ +# Copyright (c) 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM vexxhost/python-builder as builder +FROM vexxhost/python-base AS cinder-base +RUN mkdir -p /var/lib/cinder && \ + chmod 777 -R /var/lib/cinder + +FROM cinder-base AS cinder-api +COPY cinder-wsgi /usr/local/bin/cinder-wsgi +EXPOSE 8776 +ENV UWSGI_HTTP_SOCKET=:8776 UWSGI_WSGI_FILE=/usr/local/bin/cinder-wsgi +CMD ["/usr/local/bin/uwsgi", "--ini", "/etc/uwsgi/uwsgi.ini"] + +FROM cinder-base AS cinder-volume +COPY cinder-volume /usr/local/bin/cinder-volume +CMD ["/usr/local/bin/cinder-volume"] + +FROM cinder-base AS cinder-scheduler +COPY cinder-scheduler /usr/local/bin/cinder-scheduler +CMD ["/usr/local/bin/cinder-scheduler"] \ No newline at end of file diff --git a/images/cinder/bindep.txt b/images/cinder/bindep.txt new file mode 100644 index 00000000..7315ad24 --- /dev/null +++ b/images/cinder/bindep.txt @@ -0,0 +1,9 @@ +gcc [compile] +libc-dev [compile] +librados-dev [compile] +librbd-dev [compile] +librados2 +librbd1 +ceph-common +qemu-utils +sudo diff --git a/images/cinder/build-requirements.txt b/images/cinder/build-requirements.txt new file mode 100644 index 00000000..002d1b93 --- /dev/null +++ b/images/cinder/build-requirements.txt @@ -0,0 +1 @@ +Cython diff --git a/images/cinder/cinder-scheduler b/images/cinder/cinder-scheduler new file mode 100755 index 00000000..e1929669 --- /dev/null +++ b/images/cinder/cinder-scheduler @@ -0,0 +1,30 @@ +#!/usr/local/bin/python +# Copyright (c) 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pkg_resources +import re +import sys + +import sentry_sdk + +from cinder.cmd.scheduler import main + +VERSION = pkg_resources.get_distribution("cinder").version + +sentry_sdk.init(release="cinder@%s" % VERSION) + +sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) +sys.exit(main()) diff --git a/images/cinder/cinder-volume b/images/cinder/cinder-volume new file mode 100755 index 00000000..84d1998d --- /dev/null +++ b/images/cinder/cinder-volume @@ -0,0 +1,30 @@ +#!/usr/local/bin/python +# Copyright (c) 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pkg_resources +import re +import sys + +import sentry_sdk + +from cinder.cmd.volume import main + +VERSION = pkg_resources.get_distribution("cinder").version + +sentry_sdk.init(release="cinder@%s" % VERSION) + +sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) +sys.exit(main()) diff --git a/images/cinder/cinder-wsgi b/images/cinder/cinder-wsgi new file mode 100755 index 00000000..3f611f03 --- /dev/null +++ b/images/cinder/cinder-wsgi @@ -0,0 +1,29 @@ +#!/usr/local/bin/python +# Copyright (c) 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pkg_resources + +import sentry_sdk + +from cinder.wsgi.wsgi import initialize_application +from sentry_sdk.integrations import wsgi + +VERSION = pkg_resources.get_distribution("cinder").version + +sentry_sdk.init(release="cinder@%s" % VERSION) + +application = initialize_application() +application = wsgi.SentryWsgiMiddleware(application) diff --git a/images/cinder/constraints.txt b/images/cinder/constraints.txt new file mode 100644 index 00000000..e483a913 --- /dev/null +++ b/images/cinder/constraints.txt @@ -0,0 +1 @@ +--constraint https://releases.openstack.org/constraints/upper/ussuri diff --git a/images/cinder/requirements.txt b/images/cinder/requirements.txt new file mode 100644 index 00000000..30f3c12d --- /dev/null +++ b/images/cinder/requirements.txt @@ -0,0 +1,8 @@ +uWSGI +boto3 +PyMySQL +python-memcached +sentry-sdk +git+https://opendev.org/openstack/cinder@stable/ussuri +https://github.com/ceph/ceph/archive/v15.2.4.tar.gz#egg=rados&subdirectory=src/pybind/rados +https://github.com/ceph/ceph/archive/v15.2.4.tar.gz#egg=rbd&subdirectory=src/pybind/rbd diff --git a/images/cinder/setup-repos.sh b/images/cinder/setup-repos.sh new file mode 100755 index 00000000..4e7d0b52 --- /dev/null +++ b/images/cinder/setup-repos.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright (c) 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +apt-get install -y gnupg2 + +cat <