From 057aaa6ec90d037661ec88ef8af2b477e28411b4 Mon Sep 17 00:00:00 2001 From: Abhishek Kekane Date: Wed, 29 Jul 2020 07:37:16 +0000 Subject: [PATCH] Configure cinder store for glance This patch will enable user to configure single cinder store as well as multiple cinder stores for glance. Below are the parameters needs to be added in local.conf. A. For single store USE_CINDER_FOR_GLANCE=True B. For Multiple stores USE_CINDER_FOR_GLANCE=True GLANCE_ENABLE_MULTIPLE_STORES=True CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1,lvm:lvmdriver-2,nfs:nfsdriver-1,ceph:cephdriver-1} GLANCE_CINDER_DEFAULT_BACKEND=lvmdriver-1 enable_plugin devstack-plugin-nfs https://opendev.org/openstack/devstack-plugin-nfs enable_plugin devstack-plugin-ceph https://opendev.org/openstack/devstack-plugin-ceph NOTE: GLANCE_CINDER_DEFAULT_BACKEND should be one of the value from CINDER_ENABLED_BACKENDS. If you need to configure nfs and ceph backend for cinder then you need to add respective plugins in local.conf file. If GLANCE_ENABLE_MULTIPLE_STORES is True then it will not configure swift store for glance even if it is enabled in local.conf file. Needed-by: https://review.opendev.org/#/c/750018 Change-Id: Id0d63c4ea41cce389eee8dc9a96913a7d427f186 --- lib/glance | 190 +++++++++++++++++++++++++++++++++++++++++------------ stack.sh | 60 +++++++++-------- 2 files changed, 183 insertions(+), 67 deletions(-) diff --git a/lib/glance b/lib/glance index 2118636a46..a848fc7abe 100644 --- a/lib/glance +++ b/lib/glance @@ -41,6 +41,16 @@ else GLANCE_BIN_DIR=$(get_python_exec_prefix) fi +# Cinder for Glance +USE_CINDER_FOR_GLANCE=$(trueorfalse False USE_CINDER_FOR_GLANCE) +# GLANCE_CINDER_DEFAULT_BACKEND should be one of the values +# from CINDER_ENABLED_BACKENDS +GLANCE_CINDER_DEFAULT_BACKEND=${GLANCE_CINDER_DEFAULT_BACKEND:-lvmdriver-1} +GLANCE_STORE_ROOTWRAP_BASE_DIR=/usr/local/etc/glance +# NOTE (abhishekk): For opensuse data files are stored in different directory +if is_opensuse; then + GLANCE_STORE_ROOTWRAP_BASE_DIR=/usr/etc/glance +fi # Glance multi-store configuration # Boolean flag to enable multiple store configuration for glance GLANCE_ENABLE_MULTIPLE_STORES=$(trueorfalse False GLANCE_ENABLE_MULTIPLE_STORES) @@ -68,6 +78,7 @@ GLANCE_STANDALONE=${GLANCE_STANDALONE:-True} # and $DATA_DIR/glance/cheap. GLANCE_MULTISTORE_FILE_IMAGE_DIR=${GLANCE_MULTISTORE_FILE_IMAGE_DIR:=$DATA_DIR/glance} GLANCE_IMAGE_DIR=${GLANCE_IMAGE_DIR:=$DATA_DIR/glance/images} +GLANCE_NFS_MOUNTPOINT=$GLANCE_IMAGE_DIR/mnt GLANCE_LOCK_DIR=${GLANCE_LOCK_DIR:=$DATA_DIR/glance/locks} GLANCE_STAGING_DIR=${GLANCE_MULTISTORE_FILE_IMAGE_DIR:=$DATA_DIR/os_glance_staging_store} GLANCE_TASKS_DIR=${GLANCE_MULTISTORE_FILE_IMAGE_DIR:=$DATA_DIR/os_glance_tasks_store} @@ -135,6 +146,122 @@ function cleanup_glance { fi } +# Set multiple cinder store related config options for each of the cinder store +# +function configure_multiple_cinder_stores { + + local be be_name be_type enabled_backends + for be in ${CINDER_ENABLED_BACKENDS//,/ }; do + be_type=${be%%:*} + be_name=${be##*:} + enabled_backends+="${be_name}:cinder," + + set_common_cinder_store_params $be_name + iniset $GLANCE_API_CONF $be_name cinder_volume_type ${be_name} + if [[ "$be_type" == "nfs" ]]; then + mkdir -p "$GLANCE_NFS_MOUNTPOINT" + iniset $GLANCE_API_CONF $be_name cinder_mount_point_base "$GLANCE_NFS_MOUNTPOINT" + fi + done + iniset $GLANCE_API_CONF DEFAULT enabled_backends ${enabled_backends::-1} + iniset $GLANCE_API_CONF glance_store default_backend $GLANCE_CINDER_DEFAULT_BACKEND +} + +# Set common cinder store options to given config section +# +# Arguments: +# config_section +# +function set_common_cinder_store_params { + local config_section="$1" + iniset $GLANCE_API_CONF $config_section cinder_store_auth_address $KEYSTONE_SERVICE_URI_V3 + iniset $GLANCE_API_CONF $config_section cinder_store_user_name glance + iniset $GLANCE_API_CONF $config_section cinder_store_password $SERVICE_PASSWORD + iniset $GLANCE_API_CONF $config_section cinder_store_project_name $SERVICE_PROJECT_NAME +} + +# Configure multiple file stores options for each file store +# +# Arguments: +# +function configure_multiple_file_stores { + local store enabled_backends + enabled_backends="" + for store in $(echo $GLANCE_MULTIPLE_FILE_STORES | tr "," "\n"); do + enabled_backends+="${store}:file," + done + iniset $GLANCE_API_CONF DEFAULT enabled_backends ${enabled_backends::-1} + + # Glance multiple store Store specific configs + iniset $GLANCE_API_CONF glance_store default_backend $GLANCE_DEFAULT_BACKEND + local store + for store in $(echo $glance_multiple_file_stores | tr "," "\n"); do + iniset $GLANCE_API_CONF $store filesystem_store_datadir "${GLANCE_MULTISTORE_FILE_IMAGE_DIR}/${store}/" + done +} + +# Set reserved stores for glance +function configure_reserved_stores { + iniset $GLANCE_API_CONF os_glance_staging_store filesystem_store_datadir "${GLANCE_MULTISTORE_FILE_IMAGE_DIR}/os_glance_staging_store/" + iniset $GLANCE_API_CONF os_glance_tasks_store filesystem_store_datadir "${GLANCE_MULTISTORE_FILE_IMAGE_DIR}/os_glance_tasks_store/" +} + +# Copy rootwrap file from glance_store/etc/glance to /etc/glance +# +# Arguments: +# source_path Source path to copy rootwrap files from +# +function copy_rootwrap { + local source_path="$1" + # Make glance configuration directory if it is not exists + sudo install -d -o $STACK_USER $GLANCE_CONF_DIR + cp -r $source_path/rootwrap.* $GLANCE_CONF_DIR/ +} + +# Set glance_store related config options +# +# Arguments: +# USE_CINDER_FOR_GLANCE +# GLANCE_ENABLE_MULTIPLE_STORES +# +function configure_glance_store { + local use_cinder_for_glance="$1" + local glance_enable_multiple_stores="$2" + local be + + if [[ "$glance_enable_multiple_stores" == "False" ]]; then + # Configure traditional glance_store + if [[ "$use_cinder_for_glance" == "True" ]]; then + # set common glance_store parameters + iniset $GLANCE_API_CONF glance_store stores "cinder,file,http" + iniset $GLANCE_API_CONF glance_store default_store cinder + + # set cinder related store parameters + set_common_cinder_store_params glance_store + # set nfs mount_point dir + for be in ${CINDER_ENABLED_BACKENDS//,/ }; do + local be_name=${be##*:} + if [[ "$be_name" == "nfs" ]]; then + mkdir -p $GLANCE_NFS_MOUNTPOINT + iniset $GLANCE_API_CONF glance_store cinder_mount_point_base $GLANCE_NFS_MOUNTPOINT + fi + done + fi + # Store specific configs + iniset $GLANCE_API_CONF glance_store filesystem_store_datadir $GLANCE_IMAGE_DIR/ + else + if [[ "$use_cinder_for_glance" == "True" ]]; then + # Configure multiple cinder stores for glance + configure_multiple_cinder_stores + else + # Configure multiple file stores for glance + configure_multiple_file_stores + fi + # Configure reserved stores + configure_reserved_stores + fi +} + # configure_glance() - Set config files, create data dirs, etc function configure_glance { sudo install -d -o $STACK_USER $GLANCE_CONF_DIR $GLANCE_METADEF_DIR @@ -143,16 +270,6 @@ function configure_glance { local dburl dburl=`database_connection_url glance` - # Configure multiple stores - if [[ "$GLANCE_ENABLE_MULTIPLE_STORES" == "True" ]]; then - local store enabled_backends - enabled_backends="" - for store in $(echo $GLANCE_MULTIPLE_FILE_STORES | tr "," "\n"); do - enabled_backends+="${store}:file," - done - iniset $GLANCE_API_CONF DEFAULT enabled_backends ${enabled_backends::-1} - fi - iniset $GLANCE_API_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL iniset $GLANCE_API_CONF database connection $dburl iniset $GLANCE_API_CONF DEFAULT use_syslog $SYSLOG @@ -170,21 +287,8 @@ function configure_glance { iniset $GLANCE_API_CONF DEFAULT disk_formats "ami,ari,aki,vhd,vmdk,raw,qcow2,vdi,iso,ploop" fi - # Glance multiple store Store specific configs - if [[ "$GLANCE_ENABLE_MULTIPLE_STORES" == "True" ]]; then - iniset $GLANCE_API_CONF glance_store default_backend $GLANCE_DEFAULT_BACKEND - local store - for store in $(echo $GLANCE_MULTIPLE_FILE_STORES | tr "," "\n"); do - iniset $GLANCE_API_CONF $store filesystem_store_datadir "${GLANCE_MULTISTORE_FILE_IMAGE_DIR}/${store}/" - done - - # Glance configure reserved stores - iniset $GLANCE_API_CONF os_glance_staging_store filesystem_store_datadir "${GLANCE_MULTISTORE_FILE_IMAGE_DIR}/os_glance_staging_store/" - iniset $GLANCE_API_CONF os_glance_tasks_store filesystem_store_datadir "${GLANCE_MULTISTORE_FILE_IMAGE_DIR}/os_glance_tasks_store/" - else - # Store specific configs - iniset $GLANCE_API_CONF glance_store filesystem_store_datadir $GLANCE_IMAGE_DIR/ - fi + # Configure glance_store + configure_glance_store $USE_CINDER_FOR_GLANCE $GLANCE_ENABLE_MULTIPLE_STORES # CORS feature support - to allow calls from Horizon by default if [ -n "$GLANCE_CORS_ALLOWED_ORIGIN" ]; then @@ -194,24 +298,26 @@ function configure_glance { fi # No multiple stores for swift yet - # Store the images in swift if enabled. - if is_service_enabled s-proxy; then - iniset $GLANCE_API_CONF glance_store default_store swift - iniset $GLANCE_API_CONF glance_store swift_store_create_container_on_put True + if [[ "$GLANCE_ENABLE_MULTIPLE_STORES" == "False" ]]; then + # Store the images in swift if enabled. + if is_service_enabled s-proxy; then + iniset $GLANCE_API_CONF glance_store default_store swift + iniset $GLANCE_API_CONF glance_store swift_store_create_container_on_put True - iniset $GLANCE_API_CONF glance_store swift_store_config_file $GLANCE_SWIFT_STORE_CONF - iniset $GLANCE_API_CONF glance_store default_swift_reference ref1 - iniset $GLANCE_API_CONF glance_store stores "file, http, swift" - if is_service_enabled tls-proxy; then - iniset $GLANCE_API_CONF glance_store swift_store_cacert $SSL_BUNDLE_FILE + iniset $GLANCE_API_CONF glance_store swift_store_config_file $GLANCE_SWIFT_STORE_CONF + iniset $GLANCE_API_CONF glance_store default_swift_reference ref1 + iniset $GLANCE_API_CONF glance_store stores "file, http, swift" + if is_service_enabled tls-proxy; then + iniset $GLANCE_API_CONF glance_store swift_store_cacert $SSL_BUNDLE_FILE + fi + iniset $GLANCE_API_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT" + + iniset $GLANCE_SWIFT_STORE_CONF ref1 user $SERVICE_PROJECT_NAME:glance-swift + + iniset $GLANCE_SWIFT_STORE_CONF ref1 key $SERVICE_PASSWORD + iniset $GLANCE_SWIFT_STORE_CONF ref1 auth_address $KEYSTONE_SERVICE_URI/v3 + iniset $GLANCE_SWIFT_STORE_CONF ref1 auth_version 3 fi - iniset $GLANCE_API_CONF DEFAULT graceful_shutdown_timeout "$SERVICE_GRACEFUL_SHUTDOWN_TIMEOUT" - - iniset $GLANCE_SWIFT_STORE_CONF ref1 user $SERVICE_PROJECT_NAME:glance-swift - - iniset $GLANCE_SWIFT_STORE_CONF ref1 key $SERVICE_PASSWORD - iniset $GLANCE_SWIFT_STORE_CONF ref1 auth_address $KEYSTONE_SERVICE_URI/v3 - iniset $GLANCE_SWIFT_STORE_CONF ref1 auth_version 3 fi # We need to tell glance what it's public endpoint is so that the version @@ -342,9 +448,11 @@ function install_glance { if use_library_from_git "glance_store"; then git_clone_by_name "glance_store" setup_dev_lib "glance_store" $(join_extras "${glance_store_extras[@]}") + copy_rootwrap ${DEST}/glance_store/etc/glance else # we still need to pass extras pip_install_gr_extras glance-store $(join_extras "${glance_store_extras[@]}") + copy_rootwrap $GLANCE_STORE_ROOTWRAP_BASE_DIR fi git_clone $GLANCE_REPO $GLANCE_DIR $GLANCE_BRANCH diff --git a/stack.sh b/stack.sh index ba9da638a8..b7ecb643cb 100755 --- a/stack.sh +++ b/stack.sh @@ -1222,32 +1222,6 @@ if is_service_enabled swift; then start_swift fi -# Launch the Glance services -if is_service_enabled glance; then - echo_summary "Starting Glance" - start_glance -fi - - -# Install Images -# ============== - -# Upload an image to Glance. -# -# The default image is CirrOS, a small testing image which lets you login as **root** -# CirrOS has a ``cloud-init`` analog supporting login via keypair and sending -# scripts as userdata. -# See https://help.ubuntu.com/community/CloudInit for more on ``cloud-init`` - -# NOTE(yoctozepto): limited to node hosting the database which is the controller -if is_service_enabled $DATABASE_BACKENDS && is_service_enabled glance; then - echo_summary "Uploading images" - - for image_url in ${IMAGE_URLS//,/ }; do - upload_image $image_url - done -fi - # NOTE(lyarwood): By default use a single hardcoded fixed_key across devstack # deployments. This ensures the keys match across nova and cinder across all # hosts. @@ -1315,6 +1289,40 @@ if is_service_enabled cinder; then create_volume_types fi +# This sleep is required for cinder volume service to become active and +# publish capabilities to cinder scheduler before creating the image-volume +if [[ "$USE_CINDER_FOR_GLANCE" == "True" ]]; then + sleep 30 +fi + +# Launch the Glance services +# NOTE (abhishekk): We need to start glance api service only after cinder +# service has started as on glance startup glance-api queries cinder for +# validating volume_type configured for cinder store of glance. +if is_service_enabled glance; then + echo_summary "Starting Glance" + start_glance +fi + +# Install Images +# ============== + +# Upload an image to Glance. +# +# The default image is CirrOS, a small testing image which lets you login as **root** +# CirrOS has a ``cloud-init`` analog supporting login via keypair and sending +# scripts as userdata. +# See https://help.ubuntu.com/community/CloudInit for more on ``cloud-init`` + +# NOTE(yoctozepto): limited to node hosting the database which is the controller +if is_service_enabled $DATABASE_BACKENDS && is_service_enabled glance; then + echo_summary "Uploading images" + + for image_url in ${IMAGE_URLS//,/ }; do + upload_image $image_url + done +fi + if is_service_enabled horizon; then echo_summary "Starting Horizon"