#!/bin/bash # # lib/ceph # Functions to control the configuration # and operation of the **Ceph** storage service # Dependencies: # # - ``functions`` file # - ``CEPH_DATA_DIR`` or ``DATA_DIR`` must be defined # ``stack.sh`` calls the entry points in this order: # # - install_ceph # - configure_ceph # - start_ceph # - stop_ceph # - cleanup_ceph # - cleanup_containerized_ceph # Save trace setting XTRACE=$(set +o | grep xtrace) set +o xtrace # Defaults # -------- CEPH_RELEASE=${CEPH_RELEASE:-nautilus} CEPH_RELEASE_STABLE=${CEPH_RELEASE_STABLE:-luminous} GANESHA_RELEASE=${GANESHA_RELEASE:-ceph_nautilus} GANESHA_RELEASE_STABLE=${GANESHA_RELEASE_STABLE:-ceph_luminous} # Deploy a Ceph demo container instead of a non-containerized version CEPH_CONTAINERIZED=$(trueorfalse False CEPH_CONTAINERIZED) # Set ``CEPH_DATA_DIR`` to the location of Ceph drives and objects. # Default is the common DevStack data directory. CEPH_DATA_DIR=${CEPH_DATA_DIR:-/var/lib/ceph} CEPH_DISK_IMAGE=${CEPH_DISK_IMAGE:-${CEPH_DATA_DIR}/drives/images/ceph.img} # Set ``CEPH_CONF_DIR`` to the location of the configuration files. # Default is ``/etc/ceph``. CEPH_CONF_DIR=${CEPH_CONF_DIR:-/etc/ceph} # DevStack will create a loop-back disk formatted as XFS to store the # Ceph data. Set ``CEPH_LOOPBACK_DISK_SIZE`` to the disk size in # kilobytes. VOLUME_BACKING_FILE_SIZE=${VOLUME_BACKING_FILE_SIZE:-8GB} CEPH_LOOPBACK_DISK_SIZE_DEFAULT=${CEPH_LOOPBACK_DISK_SIZE_DEFAULT:-$VOLUME_BACKING_FILE_SIZE} CEPH_LOOPBACK_DISK_SIZE=\ ${CEPH_LOOPBACK_DISK_SIZE:-$CEPH_LOOPBACK_DISK_SIZE_DEFAULT} # Common CEPH_FSID=$(uuidgen) CEPH_CONF_FILE=${CEPH_CONF_DIR}/ceph.conf MDS_ID=${MDS_ID:-a} MGR_ID=${MGR_ID:-x} # RBD configuration defaults CEPH_RBD_DEFAULT_FEATURES=${CEPH_RBD_DEFAULT_FEATURES:-1} # Glance GLANCE_CEPH_USER=${GLANCE_CEPH_USER:-glance} GLANCE_CEPH_POOL=${GLANCE_CEPH_POOL:-images} GLANCE_CEPH_POOL_PG=${GLANCE_CEPH_POOL_PG:-8} GLANCE_CEPH_POOL_PGP=${GLANCE_CEPH_POOL_PGP:-8} GLANCE_RGW_BACKEND=${GLANCE_RGW_BACKEND:-False} # Nova NOVA_CEPH_POOL=${NOVA_CEPH_POOL:-vms} NOVA_CEPH_POOL_PG=${NOVA_CEPH_POOL_PG:-8} NOVA_CEPH_POOL_PGP=${NOVA_CEPH_POOL_PGP:-8} # Cinder CINDER_CEPH_POOL=${CINDER_CEPH_POOL:-volumes} CINDER_CEPH_POOL_PG=${CINDER_CEPH_POOL_PG:-8} CINDER_CEPH_POOL_PGP=${CINDER_CEPH_POOL_PGP:-8} CINDER_CEPH_USER=${CINDER_CEPH_USER:-cinder} CINDER_CEPH_UUID=${CINDER_CEPH_UUID:-$(uuidgen)} # Manila CEPHFS_POOL_PG=${CEPHFS_POOL_PG:-8} # Multiple filesystems enable more than one devstack to share # the same REMOTE_CEPH cluster. Note that in addition to setting # CEPHFS_MULTIPLE_FILESYSTEMS and REMOTE_CEPH, each devstack # needs to set distinct values for CEPHFS_FILESYSTEM, # CEPHFS_METADATA_POOL, and CEPHFS_DATA_POOL. CEPHFS_MULTIPLE_FILESYSTEMS=${CEPHFS_MULTIPLE_FILESYSTEMS:-False} CEPHFS_FILESYSTEM=${CEPHFS_FILESYSTEM:-cephfs} CEPHFS_METADATA_POOL=${CEPHFS_METADATA_POOL:-cephfs_metadata} CEPHFS_DATA_POOL=${CEPHFS_DATA_POOL:-cephfs_data} MANILA_CEPH_DRIVER=${MANILA_CEPH_DRIVER:-cephfsnative} MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila} # Allows driver to store NFS-Ganesha exports and export counter as # RADOS objects in CephFS's data pool. This needs NFS-Ganesha v2.5.4 or later, # Ceph v12.2.2 or later, and OpenStack Queens or later. MANILA_CEPH_GANESHA_RADOS_STORE=${MANILA_CEPH_GANESHA_RADOS_STORE:-True} # Set ``CEPH_REPLICAS`` to configure how many replicas are to be # configured for your Ceph cluster. By default we are configuring # only one replica since this is way less CPU and memory intensive. If # you are planning to test Ceph replication feel free to increase this value CEPH_REPLICAS=${CEPH_REPLICAS:-1} CEPH_REPLICAS_SEQ=$(seq ${CEPH_REPLICAS}) # Rados gateway CEPH_RGW_PORT=${CEPH_RGW_PORT:-8080} CEPH_RGW_IDENTITY_API_VERSION=${CEPH_RGW_IDENTITY_API_VERSION:-3} CEPH_RGW_KEYSTONE_SSL=$(trueorfalse False CEPH_RGW_KEYSTONE_SSL) # iSCSI defaults CEPH_ISCSI_TARGET_IQN=${CEPH_ISCSI_TARGET_IQN:-iqn.1993-08.org.opendev:01:a9aa4032d2c1} CEPH_ISCSI_API_USER=${CEPH_ISCSI_API_USER:-openstack} CEPH_ISCSI_API_PASSWORD=${CEPH_ISCSI_API_PASSWORD:-openstack} CEPH_ISCSI_API_HOST=${CEPH_ISCSI_API_HOST:-localhost} CEPH_ISCSI_API_PORT=${CEPH_ISCSI_API_PORT:-5002} CEPH_ISCSI_GATEWAY_CFG=${CEPH_CONF_DIR}/iscsi-gateway.cfg # gwcli requires a pool named rbd CEPH_ISCSI_POOL="rbd" CEPH_ISCSI_POOL_PG=${CEPH_ISCSI_POOL_PG:-8} TCMU_RUNNER_REPO="https://github.com/open-iscsi/tcmu-runner.git" TARGETCLI_REPO="https://github.com/open-iscsi/targetcli-fb.git" CEPH_ISCSI_REPO="https://github.com/ceph/ceph-iscsi.git" # Ceph REST API (for containerized version only) # Default is 5000, but Keystone already listens on 5000 CEPH_REST_API_PORT=${CEPH_REST_API_PORT:-5001} # Connect to an existing Ceph cluster REMOTE_CEPH=$(trueorfalse False REMOTE_CEPH) REMOTE_CEPH_ADMIN_KEY_PATH=\ ${REMOTE_CEPH_ADMIN_KEY_PATH:-$CEPH_CONF_DIR/ceph.client.admin.keyring} REMOTE_CEPH_RGW=$(trueorfalse False REMOTE_CEPH_RGW) if [[ "$TARGET_BRANCH" =~ stable/(ocata|pike) ]]; then # not supported before Queens ATTACH_ENCRYPTED_VOLUME_AVAILABLE=False fi # Set INIT_SYSTEM to upstart, systemd, or init. In our domain it should be # safe to assume that if the init system is not upstart or systemd that it # is sysvinit rather than other theoretical possibilities like busybox. INIT_SYSTEM=$(init --version 2>/dev/null | grep -qs upstart && echo upstart \ || cat /proc/1/comm) # Functions # ------------ # Containerized Ceph function deploy_containerized_ceph { install_package docker docker.io ceph-common DOCKER_EXEC="docker exec ceph-demo" initial_configure_ceph sudo docker run -d \ --name ceph-demo \ --net=host \ -v ${CEPH_CONF_DIR}:${CEPH_CONF_DIR} \ -v ${CEPH_DATA_DIR}:${CEPH_DATA_DIR} \ -e MON_IP=${SERVICE_HOST} \ -e CEPH_PUBLIC_NETWORK=$(grep -o ${SERVICE_HOST%??}0/.. /proc/net/fib_trie | head -1) \ -e RGW_CIVETWEB_PORT=${CEPH_RGW_PORT} \ -e RESTAPI_PORT=${CEPH_REST_API_PORT} \ ceph/demo # wait for ceph to be healthy then continue ceph_status } function wait_for_daemon { timeout=20 daemon_to_test=$1 while [ $timeout -ne 0 ]; do if eval $daemon_to_test; then return 0 fi sleep 1 let timeout=timeout-1 done return 1 } function ceph_status { echo "Waiting for Ceph to be ready" return $(wait_for_daemon "sudo docker exec ceph-demo ceph health | grep -sq HEALTH_OK") } # is_ceph_enabled_for_service() - checks whether the OpenStack service # specified as an argument is enabled with Ceph as its storage backend. function is_ceph_enabled_for_service { local config config_name enabled service enabled=1 service=$1 # Construct the global variable ENABLE_CEPH_.* corresponding to a # $service. config_name=ENABLE_CEPH_$(echo $service | \ tr '[:lower:]' '[:upper:]' | tr '-' '_') config=$(eval echo "\$$config_name") if (is_service_enabled $service) && [[ $config == 'True' ]]; then enabled=0 fi return $enabled } # _get_ceph_version() - checks version of Ceph mon daemon or CLI based on an # argument. Checking mon daemon version requires the mon daemon to be up # and healthy. function _get_ceph_version { local ceph_version_str local mon_started if [[ $1 == 'cli' ]]; then # ceph --version show CLI version ceph_version_str=$(sudo ceph --version | cut -d ' ' -f 3 | \ cut -d '.' -f 1,2) elif [[ $1 == 'mon' ]]; then # ceph version show mon daemon version mon_started=$(wait_for_daemon "sudo systemctl is-active --quiet ceph-mon@$(hostname)") if $mon_started; then ceph_version_str=$(sudo ceph version | cut -d ' ' -f 3 | \ cut -f 1,2 -d '.') else die $LINENO "ceph-mon@${hostname} is not running and it's not possible to \ retrieve it's version" fi else die $LINENO "Invalid argument. The get_ceph_version function needs \ an argument that can be 'cli' or 'mon'." fi echo $ceph_version_str } # import_libvirt_secret_ceph() - Imports Cinder user key into libvirt # so it can connect to the Ceph cluster while attaching a Cinder block device function import_libvirt_secret_ceph { cat </dev/null ${CINDER_CEPH_UUID} client.${CINDER_CEPH_USER} secret EOF sudo virsh secret-define --file secret.xml sudo virsh secret-set-value --secret ${CINDER_CEPH_UUID} \ --base64 $(sudo ceph -c ${CEPH_CONF_FILE} \ auth get-key client.${CINDER_CEPH_USER}) sudo rm -f secret.xml } # _undefine_virsh_secret() - Undefine Cinder key secret from libvirt function _undefine_virsh_secret { if is_ceph_enabled_for_service cinder || \ is_ceph_enabled_for_service nova; then local virsh_uuid virsh_uuid=$(sudo virsh secret-list | awk '/^ ?[0-9a-z]/ { print $1 }') sudo virsh secret-undefine ${virsh_uuid} &>/dev/null fi } # check_os_support_ceph() - Check if the OS provides a decent version of Ceph function check_os_support_ceph { if [[ ${DISTRO} =~ f[0-9][0-9] ]]; then # Assume all versions of Fedora are fine as of 2018. return fi if [[ ! ${DISTRO} =~ (focal|bionic|xenial|f31|f32) ]]; then echo "WARNING: your distro $DISTRO does not provide \ (at least) the Luminous release. \ Please use Ubuntu Xenial, Ubuntu Bionic, Ubuntu Focal, Fedora 31 or Fedora 32" if [[ "$FORCE_CEPH_INSTALL" != "yes" ]]; then die $LINENO "If you wish to install Ceph on this distribution \ anyway run with FORCE_CEPH_INSTALL=yes, \ this assumes that YOU will setup the proper repositories" fi NO_UPDATE_REPOS=False fi if [[ ! $INIT_SYSTEM == 'systemd' ]]; then die "This plugin is only supported on systemd enabled systems currently." fi } # check_os_support_ceph_iscsi() - Make sure kernel supports iscsi requirements function check_os_support_for_iscsi { KERNEL_CONFIG="/boot/config-$(uname -r)" target_core=$(grep -E '(CONFIG_TARGET_CORE=m|CONFIG_TARGET_CORE=y)' $KERNEL_CONFIG) tcm_user=$(grep -E '(CONFIG_TCM_USER2=m|CONFIG_TCM_USER2=y)' $KERNEL_CONFIG) iscsi_target=$(grep -E '(CONFIG_ISCSI_TARGET=m|CONFIG_ISCSI_TARGET=y)' $KERNEL_CONFIG) if [ -z "$target_core" ] || [ -z "$tcm_user" ] || [ -z "$iscsi_target" ]; then die "Ceph iSCSI cannot work. The required kernel modules are not installed." fi } # cleanup_ceph() - Remove residual data files, anything left over from previous # runs that a clean run would need to clean up function cleanup_ceph_remote { # do a proper cleanup from here to avoid leftover on the remote Ceph cluster if is_ceph_enabled_for_service glance; then sudo ceph -c ${CEPH_CONF_FILE} osd pool delete $GLANCE_CEPH_POOL $GLANCE_CEPH_POOL \ --yes-i-really-really-mean-it > /dev/null 2>&1 sudo ceph -c ${CEPH_CONF_FILE} auth del client.$GLANCE_CEPH_USER > /dev/null 2>&1 fi if is_ceph_enabled_for_service cinder; then sudo ceph -c ${CEPH_CONF_FILE} osd pool delete $CINDER_CEPH_POOL $CINDER_CEPH_POOL \ --yes-i-really-really-mean-it > /dev/null 2>&1 sudo ceph -c ${CEPH_CONF_FILE} auth del client.$CINDER_CEPH_USER > /dev/null 2>&1 fi if is_ceph_enabled_for_service c-bak; then sudo ceph -c ${CEPH_CONF_FILE} osd pool delete $CINDER_BAK_CEPH_POOL $CINDER_BAK_CEPH_POOL \ --yes-i-really-really-mean-it > /dev/null 2>&1 sudo ceph -c ${CEPH_CONF_FILE} auth del client.$CINDER_BAK_CEPH_USER > /dev/null 2>&1 fi if is_ceph_enabled_for_service nova; then iniset $NOVA_CONF libvirt rbd_secret_uuid "" sudo ceph -c ${CEPH_CONF_FILE} osd pool delete $NOVA_CEPH_POOL $NOVA_CEPH_POOL \ --yes-i-really-really-mean-it > /dev/null 2>&1 fi } function cleanup_ceph_embedded { sudo killall -w -9 ceph-mon ceph-osd ceph-mds if [ "$ENABLE_CEPH_RGW" = "True" ]; then sudo killall -w -9 radosgw fi if [ "$ENABLE_CEPH_ISCSI" = "True" ]; then stop_ceph_iscsi fi sudo rm -rf ${CEPH_DATA_DIR}/*/* if egrep -q ${CEPH_DATA_DIR} /proc/mounts; then sudo umount ${CEPH_DATA_DIR} fi if [[ -e ${CEPH_DISK_IMAGE} ]]; then sudo rm -f ${CEPH_DISK_IMAGE} fi # purge ceph config file and keys sudo rm -rf ${CEPH_CONF_DIR}/* } function cleanup_ceph_general { _undefine_virsh_secret if is_ceph_enabled_for_service manila && [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then cleanup_nfs_ganesha cleanup_repo_nfsganesha fi if is_ceph_enabled_for_service manila; then sudo ceph -c ${CEPH_CONF_FILE} fs rm $CEPHFS_FILESYSTEM \ --yes-i-really-mean-it sudo ceph -c ${CEPH_CONF_FILE} osd pool delete $CEPHFS_METADATA_POOL $CEPHFS_METADATA_POOL \ --yes-i-really-really-mean-it > /dev/null 2>&1 sudo ceph -c ${CEPH_CONF_FILE} osd pool delete $CEPHFS_DATA_POOL $CEPHFS_DATA_POOL \ --yes-i-really-really-mean-it > /dev/null 2>&1 sudo ceph -c ${CEPH_CONF_FILE} auth del client.$MANILA_CEPH_USER > /dev/null 2>&1 fi cleanup_repo_ceph } function cleanup_containerized_ceph { sudo docker rm -f ceph-demo sudo rm -rf ${CEPH_CONF_DIR}/* sudo rm -rf ${CEPH_DATA_DIR} } function initial_configure_ceph { # create a backing file disk create_disk ${CEPH_DISK_IMAGE} ${CEPH_DATA_DIR} ${CEPH_LOOPBACK_DISK_SIZE} # populate ceph directory sudo mkdir -p \ ${CEPH_DATA_DIR}/{bootstrap-mds,bootstrap-osd,bootstrap-mgr,bootstrap-rgw,mgr,rgw,mds,mon,osd,tmp,radosgw} } # configure_ceph() - Set config files, create data dirs, etc function configure_ceph { local count=0 initial_configure_ceph # create ceph monitor initial key and directory sudo ceph-authtool ${CEPH_DATA_DIR}/tmp/keyring.mon.$(hostname) \ --create-keyring --name=mon. --add-key=$(ceph-authtool --gen-print-key) \ --cap mon 'allow *' # gen admin keyring, gen client.admin user and add user to keyring sudo ceph-authtool ${CEPH_CONF_DIR}/ceph.client.admin.keyring \ --create-keyring --gen-key -n client.admin --cap mon 'allow *' \ --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' # add gen keys to ceph.mon.keyring sudo ceph-authtool ${CEPH_DATA_DIR}/tmp/keyring.mon.$(hostname) \ --import-keyring ${CEPH_CONF_DIR}/ceph.client.admin.keyring sudo mkdir -p ${CEPH_DATA_DIR}/mon/ceph-$(hostname) # create a default ceph configuration file iniset -sudo ${CEPH_CONF_FILE} global "fsid" "${CEPH_FSID}" iniset -sudo ${CEPH_CONF_FILE} global "mon_initial_members" "$(hostname)" iniset -sudo ${CEPH_CONF_FILE} global "mon_host" "${SERVICE_HOST}" iniset -sudo ${CEPH_CONF_FILE} global "auth_cluster_required" "cephx" iniset -sudo ${CEPH_CONF_FILE} global "auth_service_required" "cephx" iniset -sudo ${CEPH_CONF_FILE} global "auth_client_required" "cephx" iniset -sudo ${CEPH_CONF_FILE} global "filestore_xattr_use_omap" "true" iniset -sudo ${CEPH_CONF_FILE} global "osd crush chooseleaf type" "0" iniset -sudo ${CEPH_CONF_FILE} global "osd journal size" "100" iniset -sudo ${CEPH_CONF_FILE} global "osd pool default size" "${CEPH_REPLICAS}" iniset -sudo ${CEPH_CONF_FILE} global "rbd default features" "${CEPH_RBD_DEFAULT_FEATURES}" iniset -sudo ${CEPH_CONF_FILE} client "debug_client" "10" iniset -sudo ${CEPH_CONF_FILE} client "debug_ms" "1" # bootstrap the ceph monitor sudo ceph-mon -c ${CEPH_CONF_FILE} --mkfs -i $(hostname) \ --keyring ${CEPH_DATA_DIR}/tmp/keyring.mon.$(hostname) sudo chown -R ceph. ${CEPH_DATA_DIR} sudo systemctl enable ceph-mon@$(hostname) sudo systemctl start ceph-mon@$(hostname) local ceph_version ceph_version=$(_get_ceph_version mon) if vercmp "$ceph_version" ">=" "14.0"; then for key in bootstrap-{mds,osd,rgw}; do sudo ceph auth get client.$key -o ${CEPH_DATA_DIR}/$key/ceph.keyring done fi sudo mkdir -p ${CEPH_DATA_DIR}/mgr/ceph-${MGR_ID} sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create mgr.${MGR_ID} \ mon 'allow profile mgr' mds 'allow *' osd 'allow *' \ -o ${CEPH_DATA_DIR}/mgr/ceph-${MGR_ID}/keyring sudo chown -R ceph. ${CEPH_DATA_DIR}/mgr # create a simple rule to take OSDs instead of hosts with CRUSH # then apply this rule to the default pool if [[ $CEPH_REPLICAS -ne 1 ]]; then sudo ceph -c ${CEPH_CONF_FILE} \ osd crush rule create-simple devstack default osd RULE_ID=$(sudo ceph -c ${CEPH_CONF_FILE} \ osd crush rule dump devstack | \ awk '/rule_id/ {print $2}' | \ cut -d ',' -f1) sudo ceph -c ${CEPH_CONF_FILE} \ osd pool set rbd crush_ruleset ${RULE_ID} sudo ceph -c ${CEPH_CONF_FILE} \ osd pool set data crush_ruleset ${RULE_ID} sudo ceph -c ${CEPH_CONF_FILE} \ osd pool set metadata crush_ruleset ${RULE_ID} fi # create the OSD(s) for rep in ${CEPH_REPLICAS_SEQ}; do OSD_ID=$(sudo ceph -c ${CEPH_CONF_FILE} osd create) sudo mkdir -p ${CEPH_DATA_DIR}/osd/ceph-${OSD_ID} sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create osd.${OSD_ID} \ mon 'allow profile osd ' osd 'allow *' | \ sudo tee ${CEPH_DATA_DIR}/osd/ceph-${OSD_ID}/keyring sudo chown ceph. ${CEPH_DATA_DIR}/osd/ceph-${OSD_ID} sudo chown ceph. ${CEPH_DATA_DIR}/osd/ceph-${OSD_ID}/keyring sudo ceph-osd -c ${CEPH_CONF_FILE} --setuser ceph --setgroup ceph -i ${OSD_ID} --mkfs sudo ceph-osd -c ${CEPH_CONF_FILE} -i ${OSD_ID} --mkfs sudo systemctl enable ceph-osd@${OSD_ID} done if is_ceph_enabled_for_service manila; then # create a MDS sudo mkdir -p ${CEPH_DATA_DIR}/mds/ceph-${MDS_ID} sudo chown ceph. ${CEPH_DATA_DIR}/mds/ceph-${MDS_ID} sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create mds.${MDS_ID} \ mon 'allow profile mds ' osd 'allow rw' mds 'allow' \ -o ${CEPH_DATA_DIR}/mds/ceph-${MDS_ID}/keyring sudo chown ceph. /var/lib/ceph/mds/ceph-${MDS_ID}/keyring sudo systemctl enable ceph-mds@${MDS_ID} fi if [ "$ENABLE_CEPH_RGW" = "True" ]; then _configure_ceph_rgw fi if [ "$ENABLE_CEPH_ISCSI" = "True" ]; then _configure_ceph_iscsi fi } function _configure_rgw_ceph_section { configure_ceph_embedded_rgw_paths iniset -sudo ${CEPH_CONF_FILE} ${key} "host" "$(hostname)" iniset -sudo ${CEPH_CONF_FILE} ${key} "keyring" "${dest}/keyring" iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw socket path" "/tmp/radosgw-$(hostname).sock" iniset -sudo ${CEPH_CONF_FILE} ${key} "log file" "/var/log/ceph/radosgw-$(hostname).log" iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw data" "${dest}" iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw print continue" "false" iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw frontends" "civetweb port=${CEPH_RGW_PORT}" iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone url" "$KEYSTONE_SERVICE_URI" iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw s3 auth use keystone" "true" iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone admin user" "radosgw" iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone admin password" "$SERVICE_PASSWORD" iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone accepted roles" "Member, _member_, admin, ResellerAdmin" if [ "$CEPH_RGW_KEYSTONE_SSL" = "True" ]; then iniset -sudo ${CEPH_CONF_FILE} ${key} "nss db path" "${dest}/nss" else iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone verify ssl" "false" fi if [[ $CEPH_RGW_IDENTITY_API_VERSION == '2.0' ]]; then iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone admin tenant" "$SERVICE_PROJECT_NAME" else iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone admin project" "$SERVICE_PROJECT_NAME" iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone admin domain" "$SERVICE_DOMAIN_NAME" iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone api version" "3" fi } function _configure_ceph_rgw_container { _configure_rgw_ceph_section sudo docker restart ceph-demo } function _configure_ceph_rgw { # bootstrap rados gateway _configure_rgw_ceph_section sudo mkdir -p $dest sudo ceph auth get-or-create $key \ osd 'allow rwx' mon 'allow rw' \ -o ${dest}/keyring sudo systemctl enable ceph-radosgw@rgw.$(hostname) sudo chown -R ceph. ${CEPH_DATA_DIR} } function _configure_ceph_iscsi_gateway { iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "trusted_ip_list" "$HOST_IP,localhost" iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "cluster_name" "ceph" iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "gateway_keyring" "ceph.client.admin.keyring" iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "prometheus_host" "$CEPH_ISCSI_API_HOST" iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "api_secure" "false" iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "api_user" "$CEPH_ISCSI_API_USER" iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "api_password" "$CEPH_ISCSI_API_PASSWORD" iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "api_port" "$CEPH_ISCSI_API_PORT" iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "api_host" "$CEPH_ISCSI_API_HOST" } function _configure_ceph_iscsi { _configure_ceph_iscsi_gateway sudo ceph -c ${CEPH_CONF_FILE} \ osd pool create ${CEPH_ISCSI_POOL} ${CEPH_ISCSI_POOL_PG} sudo systemctl daemon-reload sudo systemctl enable tcmu-runner sudo systemctl enable rbd-target-gw sudo systemctl enable rbd-target-api } function _post_start_configure_iscsi_gateway { # Now we setup the rbd-target-gw and rbd-target-api for use GWCLI=$(which gwcli) removeme=$(sudo systemctl status rbd-target-api) HOSTNAME=$(hostname) # create the target_iqn for exporting all volumes sudo $GWCLI /iscsi-targets create $CEPH_ISCSI_TARGET_IQN # now we add the gateway definition # Didn't find the gateway, so lets create it sudo $GWCLI /iscsi-targets/$CEPH_ISCSI_TARGET_IQN/gateways create $HOSTNAME $HOST_IP skipchecks=true } function start_ceph_iscsi { sudo systemctl start tcmu-runner sudo systemctl start rbd-target-gw sudo systemctl start rbd-target-api sleep 10 # we have to setup the gateway and api after they start _post_start_configure_iscsi_gateway } function stop_ceph_iscsi { HOSTNAME=$(hostname) sudo $GWCLI /iscsi-targets/$CEPH_ISCSI_TARGET_IQN/gateways delete $HOSTNAME sudo $GWCLI /iscsi-targets delete $CEPH_ISCSI_TARGET_IQN sudo systemctl stop rbd-target-api sudo systemctl stop rbd-target-gw sudo systemctl stop tcmu-runner } function _create_swift_endpoint { local swift_service swift_service=$(get_or_create_service "swift" "object-store" "Swift Service") local swift_endpoint swift_endpoint="$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:${CEPH_RGW_PORT}/swift/v1" get_or_create_endpoint $swift_service \ "$REGION_NAME" $swift_endpoint $swift_endpoint $swift_endpoint } function configure_ceph_embedded_rgw_paths { if [[ "$CEPH_CONTAINERIZED" == "True" ]]; then dest=${CEPH_DATA_DIR}/radosgw/$(hostname) key=client.radosgw.gateway else dest=${CEPH_DATA_DIR}/radosgw/ceph-rgw.$(hostname) key=client.rgw.$(hostname) fi } function configure_ceph_embedded_rgw { configure_ceph_embedded_rgw_paths # keystone endpoint for radosgw _create_swift_endpoint # Create radosgw service user with admin privileges create_service_user "radosgw" "admin" if [ "$CEPH_RGW_KEYSTONE_SSL" = "True" ]; then # radosgw needs to access keystone's revocation list sudo mkdir -p ${dest}/nss sudo openssl x509 -in /etc/keystone/ssl/certs/ca.pem -pubkey | \ sudo certutil -d ${dest}/nss -A -n ca -t "TCu,Cu,Tuw" sudo openssl x509 -in /etc/keystone/ssl/certs/signing_cert.pem -pubkey | \ sudo certutil -A -d ${dest}/nss -n signing_cert -t "P,P,P" fi } function start_ceph_embedded_rgw { sudo systemctl enable ceph-radosgw@rgw.$(hostname) sudo systemctl start ceph-radosgw@rgw.$(hostname) } function configure_ceph_embedded_glance { # configure Glance service options, ceph pool, ceph user and ceph key if [[ $CEPH_REPLICAS -ne 1 ]]; then sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool \ set ${GLANCE_CEPH_POOL} crush_ruleset ${RULE_ID} fi } # configure_ceph_glance() - Glance config needs to come after Glance is set up function configure_ceph_glance { if [[ "$GLANCE_RGW_BACKEND" = "True" && "$ENABLE_CEPH_RGW" = "True" ]]; then # common glance accounts for swift create_service_user "glance-swift" "ResellerAdmin" iniset $GLANCE_SWIFT_STORE_CONF ref1 user $SERVICE_PROJECT_NAME:glance-swift AUTH_URL=$KEYSTONE_SERVICE_URI/v$CEPH_RGW_IDENTITY_API_VERSION iniset $GLANCE_SWIFT_STORE_CONF ref1 key $SERVICE_PASSWORD iniset $GLANCE_SWIFT_STORE_CONF ref1 auth_address $AUTH_URL iniset $GLANCE_SWIFT_STORE_CONF ref1 user_domain_name $SERVICE_DOMAIN_NAME iniset $GLANCE_SWIFT_STORE_CONF ref1 project_domain_name $SERVICE_DOMAIN_NAME iniset $GLANCE_SWIFT_STORE_CONF ref1 auth_version $CEPH_RGW_IDENTITY_API_VERSION iniset $GLANCE_API_CONF glance_store default_store swift iniset $GLANCE_API_CONF glance_store swift_store_create_container_on_put True iniset $GLANCE_API_CONF glance_store swift_store_config_file $GLANCE_SWIFT_STORE_CONF iniset $GLANCE_API_CONF glance_store default_swift_reference ref1 iniset $GLANCE_API_CONF glance_store stores "file, http, swift" else sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool create \ ${GLANCE_CEPH_POOL} ${GLANCE_CEPH_POOL_PG} ${GLANCE_CEPH_POOL_PGP} sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} auth \ get-or-create client.${GLANCE_CEPH_USER} \ mon "allow r" \ osd "allow class-read object_prefix rbd_children, \ allow rwx pool=${GLANCE_CEPH_POOL}" | \ sudo tee ${CEPH_CONF_DIR}/ceph.client.${GLANCE_CEPH_USER}.keyring sudo chown ${STACK_USER}:$(id -g -n $whoami) \ ${CEPH_CONF_DIR}/ceph.client.${GLANCE_CEPH_USER}.keyring iniset $GLANCE_API_CONF DEFAULT show_multiple_locations True iniset $GLANCE_API_CONF glance_store default_store rbd iniset $GLANCE_API_CONF glance_store stores "file, http, rbd" iniset $GLANCE_API_CONF glance_store rbd_store_ceph_conf $CEPH_CONF_FILE iniset $GLANCE_API_CONF glance_store rbd_store_user $GLANCE_CEPH_USER iniset $GLANCE_API_CONF glance_store rbd_store_pool $GLANCE_CEPH_POOL fi } function configure_ceph_manila { sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool create ${CEPHFS_METADATA_POOL} \ ${CEPHFS_POOL_PG} sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool create ${CEPHFS_DATA_POOL} \ ${CEPHFS_POOL_PG} if [[ ${CEPHFS_MULTIPLE_FILESYSTEMS} == 'True' ]]; then sudo ceph -c ${CEPH_CONF_FILE} fs flag set enable_multiple true \ --yes-i-really-mean-it fi sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} fs new ${CEPHFS_FILESYSTEM} ${CEPHFS_METADATA_POOL} \ ${CEPHFS_DATA_POOL} sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} auth get-or-create \ client.${MANILA_CEPH_USER} \ mon "allow *" osd "allow rw" mds "allow *" \ -o ${CEPH_CONF_DIR}/ceph.client.${MANILA_CEPH_USER}.keyring sudo chown ${STACK_USER}:$(id -g -n $whoami) \ ${CEPH_CONF_DIR}/ceph.client.${MANILA_CEPH_USER}.keyring # Enable snapshots in CephFS. sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} fs set ${CEPHFS_FILESYSTEM} allow_new_snaps true \ --yes-i-really-mean-it # Make manila's libcephfs client a root user. iniset -sudo ${CEPH_CONF_FILE} client.${MANILA_CEPH_USER} "client mount uid" "0" iniset -sudo ${CEPH_CONF_FILE} client.${MANILA_CEPH_USER} "client mount gid" "0" if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then configure_nfs_ganesha # NFS-Ganesha server cannot run alongwith with other kernel NFS server. sudo systemctl stop nfs-server || true sudo systemctl disable nfs-server || true sudo systemctl enable nfs-ganesha sudo systemctl start nfs-ganesha || ( echo "Ganesha didn't start. Let's debug..." >&2 sudo systemctl status nfs-ganesha || true echo "**Ganesha conf file**" >&2 sudo cat /etc/ganesha/ganesha.conf || true echo "**Ganesha log file**" >&2 sudo cat /var/log/ganesha/ganesha.log || true echo "**Exiting**" >&2 exit 1 ) echo "Ganesha started successfully!" >&2 fi # RESTART DOCKER CONTAINER } function configure_nfs_ganesha { # Configure NFS-Ganesha to work with Manila's CephFS driver sudo mkdir -p /etc/ganesha/export.d if [ $MANILA_CEPH_GANESHA_RADOS_STORE == 'True' ]; then # Create an empty placeholder ganesha export index object echo | sudo rados -p ${CEPHFS_DATA_POOL} put ganesha-export-index - cat </dev/null RADOS_URLS { ceph_conf = ${CEPH_CONF_FILE}; userid = admin; } %url rados://${CEPHFS_DATA_POOL}/ganesha-export-index EOF else sudo touch /etc/ganesha/export.d/INDEX.conf echo "%include /etc/ganesha/export.d/INDEX.conf" | sudo tee /etc/ganesha/ganesha.conf fi } function cleanup_nfs_ganesha { sudo systemctl stop nfs-ganesha sudo systemctl disable nfs-ganesha sudo uninstall_package nfs-ganesha nfs-ganesha-ceph } function configure_ceph_embedded_manila { if [[ $CEPH_REPLICAS -ne 1 ]]; then sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_DATA_POOL} \ crush_ruleset ${RULE_ID} sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_METADATA_POOL} \ crush_ruleset ${RULE_ID} fi } function configure_ceph_embedded_nova { # configure Nova service options, ceph pool, ceph user and ceph key if [[ $CEPH_REPLICAS -ne 1 ]]; then sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool \ set ${NOVA_CEPH_POOL} crush_ruleset ${RULE_ID} fi } # configure_ceph_nova() - Nova config needs to come after Nova is set up function configure_ceph_nova { sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool create \ ${NOVA_CEPH_POOL} ${NOVA_CEPH_POOL_PG} ${NOVA_CEPH_POOL_PGP} iniset $NOVA_CONF libvirt rbd_user ${CINDER_CEPH_USER} iniset $NOVA_CONF libvirt rbd_secret_uuid ${CINDER_CEPH_UUID} iniset $NOVA_CONF libvirt inject_key false iniset $NOVA_CONF libvirt inject_partition -2 iniset $NOVA_CONF libvirt disk_cachemodes "network=writeback" iniset $NOVA_CONF libvirt images_type rbd iniset $NOVA_CONF libvirt images_rbd_pool ${NOVA_CEPH_POOL} iniset $NOVA_CONF libvirt images_rbd_ceph_conf ${CEPH_CONF_FILE} if ! is_ceph_enabled_for_service cinder; then sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} \ auth get-or-create client.${CINDER_CEPH_USER} \ mon "allow r" \ osd "allow class-read object_prefix rbd_children, \ allow rwx pool=${CINDER_CEPH_POOL}, \ allow rwx pool=${NOVA_CEPH_POOL}, \ allow rwx pool=${GLANCE_CEPH_POOL}" | \ sudo tee \ ${CEPH_CONF_DIR}/ceph.client.${CINDER_CEPH_USER}.keyring \ > /dev/null sudo chown ${STACK_USER}:$(id -g -n $whoami) \ ${CEPH_CONF_DIR}/ceph.client.${CINDER_CEPH_USER}.keyring fi } function configure_ceph_embedded_cinder { # Configure Cinder service options, ceph pool, ceph user and ceph key if [[ $CEPH_REPLICAS -ne 1 ]]; then sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool \ set ${CINDER_CEPH_POOL} crush_ruleset ${RULE_ID} fi } # configure_ceph_cinder() - Cinder config needs to come after Cinder is set up function configure_ceph_cinder { sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool create \ ${CINDER_CEPH_POOL} ${CINDER_CEPH_POOL_PG} ${CINDER_CEPH_POOL_PGP} sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} auth get-or-create \ client.${CINDER_CEPH_USER} \ mon "allow r" \ osd "allow class-read object_prefix rbd_children, \ allow rwx pool=${CINDER_CEPH_POOL}, allow rwx pool=${NOVA_CEPH_POOL}, \ allow rwx pool=${GLANCE_CEPH_POOL}" | \ sudo tee ${CEPH_CONF_DIR}/ceph.client.${CINDER_CEPH_USER}.keyring sudo chown ${STACK_USER}:$(id -g -n $whoami) \ ${CEPH_CONF_DIR}/ceph.client.${CINDER_CEPH_USER}.keyring } # install_ceph() - Collect source and prepare function install_ceph_remote { install_package ceph-common # ceph-common in Bionic (18.04) installs only the python2 variants of # required packages, meaning we need to install the python3 variants # manually. Hopefully this won't be necessary in Focal (20.04) # https://packages.ubuntu.com/bionic/ceph-common if python3_enabled; then install_package python3-cephfs python3-prettytable python3-rados python3-rbd python3-requests fi } # configure_repo_ceph() - Configure Ceph repositories # Usage: configure_repo_ceph \ # [] # - package_manager: apt-get or yum # - ceph_release: luminous, ... # - distro_type: centos, ubuntu # - distro_release: 7, xenial, bionic # - repo_type: latest, stable (only latest is supported right now) function configure_repo_ceph { local package_manager="$1" local ceph_release="$2" local distro_type="$3" local distro_release="$4" local repo_type="${5:-latest}" local repo_file_name="" if [ "${package_manager}" = "apt-get" ]; then repo_file_name="/etc/apt/sources.list.d/ceph.list" elif [ "${package_manager}" = "yum" ]; then repo_file_name="/etc/yum.repos.d/ext-ceph.repo" fi if [ -n "${repo_file_name}" ]; then repo=$(curl -L "https://shaman.ceph.com/api/search/?project=ceph&distros=${distro_type}/${distro_release}&ref=${ceph_release}&sha1=${repo_type}" | \ jq '.[0] | .chacra_url' | tr -d '"' | awk '{print $1"repo"}') curl -L $repo | sudo tee ${repo_file_name} sudo ${package_manager} -y update fi } # cleanup_repo_ceph() - Remove Ceph repositories # Usage: cleanup_repo_ceph function cleanup_repo_ceph { if is_ubuntu; then sudo rm -rf /etc/apt/sources.list.d/ceph.list elif is_fedora; then sudo rm -rf /etc/yum.repos.d/ext-ceph.repo fi } # configure_repo_nfsganesha() - Configure NFS Ganesha repositories # Usage: configure_repo_nfsganesha \ # [] # - package_manager: apt-get or yum # - flavor: ceph_luminous, ceph_master, ... # - distro_type: centos, ubuntu # - distro_release: 7, xenial, bionic # - repo_type: latest (only latest is supported right now) function configure_repo_nfsganesha { local package_manager="$1" local ganesha_flavor="$2" local distro_type="$3" local distro_release="$4" local repo_type="${5:-latest}" local repo_file_name="" if [ "${package_manager}" = "apt-get" ]; then repo_file_name="/etc/apt/sources.list.d/ext-nfs-ganesha.list" elif [ "${package_manager}" = "yum" ]; then repo_file_name="/etc/yum.repos.d/ext-nfs-ganesha.repo" fi if [ -n "${repo_file_name}" ]; then repo=$(curl -L "https://shaman.ceph.com/api/search/?project=nfs-ganesha-stable&distros=${distro_type}/${distro_release}&flavor=${ganesha_flavor}&sha1=${repo_type}" | \ jq '.[0] | .chacra_url' | tr -d '"' | awk '{print $1"repo"}') curl -L $repo | sudo tee ${repo_file_name} sudo ${package_manager} -y update fi } # cleanup_repo_nfsganesha() - Remove NFS Ganesha repositories # Usage: cleanup_repo_nfsganesha function cleanup_repo_nfsganesha { if is_ubuntu; then sudo rm -rf /etc/apt/sources.list.d/ext-nfs-ganesha.list elif is_fedora; then sudo rm -rf /etc/yum.repos.d/ext-nfs-ganesha.repo fi } function setup_packages_for_iscsi { CWD=$(pwd) # We have to install the following required packages for ceph-iscsi # tcmu-runner, trslib-fb, configshell-fb, targetcli-fb, ceph-iscsi git_clone $TCMU_RUNNER_REPO $DEST/tcmu-runner master cd $DEST/tcmu-runner if is_fedora; then # we can run the install extras from tcmu-runner here extra/install_dep.sh fi cmake -Dwith-glfs=false -Dwith-qcow=false -DSUPPORT_SYSTEMD=on -DCMAKE_INSTALL_PREFIX=/usr sudo make install pip_install rtslib-fb pip_install configshell-fb # not in pypi yet. git_clone $TARGETCLI_REPO $DEST/targetcli-fb master cd $DEST/targetcli-fb sudo python setup.py install if [ ! -d /etc/target ]; then sudo mkdir /etc/target fi if [ ! -d /var/target ]; then sudo mkdir /var/target fi git_clone $CEPH_ISCSI_REPO $DEST/ceph-iscsi master cd $DEST/ceph-iscsi sudo python setup.py install --install-scripts=/usr/bin sudo cp usr/lib/systemd/system/rbd-target-gw.service /lib/systemd/system sudo cp usr/lib/systemd/system/rbd-target-api.service /lib/systemd/system cd $CWD } function setup_packages_for_manila_on_ubuntu { # The 'apt-get' package manager needs the following package to access # HTTPS enabled repositories such as the Ceph repos hosted by the # shaman/chacra system. install_package apt-transport-https CEPH_PACKAGES="${CEPH_PACKAGES} ceph-mds libcephfs2" if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then if [[ $os_CODENAME =~ (xenial) ]]; then configure_repo_nfsganesha "apt-get" "$GANESHA_RELEASE_STABLE" "ubuntu" "$os_CODENAME" else configure_repo_nfsganesha "apt-get" "$GANESHA_RELEASE" "ubuntu" "$os_CODENAME" fi CEPH_PACKAGES="${CEPH_PACKAGES} nfs-ganesha nfs-ganesha-ceph" fi if python3_enabled; then CEPH_PACKAGES="${CEPH_PACKAGES} python3-cephfs" fi } function setup_packages_for_manila_on_fedora_family { if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then # NOTE(vkmc) shaman currently does not build for Fedora # we need to stick to CentOS 7 packages configure_repo_nfsganesha "yum" "$CEPH_RELEASE" "centos" "7" CEPH_PACKAGES="${CEPH_PACKAGES} nfs-ganesha nfs-ganesha-ceph" fi } function install_ceph { install_package jq if is_ubuntu; then if ! [[ $os_CODENAME =~ (focal|xenial|bionic) ]]; then die $LINENO "Supported for Ubuntu Xenial, Bionic or Focal. Not supported for other releases." fi # NOTE(vkmc) Dependencies for setting up repos install_package software-properties-common if [[ $os_CODENAME =~ (xenial) ]]; then configure_repo_ceph "apt-get" "$CEPH_RELEASE_STABLE" "ubuntu" "$os_CODENAME" else configure_repo_ceph "apt-get" "$CEPH_RELEASE" "ubuntu" "$os_CODENAME" fi CEPH_PACKAGES="ceph libnss3-tools" if python3_enabled; then CEPH_PACKAGES="${CEPH_PACKAGES} python3-rados python3-rbd" fi if is_ceph_enabled_for_service manila; then setup_packages_for_manila_on_ubuntu fi if [ "$ENABLE_CEPH_RGW" = "True" ]; then CEPH_PACKAGES="${CEPH_PACKAGES} radosgw" fi if [ "$ENABLE_CEPH_ISCSI" = "True" ]; then CEPH_PACKAGES="${CEPH_PACKAGES} python-pyparsing python-kmodpy python-pyudev" CEPH_PACKAGES="${CEPH_PACKAGES} python-gobject python-urwid python-rados python-rbd" CEPH_PACKAGES="${CEPH_PACKAGES} python-netifaces python-crypto python-requests python-flask" CEPH_PACKAGES="${CEPH_PACKAGES} python-openssl python-rpm open-iscsi" # The packages needed to build tcmu-runner, since there is no # ubuntu package yet. CEPH_PACKAGES="${CEPH_PACKAGES} cmake make gcc libnl-genl-3-200" CEPH_PACKAGES="${CEPH_PACKAGES} librbd1 librbd-dev libglib2.0-0 libglib2.0-bin zlib1g" CEPH_PACKAGES="${CEPH_PACKAGES} libkmod2 libnl-genl-3-dev libglib2.0-dev" CEPH_PACKAGES="${CEPH_PACKAGES} zlib1g-dev libkmod-dev" CEPH_PACKAGES="${CEPH_PACKAGES} libgoogle-perftools4 libgoogle-perftools-dev" fi install_package ${CEPH_PACKAGES} elif is_fedora; then if ! [[ $os_VENDOR =~ Fedora ]] && [[ $os_RELEASE =~ (31|32) ]]; then die $LINENO "Supported for Fedora 31 and 32. Not supported for other releases." fi # NOTE(vkmc) shaman currently does not build for Fedora # we need to stick to CentOS 7 packages configure_repo_ceph "yum" "$CEPH_RELEASE" "centos" "7" CEPH_PACKAGES="ceph" if is_ceph_enabled_for_service manila; then setup_packages_for_manila_on_fedora_family fi if [ "$ENABLE_CEPH_RGW" = "True" ]; then CEPH_PACKAGES="${CEPH_PACKAGES} ceph-radosgw" fi if [ "$ENABLE_CEPH_ISCSI" = "True" ]; then CEPH_PACKAGES="${CEPH_PACKAGES} librbd1 libkmod2 python-pyparsing python-kmod python-pyudev" CEPH_PACKAGES="${CEPH_PACKAGES} python-gobject python-urwid python-rados python-rbd" CEPH_PACKAGES="${CEPH_PACKAGES} python-netifaces python-crypto python-requests python-flask" CEPH_PACKAGES="${CEPH_PACKAGES} python-openssl iscsi-initiator-utils" fi install_package ${CEPH_PACKAGES} else die $LINENO "${os_VENDOR} is not supported by the Ceph plugin for Devstack" fi if [ "$ENABLE_CEPH_ISCSI" = "True" ]; then setup_packages_for_iscsi fi } # start_ceph() - Start running processes, including screen function start_ceph { sudo chown -R ceph. ${CEPH_DATA_DIR} sudo systemctl start ceph-mon@$(hostname) local ceph_version ceph_version=$(_get_ceph_version mon) sudo systemctl start ceph-mgr@${MGR_ID} # use `tell mgr` as the mgr might not have been activated # yet to register the python module commands. if ! sudo ceph -c ${CEPH_CONF_FILE} tell mgr restful create-self-signed-cert; then echo MGR Restful is not working, perhaps the package is not installed? fi for id in $(sudo ceph -c ${CEPH_CONF_FILE} osd ls); do sudo systemctl start ceph-osd@$id done if is_ceph_enabled_for_service manila; then sudo systemctl start ceph-mds@${MDS_ID} fi if [ "$ENABLE_CEPH_ISCSI" = "True" ]; then start_ceph_iscsi fi } # stop_ceph() - Stop running processes (non-screen) function stop_ceph { if [ "$ENABLE_CEPH_RGW" = "True" ]; then sudo systemctl stop ceph-radosgw@rgw.$(hostname) sudo systemctl disable ceph-radosgw@rgw.$(hostname) fi if [ "$ENABLE_CEPH_ISCSI" = "True" ]; then stop_ceph_iscsi fi if is_ceph_enabled_for_service manila; then if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then sudo systemctl stop nfs-ganesha sudo systemctl disable nfs-ganesha fi sudo systemctl stop ceph-mds@${MDS_ID} sudo systemctl disable ceph-mds@${MDS_ID} fi # if mon is dead or unhealthy we won't get the list # of osds but should continue anyways. ids=$(sudo ceph -c ${CEPH_CONF_FILE} osd ls 2>/dev/null --connect-timeout 5) for id in $ids; do sudo systemctl stop ceph-osd@$id sudo systemctl disable ceph-osd@$id done sudo systemctl stop ceph-mgr@${MGR_ID} sudo systemctl disable ceph-mgr@${MGR_ID} # In nautilus we have the new ceph-crash service for monitoring # Try to stop it. If there is no service, stop/disable do nothing sudo systemctl stop ceph-crash sudo systemctl disable ceph-crash sudo systemctl stop ceph-mon@$(hostname) sudo systemctl disable ceph-mon@$(hostname) # NOTE(vkmc) Cleanup any leftover unit files sudo rm -f /etc/systemd/system/ceph* } # Restore xtrace $XTRACE ## Local variables: ## mode: shell-script ## End: