openstack
/
devstack-plugin-ceph
Code Issues Proposed changes
DevStack plugin to configure Ceph backend.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
273 Commits
8 Branches
0 Tags
3.9 MiB
 
Tree: a546b76407
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'a546b76407'
${ noResults }
devstack-plugin-ceph/devstack/lib/ceph

1256 lines
47 KiB
Raw Blame History

#!/bin/bash
#
# lib/ceph
# Functions to control the configuration
# and operation of the **Ceph** storage service
# Dependencies:
#
# - ``functions`` file
# - ``CEPH_DATA_DIR`` or ``DATA_DIR`` must be defined
# ``stack.sh`` calls the entry points in this order:
#
# - install_ceph
# - configure_ceph
# - start_ceph
# - stop_ceph
# - cleanup_ceph
# - cleanup_containerized_ceph
# Save trace setting
XTRACE=$(set +o | grep xtrace)
set +o xtrace
# Defaults
# --------
TEST_MASTER=$(trueorfalse False TEST_MASTER)
CEPH_RELEASE=${CEPH_RELEASE:-pacific}
GANESHA_RELEASE=${GANESHA_RELEASE:-V3.5-stable}
# Deploy a Ceph demo container instead of a non-containerized version
CEPH_CONTAINERIZED=$(trueorfalse False CEPH_CONTAINERIZED)
# Set ``CEPH_DATA_DIR`` to the location of Ceph drives and objects.
# Default is the common DevStack data directory.
CEPH_DATA_DIR=${CEPH_DATA_DIR:-/var/lib/ceph}
CEPH_DISK_IMAGE=${CEPH_DISK_IMAGE:-${CEPH_DATA_DIR}/drives/images/ceph.img}
# Set ``CEPH_CONF_DIR`` to the location of the configuration files.
# Default is ``/etc/ceph``.
CEPH_CONF_DIR=${CEPH_CONF_DIR:-/etc/ceph}
# DevStack will create a loop-back disk formatted as XFS to store the
# Ceph data. Set ``CEPH_LOOPBACK_DISK_SIZE`` to the disk size in
# kilobytes.
VOLUME_BACKING_FILE_SIZE=${VOLUME_BACKING_FILE_SIZE:-8GB}
CEPH_LOOPBACK_DISK_SIZE_DEFAULT=${CEPH_LOOPBACK_DISK_SIZE_DEFAULT:-$VOLUME_BACKING_FILE_SIZE}
CEPH_LOOPBACK_DISK_SIZE=\
${CEPH_LOOPBACK_DISK_SIZE:-$CEPH_LOOPBACK_DISK_SIZE_DEFAULT}
# Common
CEPH_FSID=$(uuidgen)
CEPH_CONF_FILE=${CEPH_CONF_DIR}/ceph.conf
MDS_ID=${MDS_ID:-a}
MGR_ID=${MGR_ID:-x}
# RBD configuration defaults
if [[ ${DISTRO} =~ (bionic|xenial) ]]; then
CEPH_RBD_DEFAULT_FEATURES=${CEPH_RBD_DEFAULT_FEATURES:-"layering, exclusive-lock"}
else
CEPH_RBD_DEFAULT_FEATURES=${CEPH_RBD_DEFAULT_FEATURES:-"layering, exclusive-lock, object-map, fast-diff"}
fi
# Glance
GLANCE_CEPH_USER=${GLANCE_CEPH_USER:-glance}
GLANCE_CEPH_POOL=${GLANCE_CEPH_POOL:-images}
GLANCE_CEPH_POOL_PG=${GLANCE_CEPH_POOL_PG:-8}
GLANCE_CEPH_POOL_PGP=${GLANCE_CEPH_POOL_PGP:-8}
GLANCE_RGW_BACKEND=${GLANCE_RGW_BACKEND:-False}
# Nova
NOVA_CEPH_POOL=${NOVA_CEPH_POOL:-vms}
NOVA_CEPH_POOL_PG=${NOVA_CEPH_POOL_PG:-8}
NOVA_CEPH_POOL_PGP=${NOVA_CEPH_POOL_PGP:-8}
# Cinder
CINDER_CEPH_POOL=${CINDER_CEPH_POOL:-volumes}
CINDER_CEPH_POOL_PG=${CINDER_CEPH_POOL_PG:-8}
CINDER_CEPH_POOL_PGP=${CINDER_CEPH_POOL_PGP:-8}
CINDER_CEPH_USER=${CINDER_CEPH_USER:-cinder}
CINDER_CEPH_UUID=${CINDER_CEPH_UUID:-$(uuidgen)}
# Manila
CEPHFS_POOL_PG=${CEPHFS_POOL_PG:-8}
# Multiple filesystems enable more than one devstack to share
# the same REMOTE_CEPH cluster. Note that in addition to setting
# CEPHFS_MULTIPLE_FILESYSTEMS and REMOTE_CEPH, each devstack
# needs to set distinct values for CEPHFS_FILESYSTEM,
# CEPHFS_METADATA_POOL, and CEPHFS_DATA_POOL.
CEPHFS_MULTIPLE_FILESYSTEMS=${CEPHFS_MULTIPLE_FILESYSTEMS:-False}
CEPHFS_FILESYSTEM=${CEPHFS_FILESYSTEM:-cephfs}
CEPHFS_METADATA_POOL=${CEPHFS_METADATA_POOL:-cephfs_metadata}
CEPHFS_DATA_POOL=${CEPHFS_DATA_POOL:-cephfs_data}
MANILA_CEPH_DRIVER=${MANILA_CEPH_DRIVER:-cephfsnative}
MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila}
# Allows driver to store NFS-Ganesha exports and export counter as
# RADOS objects in CephFS's data pool. This needs NFS-Ganesha v2.5.4 or later,
# Ceph v12.2.2 or later, and OpenStack Queens or later.
MANILA_CEPH_GANESHA_RADOS_STORE=${MANILA_CEPH_GANESHA_RADOS_STORE:-True}
# Set ``CEPH_REPLICAS`` to configure how many replicas are to be
# configured for your Ceph cluster. By default we are configuring
# only one replica since this is way less CPU and memory intensive. If
# you are planning to test Ceph replication feel free to increase this value
CEPH_REPLICAS=${CEPH_REPLICAS:-1}
CEPH_REPLICAS_SEQ=$(seq ${CEPH_REPLICAS})
# Rados gateway
CEPH_RGW_PORT=${CEPH_RGW_PORT:-8080}
CEPH_RGW_IDENTITY_API_VERSION=${CEPH_RGW_IDENTITY_API_VERSION:-3}
CEPH_RGW_KEYSTONE_SSL=$(trueorfalse False CEPH_RGW_KEYSTONE_SSL)
# iSCSI defaults
CEPH_ISCSI_TARGET_IQN=${CEPH_ISCSI_TARGET_IQN:-iqn.1993-08.org.opendev:01:a9aa4032d2c1}
CEPH_ISCSI_API_USER=${CEPH_ISCSI_API_USER:-openstack}
CEPH_ISCSI_API_PASSWORD=${CEPH_ISCSI_API_PASSWORD:-openstack}
CEPH_ISCSI_API_HOST=${CEPH_ISCSI_API_HOST:-$SERVICE_HOST}
CEPH_ISCSI_API_PORT=${CEPH_ISCSI_API_PORT:-5002}
CEPH_ISCSI_GATEWAY_CFG=${CEPH_CONF_DIR}/iscsi-gateway.cfg
CEPH_ISCSI_MINIMUM_GATEWAYS=${CEPH_ISCSI_MINIMUM_GATEWAYS:-1}
# gwcli requires a pool named rbd
CEPH_ISCSI_POOL="rbd"
CEPH_ISCSI_POOL_PG=${CEPH_ISCSI_POOL_PG:-8}
# Ceph REST API (for containerized version only)
# Default is 5000, but Keystone already listens on 5000
CEPH_REST_API_PORT=${CEPH_REST_API_PORT:-5001}
# Connect to an existing Ceph cluster
REMOTE_CEPH=$(trueorfalse False REMOTE_CEPH)
REMOTE_CEPH_ADMIN_KEY_PATH=\
${REMOTE_CEPH_ADMIN_KEY_PATH:-$CEPH_CONF_DIR/ceph.client.admin.keyring}
REMOTE_CEPH_RGW=$(trueorfalse False REMOTE_CEPH_RGW)
if [[ "$TARGET_BRANCH" =~ stable/(ocata|pike) ]]; then
# not supported before Queens
ATTACH_ENCRYPTED_VOLUME_AVAILABLE=False
fi
# Set INIT_SYSTEM to upstart, systemd, or init. In our domain it should be
# safe to assume that if the init system is not upstart or systemd that it
# is sysvinit rather than other theoretical possibilities like busybox.
INIT_SYSTEM=$(init --version 2>/dev/null | grep -qs upstart && echo upstart \
|| cat /proc/1/comm)
# Functions
# ------------
# Containerized Ceph
function deploy_containerized_ceph {
install_package docker docker.io ceph-common
DOCKER_EXEC="docker exec ceph-demo"
initial_configure_ceph
sudo docker run -d \
--name ceph-demo \
--net=host \
-v ${CEPH_CONF_DIR}:${CEPH_CONF_DIR} \
-v ${CEPH_DATA_DIR}:${CEPH_DATA_DIR} \
-e MON_IP=${SERVICE_HOST} \
-e CEPH_PUBLIC_NETWORK=$(grep -o ${SERVICE_HOST%??}0/.. /proc/net/fib_trie | head -1) \
-e RGW_CIVETWEB_PORT=${CEPH_RGW_PORT} \
-e RESTAPI_PORT=${CEPH_REST_API_PORT} \
ceph/demo
# wait for ceph to be healthy then continue
ceph_status
}
function wait_for_daemon {
timeout=20
daemon_to_test=$1
while [ $timeout -ne 0 ]; do
if eval $daemon_to_test; then
return 0
fi
sleep 1
let timeout=timeout-1
done
return 1
}
function ceph_status {
echo "Waiting for Ceph to be ready"
return $(wait_for_daemon "sudo docker exec ceph-demo ceph health | grep -sq HEALTH_OK")
}
# is_ceph_enabled_for_service() - checks whether the OpenStack service
# specified as an argument is enabled with Ceph as its storage backend.
function is_ceph_enabled_for_service {
local config config_name enabled service
enabled=1
service=$1
# Construct the global variable ENABLE_CEPH_.* corresponding to a
# $service.
config_name=ENABLE_CEPH_$(echo $service | \
tr '[:lower:]' '[:upper:]' | tr '-' '_')
config=$(eval echo "\$$config_name")
if (is_service_enabled $service) && [[ $config == 'True' ]]; then
enabled=0
fi
return $enabled
}
# _get_ceph_version() - checks version of Ceph mon daemon or CLI based on an
# argument. Checking mon daemon version requires the mon daemon to be up
# and healthy.
function _get_ceph_version {
local ceph_version_str
local mon_started
if [[ $1 == 'cli' ]]; then
# ceph --version show CLI version
ceph_version_str=$(sudo ceph --version | cut -d ' ' -f 3 | \
cut -d '.' -f 1,2)
elif [[ $1 == 'mon' ]]; then
# ceph version show mon daemon version
mon_started=$(wait_for_daemon "sudo systemctl is-active --quiet ceph-mon@$(hostname)")
if $mon_started; then
ceph_version_str=$(sudo ceph version | cut -d ' ' -f 3 | \
cut -f 1,2 -d '.')
else
die $LINENO "ceph-mon@${hostname} is not running and it's not possible to \
retrieve it's version"
fi
else
die $LINENO "Invalid argument. The get_ceph_version function needs \
an argument that can be 'cli' or 'mon'."
fi
echo $ceph_version_str
}
# import_libvirt_secret_ceph() - Imports Cinder user key into libvirt
# so it can connect to the Ceph cluster while attaching a Cinder block device
function import_libvirt_secret_ceph {
cat <<EOF | sudo tee secret.xml>/dev/null
<secret ephemeral='no' private='no'>
<uuid>${CINDER_CEPH_UUID}</uuid>
<usage type='ceph'>
<name>client.${CINDER_CEPH_USER} secret</name>
</usage>
</secret>
EOF
sudo virsh secret-define --file secret.xml
sudo virsh secret-set-value --secret ${CINDER_CEPH_UUID} \
--base64 $(sudo ceph -c ${CEPH_CONF_FILE} \
auth get-key client.${CINDER_CEPH_USER})
sudo rm -f secret.xml
}
# _undefine_virsh_secret() - Undefine Cinder key secret from libvirt
function _undefine_virsh_secret {
if is_ceph_enabled_for_service cinder || \
is_ceph_enabled_for_service nova; then
local virsh_uuid
virsh_uuid=$(sudo virsh secret-list | awk '/^ ?[0-9a-z]/ { print $1 }')
sudo virsh secret-undefine ${virsh_uuid} &>/dev/null
fi
}
# check_os_support_ceph() - Check if the OS provides a decent version of Ceph
function check_os_support_ceph {
if [[ ! ${DISTRO} =~ (focal|bionic|xenial|f31|f32|f33|f34|rhel8) ]]; then
echo "WARNING: your distro $DISTRO does not provide \
(at least) the Luminous release. \
Please use Ubuntu Xenial, Ubuntu Bionic, Ubuntu Focal, Fedora 31-34,
or CentOS Stream 8."
if [[ "$FORCE_CEPH_INSTALL" != "yes" ]]; then
die $LINENO "If you wish to install Ceph on this distribution \
anyway run with FORCE_CEPH_INSTALL=yes, \
this assumes that YOU will setup the proper repositories"
fi
NO_UPDATE_REPOS=False
fi
if [[ ! $INIT_SYSTEM == 'systemd' ]]; then
die "This plugin is only supported on systemd enabled systems currently."
fi
}
# check_os_support_ceph_iscsi() - Make sure kernel supports iscsi requirements
function check_os_support_for_iscsi {
KERNEL_CONFIG="/boot/config-$(uname -r)"
target_core=$(grep -E '(CONFIG_TARGET_CORE=m|CONFIG_TARGET_CORE=y)' $KERNEL_CONFIG)
tcm_user=$(grep -E '(CONFIG_TCM_USER2=m|CONFIG_TCM_USER2=y)' $KERNEL_CONFIG)
iscsi_target=$(grep -E '(CONFIG_ISCSI_TARGET=m|CONFIG_ISCSI_TARGET=y)' $KERNEL_CONFIG)
if [ -z "$target_core" ] || [ -z "$tcm_user" ] || [ -z "$iscsi_target" ]; then
die "Ceph iSCSI cannot work. The required kernel modules are not installed."
fi
}
# cleanup_ceph() - Remove residual data files, anything left over from previous
# runs that a clean run would need to clean up
function cleanup_ceph_remote {
# do a proper cleanup from here to avoid leftover on the remote Ceph cluster
if is_ceph_enabled_for_service glance; then
sudo ceph -c ${CEPH_CONF_FILE} osd pool delete $GLANCE_CEPH_POOL $GLANCE_CEPH_POOL \
--yes-i-really-really-mean-it > /dev/null 2>&1
sudo ceph -c ${CEPH_CONF_FILE} auth del client.$GLANCE_CEPH_USER > /dev/null 2>&1
fi
if is_ceph_enabled_for_service cinder; then
sudo ceph -c ${CEPH_CONF_FILE} osd pool delete $CINDER_CEPH_POOL $CINDER_CEPH_POOL \
--yes-i-really-really-mean-it > /dev/null 2>&1
sudo ceph -c ${CEPH_CONF_FILE} auth del client.$CINDER_CEPH_USER > /dev/null 2>&1
fi
if is_ceph_enabled_for_service c-bak; then
sudo ceph -c ${CEPH_CONF_FILE} osd pool delete $CINDER_BAK_CEPH_POOL $CINDER_BAK_CEPH_POOL \
--yes-i-really-really-mean-it > /dev/null 2>&1
sudo ceph -c ${CEPH_CONF_FILE} auth del client.$CINDER_BAK_CEPH_USER > /dev/null 2>&1
fi
if is_ceph_enabled_for_service nova; then
iniset $NOVA_CONF libvirt rbd_secret_uuid ""
sudo ceph -c ${CEPH_CONF_FILE} osd pool delete $NOVA_CEPH_POOL $NOVA_CEPH_POOL \
--yes-i-really-really-mean-it > /dev/null 2>&1
fi
# Clean up the disk image and mount that we created
destroy_disk ${CEPH_DISK_IMAGE} ${CEPH_DATA_DIR}
}
function cleanup_ceph_embedded {
sudo killall -w -9 ceph-mon ceph-osd ceph-mds
if [ "$ENABLE_CEPH_RGW" = "True" ]; then
sudo killall -w -9 radosgw
fi
sudo rm -rf ${CEPH_DATA_DIR}/*/*
if egrep -q ${CEPH_DATA_DIR} /proc/mounts; then
sudo umount ${CEPH_DATA_DIR}
fi
if [[ -e ${CEPH_DISK_IMAGE} ]]; then
sudo rm -f ${CEPH_DISK_IMAGE}
fi
# purge ceph config file and keys
sudo rm -rf ${CEPH_CONF_DIR}/*
}
function cleanup_ceph_general {
_undefine_virsh_secret
if is_ceph_enabled_for_service manila && [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
cleanup_nfs_ganesha
cleanup_repo_nfsganesha
fi
if is_ceph_enabled_for_service manila; then
sudo ceph -c ${CEPH_CONF_FILE} fs rm $CEPHFS_FILESYSTEM \
--yes-i-really-mean-it
sudo ceph -c ${CEPH_CONF_FILE} osd pool delete $CEPHFS_METADATA_POOL $CEPHFS_METADATA_POOL \
--yes-i-really-really-mean-it > /dev/null 2>&1
sudo ceph -c ${CEPH_CONF_FILE} osd pool delete $CEPHFS_DATA_POOL $CEPHFS_DATA_POOL \
--yes-i-really-really-mean-it > /dev/null 2>&1
sudo ceph -c ${CEPH_CONF_FILE} auth del client.$MANILA_CEPH_USER > /dev/null 2>&1
fi
cleanup_repo_ceph
}
function cleanup_containerized_ceph {
sudo docker rm -f ceph-demo
sudo rm -rf ${CEPH_CONF_DIR}/*
sudo rm -rf ${CEPH_DATA_DIR}
}
function initial_configure_ceph {
# create a backing file disk
create_disk ${CEPH_DISK_IMAGE} ${CEPH_DATA_DIR} ${CEPH_LOOPBACK_DISK_SIZE}
# populate ceph directory
sudo mkdir -p \
${CEPH_DATA_DIR}/{bootstrap-mds,bootstrap-osd,bootstrap-mgr,bootstrap-rgw,mgr,rgw,mds,mon,osd,tmp,radosgw}
}
# configure_ceph() - Set config files, create data dirs, etc
function configure_ceph {
local count=0
initial_configure_ceph
# create ceph monitor initial key and directory
sudo ceph-authtool ${CEPH_DATA_DIR}/tmp/keyring.mon.$(hostname) \
--create-keyring --name=mon. --add-key=$(ceph-authtool --gen-print-key) \
--cap mon 'allow *'
# gen admin keyring, gen client.admin user and add user to keyring
sudo ceph-authtool ${CEPH_CONF_DIR}/ceph.client.admin.keyring \
--create-keyring --gen-key -n client.admin --cap mon 'allow *' \
--cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
# add gen keys to ceph.mon.keyring
sudo ceph-authtool ${CEPH_DATA_DIR}/tmp/keyring.mon.$(hostname) \
--import-keyring ${CEPH_CONF_DIR}/ceph.client.admin.keyring
sudo mkdir -p ${CEPH_DATA_DIR}/mon/ceph-$(hostname)
# create a default ceph configuration file
iniset -sudo ${CEPH_CONF_FILE} global "fsid" "${CEPH_FSID}"
iniset -sudo ${CEPH_CONF_FILE} global "mon_initial_members" "$(hostname)"
iniset -sudo ${CEPH_CONF_FILE} global "mon_host" "${SERVICE_HOST}"
iniset -sudo ${CEPH_CONF_FILE} global "auth_cluster_required" "cephx"
iniset -sudo ${CEPH_CONF_FILE} global "auth_service_required" "cephx"
iniset -sudo ${CEPH_CONF_FILE} global "auth_client_required" "cephx"
iniset -sudo ${CEPH_CONF_FILE} global "filestore_xattr_use_omap" "true"
iniset -sudo ${CEPH_CONF_FILE} global "osd crush chooseleaf type" "0"
iniset -sudo ${CEPH_CONF_FILE} global "osd journal size" "100"
iniset -sudo ${CEPH_CONF_FILE} global "osd pool default size" "${CEPH_REPLICAS}"
iniset -sudo ${CEPH_CONF_FILE} global "rbd default features" "${CEPH_RBD_DEFAULT_FEATURES}"
iniset -sudo ${CEPH_CONF_FILE} client "debug_client" "10"
local gigs
gigs=$(echo $CEPH_LOOPBACK_DISK_SIZE | grep -o '^[0-9]*')
iniset -sudo ${CEPH_CONF_FILE} global "bluestore_block_size" $((($gigs - 4) << 30))
# bootstrap the ceph monitor
sudo ceph-mon -c ${CEPH_CONF_FILE} --mkfs -i $(hostname) \
--keyring ${CEPH_DATA_DIR}/tmp/keyring.mon.$(hostname)
sudo chown -R ceph. ${CEPH_DATA_DIR}
sudo systemctl enable ceph-mon@$(hostname)
sudo systemctl start ceph-mon@$(hostname)
local ceph_version
ceph_version=$(_get_ceph_version mon)
if vercmp "$ceph_version" ">=" "14.0"; then
for key in bootstrap-{mds,osd,rgw}; do
sudo ceph auth get client.$key -o ${CEPH_DATA_DIR}/$key/ceph.keyring
done
fi
sudo mkdir -p ${CEPH_DATA_DIR}/mgr/ceph-${MGR_ID}
sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create mgr.${MGR_ID} \
mon 'allow profile mgr' mds 'allow *' osd 'allow *' \
-o ${CEPH_DATA_DIR}/mgr/ceph-${MGR_ID}/keyring
sudo chown -R ceph. ${CEPH_DATA_DIR}/mgr
# create a simple rule to take OSDs instead of hosts with CRUSH
# then apply this rule to the default pool
if [[ $CEPH_REPLICAS -ne 1 ]]; then
sudo ceph -c ${CEPH_CONF_FILE} \
osd crush rule create-simple devstack default osd
RULE_ID=$(sudo ceph -c ${CEPH_CONF_FILE} \
osd crush rule dump devstack | \
awk '/rule_id/ {print $2}' | \
cut -d ',' -f1)
sudo ceph -c ${CEPH_CONF_FILE} \
osd pool set rbd crush_ruleset ${RULE_ID}
sudo ceph -c ${CEPH_CONF_FILE} \
osd pool set data crush_ruleset ${RULE_ID}
sudo ceph -c ${CEPH_CONF_FILE} \
osd pool set metadata crush_ruleset ${RULE_ID}
fi
# create the OSD(s)
for rep in ${CEPH_REPLICAS_SEQ}; do
OSD_ID=$(sudo ceph -c ${CEPH_CONF_FILE} osd create)
sudo mkdir -p ${CEPH_DATA_DIR}/osd/ceph-${OSD_ID}
sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create osd.${OSD_ID} \
mon 'allow profile osd ' osd 'allow *' | \
sudo tee ${CEPH_DATA_DIR}/osd/ceph-${OSD_ID}/keyring
sudo chown ceph. ${CEPH_DATA_DIR}/osd/ceph-${OSD_ID}
sudo chown ceph. ${CEPH_DATA_DIR}/osd/ceph-${OSD_ID}/keyring
sudo ceph-osd -c ${CEPH_CONF_FILE} --setuser ceph --setgroup ceph -i ${OSD_ID} --mkfs
sudo ceph-osd -c ${CEPH_CONF_FILE} -i ${OSD_ID} --mkfs
sudo systemctl enable ceph-osd@${OSD_ID}
done
if is_ceph_enabled_for_service manila; then
# create a MDS
sudo mkdir -p ${CEPH_DATA_DIR}/mds/ceph-${MDS_ID}
sudo chown ceph. ${CEPH_DATA_DIR}/mds/ceph-${MDS_ID}
sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create mds.${MDS_ID} \
mon 'allow profile mds ' osd 'allow rw' mds 'allow' \
-o ${CEPH_DATA_DIR}/mds/ceph-${MDS_ID}/keyring
sudo chown ceph. ${CEPH_DATA_DIR}/mds/ceph-${MDS_ID}/keyring
sudo systemctl enable ceph-mds@${MDS_ID}
fi
if [ "$ENABLE_CEPH_RGW" = "True" ]; then
_configure_ceph_rgw
fi
if [ "$ENABLE_CEPH_ISCSI" = "True" ]; then
_configure_ceph_iscsi
fi
}
function _configure_rgw_ceph_section {
configure_ceph_embedded_rgw_paths
iniset -sudo ${CEPH_CONF_FILE} ${key} "host" "$(hostname)"
iniset -sudo ${CEPH_CONF_FILE} ${key} "keyring" "${dest}/keyring"
iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw socket path" "/tmp/radosgw-$(hostname).sock"
iniset -sudo ${CEPH_CONF_FILE} ${key} "log file" "/var/log/ceph/radosgw-$(hostname).log"
iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw data" "${dest}"
iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw print continue" "false"
iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw frontends" "civetweb port=${CEPH_RGW_PORT}"
iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone url" "$KEYSTONE_SERVICE_URI"
iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw s3 auth use keystone" "true"
iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone admin user" "radosgw"
iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone admin password" "$SERVICE_PASSWORD"
iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone accepted roles" "Member, _member_, admin, ResellerAdmin"
if [ "$CEPH_RGW_KEYSTONE_SSL" = "True" ]; then
iniset -sudo ${CEPH_CONF_FILE} ${key} "nss db path" "${dest}/nss"
else
iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone verify ssl" "false"
fi
if [[ $CEPH_RGW_IDENTITY_API_VERSION == '2.0' ]]; then
iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone admin tenant" "$SERVICE_PROJECT_NAME"
else
iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone admin project" "$SERVICE_PROJECT_NAME"
iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone admin domain" "$SERVICE_DOMAIN_NAME"
iniset -sudo ${CEPH_CONF_FILE} ${key} "rgw keystone api version" "3"
fi
}
function _configure_ceph_rgw_container {
_configure_rgw_ceph_section
sudo docker restart ceph-demo
}
function _configure_ceph_rgw {
# bootstrap rados gateway
_configure_rgw_ceph_section
sudo mkdir -p $dest
sudo ceph auth get-or-create $key \
osd 'allow rwx' mon 'allow rw' \
-o ${dest}/keyring
sudo systemctl enable ceph-radosgw@rgw.$(hostname)
sudo chown -R ceph. ${CEPH_DATA_DIR}
}
function _configure_ceph_iscsi_gateway {
iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "minimum_gateways" $CEPH_ISCSI_MINIMUM_GATEWAYS
iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "trusted_ip_list" "$HOST_IP,localhost"
iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "cluster_name" "ceph"
iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "gateway_keyring" "ceph.client.admin.keyring"
iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "prometheus_host" "$CEPH_ISCSI_API_HOST"
iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "api_secure" "false"
iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "api_user" "$CEPH_ISCSI_API_USER"
iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "api_password" "$CEPH_ISCSI_API_PASSWORD"
iniset -sudo ${CEPH_ISCSI_GATEWAY_CFG} config "api_port" "$CEPH_ISCSI_API_PORT"
}
function _configure_ceph_iscsi {
_configure_ceph_iscsi_gateway
sudo ceph -c ${CEPH_CONF_FILE} \
osd pool create ${CEPH_ISCSI_POOL} ${CEPH_ISCSI_POOL_PG}
sudo systemctl daemon-reload
sudo systemctl enable tcmu-runner
sudo systemctl enable rbd-target-gw
sudo systemctl enable rbd-target-api
}
function _post_start_configure_iscsi_gateway {
# Now we setup the rbd-target-gw and rbd-target-api for use
GWCLI=$(which gwcli)
removeme=$(sudo systemctl status rbd-target-api)
FQDN=$(hostname -f)
# create the target_iqn for exporting all volumes
sudo $GWCLI /iscsi-targets create $CEPH_ISCSI_TARGET_IQN
# now we add the gateway definition
# Didn't find the gateway, so lets create it
sudo $GWCLI /iscsi-targets/$CEPH_ISCSI_TARGET_IQN/gateways create $FQDN $HOST_IP skipchecks=true
}
function start_ceph_iscsi {
sudo systemctl start tcmu-runner
sudo systemctl start rbd-target-gw
sudo systemctl start rbd-target-api
sleep 10
# we have to setup the gateway and api after they start
_post_start_configure_iscsi_gateway
}
function stop_ceph_iscsi {
GWCLI=$(which gwcli)
FQDN=$(hostname -f)
sudo $GWCLI /iscsi-targets/$CEPH_ISCSI_TARGET_IQN/gateways delete $FQDN confirm=true
sudo $GWCLI /iscsi-targets delete $CEPH_ISCSI_TARGET_IQN
sudo systemctl stop rbd-target-api
sudo systemctl stop rbd-target-gw
sudo systemctl stop tcmu-runner
}
function _create_swift_endpoint {
local swift_service
swift_service=$(get_or_create_service "swift" "object-store" "Swift Service")
local swift_endpoint
swift_endpoint="$SWIFT_SERVICE_PROTOCOL://$SERVICE_HOST:${CEPH_RGW_PORT}/swift/v1"
get_or_create_endpoint $swift_service \
"$REGION_NAME" $swift_endpoint $swift_endpoint $swift_endpoint
}
function configure_ceph_embedded_rgw_paths {
if [[ "$CEPH_CONTAINERIZED" == "True" ]]; then
dest=${CEPH_DATA_DIR}/radosgw/$(hostname)
key=client.radosgw.gateway
else
dest=${CEPH_DATA_DIR}/radosgw/ceph-rgw.$(hostname)
key=client.rgw.$(hostname)
fi
}
function configure_ceph_embedded_rgw {
configure_ceph_embedded_rgw_paths
# keystone endpoint for radosgw
_create_swift_endpoint
# Create radosgw service user with admin privileges
create_service_user "radosgw" "admin"
if [ "$CEPH_RGW_KEYSTONE_SSL" = "True" ]; then
# radosgw needs to access keystone's revocation list
sudo mkdir -p ${dest}/nss
sudo openssl x509 -in /etc/keystone/ssl/certs/ca.pem -pubkey | \
sudo certutil -d ${dest}/nss -A -n ca -t "TCu,Cu,Tuw"
sudo openssl x509 -in /etc/keystone/ssl/certs/signing_cert.pem -pubkey | \
sudo certutil -A -d ${dest}/nss -n signing_cert -t "P,P,P"
fi
}
function start_ceph_embedded_rgw {
sudo systemctl enable ceph-radosgw@rgw.$(hostname)
sudo systemctl start ceph-radosgw@rgw.$(hostname)
}
function configure_ceph_embedded_glance {
# configure Glance service options, ceph pool, ceph user and ceph key
if [[ $CEPH_REPLICAS -ne 1 ]]; then
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool \
set ${GLANCE_CEPH_POOL} crush_ruleset ${RULE_ID}
fi
}
# configure_ceph_glance() - Glance config needs to come after Glance is set up
function configure_ceph_glance {
if [[ "$GLANCE_RGW_BACKEND" = "True" && "$ENABLE_CEPH_RGW" = "True" ]]; then
# common glance accounts for swift
create_service_user "glance-swift" "ResellerAdmin"
iniset $GLANCE_SWIFT_STORE_CONF ref1 user $SERVICE_PROJECT_NAME:glance-swift
AUTH_URL=$KEYSTONE_SERVICE_URI/v$CEPH_RGW_IDENTITY_API_VERSION
iniset $GLANCE_SWIFT_STORE_CONF ref1 key $SERVICE_PASSWORD
iniset $GLANCE_SWIFT_STORE_CONF ref1 auth_address $AUTH_URL
iniset $GLANCE_SWIFT_STORE_CONF ref1 user_domain_name $SERVICE_DOMAIN_NAME
iniset $GLANCE_SWIFT_STORE_CONF ref1 project_domain_name $SERVICE_DOMAIN_NAME
iniset $GLANCE_SWIFT_STORE_CONF ref1 auth_version $CEPH_RGW_IDENTITY_API_VERSION
iniset $GLANCE_API_CONF glance_store default_store swift
iniset $GLANCE_API_CONF glance_store swift_store_create_container_on_put True
iniset $GLANCE_API_CONF glance_store swift_store_config_file $GLANCE_SWIFT_STORE_CONF
iniset $GLANCE_API_CONF glance_store default_swift_reference ref1
iniset $GLANCE_API_CONF glance_store stores "file, http, swift"
else
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool create \
${GLANCE_CEPH_POOL} ${GLANCE_CEPH_POOL_PG} ${GLANCE_CEPH_POOL_PGP}
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} auth \
get-or-create client.${GLANCE_CEPH_USER} \
mon "allow r" \
osd "allow class-read object_prefix rbd_children, \
allow rwx pool=${GLANCE_CEPH_POOL}" | \
sudo tee ${CEPH_CONF_DIR}/ceph.client.${GLANCE_CEPH_USER}.keyring
sudo chown ${STACK_USER}:$(id -g -n $whoami) \
${CEPH_CONF_DIR}/ceph.client.${GLANCE_CEPH_USER}.keyring
iniset $GLANCE_API_CONF DEFAULT show_multiple_locations True
iniset $GLANCE_API_CONF glance_store default_store rbd
iniset $GLANCE_API_CONF glance_store stores "file, http, rbd"
iniset $GLANCE_API_CONF glance_store rbd_store_ceph_conf $CEPH_CONF_FILE
iniset $GLANCE_API_CONF glance_store rbd_store_user $GLANCE_CEPH_USER
iniset $GLANCE_API_CONF glance_store rbd_store_pool $GLANCE_CEPH_POOL
fi
}
function configure_ceph_manila {
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool create ${CEPHFS_METADATA_POOL} \
${CEPHFS_POOL_PG}
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool create ${CEPHFS_DATA_POOL} \
${CEPHFS_POOL_PG}
if [[ ${CEPHFS_MULTIPLE_FILESYSTEMS} == 'True' ]]; then
sudo ceph -c ${CEPH_CONF_FILE} fs flag set enable_multiple true \
--yes-i-really-mean-it
fi
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} fs new ${CEPHFS_FILESYSTEM} ${CEPHFS_METADATA_POOL} \
${CEPHFS_DATA_POOL}
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} auth get-or-create \
client.${MANILA_CEPH_USER} \
mon "allow *" osd "allow rw" mds "allow *" mgr "allow *" \
-o ${CEPH_CONF_DIR}/ceph.client.${MANILA_CEPH_USER}.keyring
sudo chown ${STACK_USER}:$(id -g -n $whoami) \
${CEPH_CONF_DIR}/ceph.client.${MANILA_CEPH_USER}.keyring
# Enable snapshots in CephFS.
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} fs set ${CEPHFS_FILESYSTEM} allow_new_snaps true \
--yes-i-really-mean-it
# Make manila's libcephfs client a root user.
iniset -sudo ${CEPH_CONF_FILE} client.${MANILA_CEPH_USER} "client mount uid" "0"
iniset -sudo ${CEPH_CONF_FILE} client.${MANILA_CEPH_USER} "client mount gid" "0"
if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
configure_nfs_ganesha
# NFS-Ganesha server cannot run alongwith with other kernel NFS server.
sudo systemctl stop nfs-server || true
sudo systemctl disable nfs-server || true
sudo systemctl enable nfs-ganesha
sudo systemctl start nfs-ganesha || (
echo "Ganesha didn't start. Let's debug..." >&2
sudo systemctl status nfs-ganesha || true
echo "**Ganesha conf file**" >&2
sudo cat /etc/ganesha/ganesha.conf || true
echo "**Ganesha log file**" >&2
sudo cat /var/log/ganesha/ganesha.log || true
echo "**Exiting**" >&2
exit 1
)
echo "Ganesha started successfully!" >&2
fi
# RESTART DOCKER CONTAINER
}
function configure_nfs_ganesha {
# Configure NFS-Ganesha to work with Manila's CephFS driver
sudo mkdir -p /etc/ganesha/export.d
if [ $MANILA_CEPH_GANESHA_RADOS_STORE == 'True' ]; then
# Create an empty placeholder ganesha export index object
echo | sudo rados -p ${CEPHFS_DATA_POOL} put ganesha-export-index -
cat <<EOF | sudo tee /etc/ganesha/ganesha.conf>/dev/null
RADOS_URLS {
ceph_conf = ${CEPH_CONF_FILE};
userid = admin;
}
CACHEINODE {
Dir_Max = 1;
Dir_Chunk = 0;
Cache_FDs = false;
NParts = 1;
Cache_Size = 1;
}
EXPORT_DEFAULTS {
Attr_Expiration_Time = 0;
}
%url rados://${CEPHFS_DATA_POOL}/ganesha-export-index
EOF
else
sudo touch /etc/ganesha/export.d/INDEX.conf
echo "%include /etc/ganesha/export.d/INDEX.conf" | sudo tee /etc/ganesha/ganesha.conf
fi
}
function cleanup_nfs_ganesha {
sudo systemctl stop nfs-ganesha
sudo systemctl disable nfs-ganesha
sudo uninstall_package nfs-ganesha nfs-ganesha-ceph libntirpc3 nfs-ganesha-rados-urls nfs-ganesha-vfs
}
function configure_ceph_embedded_manila {
if [[ $CEPH_REPLICAS -ne 1 ]]; then
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_DATA_POOL} \
crush_ruleset ${RULE_ID}
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_METADATA_POOL} \
crush_ruleset ${RULE_ID}
fi
}
function configure_ceph_embedded_nova {
# configure Nova service options, ceph pool, ceph user and ceph key
if [[ $CEPH_REPLICAS -ne 1 ]]; then
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool \
set ${NOVA_CEPH_POOL} crush_ruleset ${RULE_ID}
fi
}
# configure_ceph_nova() - Nova config needs to come after Nova is set up
function configure_ceph_nova {
# When REMOTE_CEPH=True is set on subnodes skip the creation of the nova
# pool as it has already been created on the controller that has
# REMOTE_CEPH=False.
if [[ "$REMOTE_CEPH" == "False" ]]; then
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool create \
${NOVA_CEPH_POOL} ${NOVA_CEPH_POOL_PG} ${NOVA_CEPH_POOL_PGP}
fi
iniset $NOVA_CONF libvirt rbd_user ${CINDER_CEPH_USER}
iniset $NOVA_CONF libvirt rbd_secret_uuid ${CINDER_CEPH_UUID}
iniset $NOVA_CONF libvirt inject_key false
iniset $NOVA_CONF libvirt inject_partition -2
iniset $NOVA_CONF libvirt disk_cachemodes "network=writeback"
iniset $NOVA_CONF libvirt images_type rbd
iniset $NOVA_CONF libvirt images_rbd_pool ${NOVA_CEPH_POOL}
iniset $NOVA_CONF libvirt images_rbd_ceph_conf ${CEPH_CONF_FILE}
if ! is_ceph_enabled_for_service cinder; then
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} \
auth get-or-create client.${CINDER_CEPH_USER} \
mon "allow r" \
osd "allow class-read object_prefix rbd_children, \
allow rwx pool=${CINDER_CEPH_POOL}, \
allow rwx pool=${NOVA_CEPH_POOL}, \
allow rwx pool=${GLANCE_CEPH_POOL}" | \
sudo tee \
${CEPH_CONF_DIR}/ceph.client.${CINDER_CEPH_USER}.keyring \
> /dev/null
sudo chown ${STACK_USER}:$(id -g -n $whoami) \
${CEPH_CONF_DIR}/ceph.client.${CINDER_CEPH_USER}.keyring
fi
}
function configure_ceph_embedded_cinder {
# Configure Cinder service options, ceph pool, ceph user and ceph key
if [[ $CEPH_REPLICAS -ne 1 ]]; then
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool \
set ${CINDER_CEPH_POOL} crush_ruleset ${RULE_ID}
fi
}
# configure_ceph_cinder() - Cinder config needs to come after Cinder is set up
function configure_ceph_cinder {
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool create \
${CINDER_CEPH_POOL} ${CINDER_CEPH_POOL_PG} ${CINDER_CEPH_POOL_PGP}
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} auth get-or-create \
client.${CINDER_CEPH_USER} \
mon "allow r" \
osd "allow class-read object_prefix rbd_children, \
allow rwx pool=${CINDER_CEPH_POOL}, allow rwx pool=${NOVA_CEPH_POOL}, \
allow rwx pool=${GLANCE_CEPH_POOL}" | \
sudo tee ${CEPH_CONF_DIR}/ceph.client.${CINDER_CEPH_USER}.keyring
sudo chown ${STACK_USER}:$(id -g -n $whoami) \
${CEPH_CONF_DIR}/ceph.client.${CINDER_CEPH_USER}.keyring
sudo rbd pool init ${CINDER_CEPH_POOL}
}
# install_ceph() - Collect source and prepare
function install_ceph_remote {
install_package ceph-common
# ceph-common in Bionic (18.04) installs only the python2 variants of
# required packages, meaning we need to install the python3 variants
# manually. Hopefully this won't be necessary in Focal (20.04)
# https://packages.ubuntu.com/bionic/ceph-common
if python3_enabled; then
install_package python3-cephfs python3-prettytable python3-rados python3-rbd python3-requests
fi
# Since pip10, pip will refuse to uninstall files from packages
# that were created with distutils (rather than more modern
# setuptools). This is because it technically doesn't have a
# manifest of what to remove. However, in most cases, simply
# overwriting works. So this hacks around those packages that
# have been dragged in by some other system dependency
if is_ubuntu; then
sudo rm -rf /usr/lib/python3/dist-packages/logutils*.egg-info
fi
if is_fedora; then
sudo rm -rf /usr/lib64/python3*/site-packages/logutils*.egg-info
fi
}
function dnf_add_repository_ceph {
local ceph_release=$1
local distro_release=$2
cat > ceph.repo <<EOF
[ceph]
name=Ceph packages for \$basearch
baseurl=https://download.ceph.com/rpm-${ceph_release}/${distro_release}/\$basearch
enabled=1
priority=2
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.asc
[ceph-noarch]
name=Ceph noarch packages
baseurl=https://download.ceph.com/rpm-${ceph_release}/${distro_release}/noarch
enabled=1
priority=2
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.asc
[ceph-source]
name=Ceph source packages
baseurl=https://download.ceph.com/rpm-${ceph_release}/${distro_release}/SRPMS
enabled=0
priority=2
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.asc
EOF
sudo dnf config-manager --add-repo ceph.repo
}
function dnf_add_repository_nfsganesha {
local ganesha_release=$1
local ceph_release=$2
cat > nfs-ganesha.repo <<EOF
[nfs-ganesha]
name=NFS Ganesha packages for \$basearch
baseurl=https://download.ceph.com/nfs-ganesha/${ganesha_release}/${ceph_release}/el8/\$basearch
enabled=1
priority=2
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.asc
[nfs-ganesha-noarch]
name=NFS Ganesha noarch packages
baseurl=https://download.ceph.com/nfs-ganesha/${ganesha_release}/${ceph_release}/el8/noarch
enabled=1
priority=2
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.asc
[nfs-ganesha-source]
name=NFS Ganesha source packages
baseurl=https://download.ceph.com/nfs-ganesha/${ganesha_release}/${ceph_release}/el8/SRPMS
enabled=0
priority=2
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.asc
EOF
sudo dnf config-manager --add-repo nfs-ganesha.repo
}
# configure_repo_ceph() - Configure Ceph repositories
# Usage: configure_repo_ceph <package_manager> <ceph_release> <distro_type> \
# <distro_release> [<repo_type>]
# - package_manager: apt-get or yum
# - ceph_release: luminous, ...
# - distro_release: 7, xenial, bionic
function configure_repo_ceph {
local package_manager="$1"
local ceph_release="$2"
local distro_release="$3"
if is_ubuntu; then
if [[ "${TEST_MASTER}" == "True" ]]; then
repo_file_name="/etc/apt/sources.list.d/ceph-master.list"
sudo wget -c "https://shaman.ceph.com/api/repos/ceph/master/latest/ubuntu/${distro_release}/flavors/default/repo" -O ${repo_file_name}
else
wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
sudo apt-add-repository -y "deb https://download.ceph.com/debian-${ceph_release}/ ${distro_release} main"
fi
sudo ${package_manager} -y update
elif is_fedora; then
if [[ "${TEST_MASTER}" == "True" ]]; then
repo_file_name="/etc/yum.repos.d/ceph-master.repo"
sudo wget -c "https://shaman.ceph.com/api/repos/ceph/master/latest/centos/${distro_release}/flavors/default/repo" -O ${repo_file_name}
sudo dnf config-manager --add-repo ${repo_file_name}
else
dnf_add_repository_ceph ${ceph_release} ${distro_release}
fi
fi
}
# cleanup_repo_ceph() - Remove Ceph repositories
# Usage: cleanup_repo_ceph
function cleanup_repo_ceph {
if is_ubuntu; then
sudo apt-add-repository -r -y "deb https://download.ceph.com/debian-${CEPH_RELEASE}/ ${os_CODENAME} main"
elif is_fedora; then
sudo rm -rf /etc/yum.repos.d/ceph.repo
fi
}
# configure_repo_nfsganesha() - Configure NFS Ganesha repositories
# Usage: configure_repo_nfsganesha <package_manager> <ganesha_flavor> \
# <distro_type> <distro_release> [<repo_type>]
# - package_manager: apt-get or dnf
# - ganesha_release: 2.7, 2.8, 3.0
# - ceph_release: ceph_luminous, ceph_nautilus
function configure_repo_nfsganesha {
local package_manager="$1"
local ganesha_release="$2"
local ceph_release="$3"
if is_ubuntu; then
# FIXME(vkmc) We need to use community ppa's because there are no builds
# for ubuntu bionic and ubuntu focal available for nfs-ganesha 2.7 and above
# Remove this when they provide the build in download.ceph.com
# FIXME(vkmc) Community ppa's don't provide builds for nfs-ganesha-3
# microversions (3.3, 3.5). Default to latest.
if [[ $GANESHA_RELEASE =~ V3.(3|5)-stable ]]; then
sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-3.0
sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-3.0
elif [ $GANESHA_RELEASE == 'V2.8-stable' ]; then
sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-1.8
sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-2.8
elif [ $GANESHA_RELEASE == 'V2.7-stable' ]; then
sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-1.7
sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-2.7
else
die $LINENO "GANESHA_RELEASE is not supported by the Ceph plugin for Devstack"
fi
sudo ${package_manager} -y update
elif is_fedora; then
dnf_add_repository_nfsganesha rpm-${ganesha_release} ${ceph_release}
fi
}
# cleanup_repo_nfsganesha() - Remove NFS Ganesha repositories
# Usage: cleanup_repo_nfsganesha
function cleanup_repo_nfsganesha {
if is_ubuntu; then
# FIXME(vkmc) We need to use community ppa's because there are no builds
# for ubuntu bionic and ubuntu focal available for nfs-ganesha 2.7 and above
# Remove this when they provide the builds in download.ceph.com
# FIXME(vkmc) Community ppa's don't provide builds for nfs-ganesha-3
# microversions (3.3, 3.5). Default to latest.
if [[ $GANESHA_RELEASE =~ V3.(3|5)-stable ]]; then
sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-libntirpc-3_0-*.list*
sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-nfs-ganesha-3_0-*.list*
elif [ $GANESHA_RELEASE == 'V2.8-stable' ]; then
sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-libntirpc-1_8-*.list*
sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-nfs-ganesha-2_8-*.list*
elif [ $GANESHA_RELEASE == 'V2.7-stable' ]; then
sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-libntirpc-1_7-*.list*
sudo rm -rf /etc/apt/sources.list.d/nfs-ganesha-ubuntu-nfs-ganesha-2_7-*.list*
else
die $LINENO "GANESHA_RELEASE is not supported by the Ceph plugin for Devstack"
fi
elif is_fedora; then
sudo rm -rf /etc/yum.repos.d/nfs-ganesha.repo
fi
}
function setup_packages_for_manila_on_ubuntu {
CEPH_PACKAGES="${CEPH_PACKAGES} ceph-mds libcephfs2"
if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
configure_repo_nfsganesha "apt-get" "$GANESHA_RELEASE" "$CEPH_RELEASE"
CEPH_PACKAGES="${CEPH_PACKAGES} libntirpc3 nfs-ganesha nfs-ganesha-ceph \
nfs-ganesha-rados-urls nfs-ganesha-vfs"
fi
if python3_enabled; then
CEPH_PACKAGES="${CEPH_PACKAGES} python3-cephfs"
fi
}
function setup_packages_for_manila_on_fedora_family {
if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
# NOTE(vkmc) el7 packages work on Fedora
configure_repo_nfsganesha "dnf" "$GANESHA_RELEASE" "$CEPH_RELEASE"
CEPH_PACKAGES="${CEPH_PACKAGES} nfs-ganesha nfs-ganesha-ceph \
nfs-ganesha-rados-urls nfs-ganesha-vfs"
fi
}
function install_ceph {
if is_ubuntu; then
if ! [[ $os_CODENAME =~ (focal|xenial|bionic) ]]; then
die $LINENO "Supported for Ubuntu Xenial, Bionic or Focal. Not supported for other releases."
fi
# NOTE(vkmc) Dependencies for setting up repos
install_package software-properties-common
configure_repo_ceph "apt-get" "$CEPH_RELEASE" "$os_CODENAME"
CEPH_PACKAGES="ceph libnss3-tools"
if python3_enabled; then
CEPH_PACKAGES="${CEPH_PACKAGES} python3-rados python3-rbd"
fi
if is_ceph_enabled_for_service manila; then
setup_packages_for_manila_on_ubuntu
fi
if [ "$ENABLE_CEPH_RGW" = "True" ]; then
CEPH_PACKAGES="${CEPH_PACKAGES} radosgw"
fi
if [ "$ENABLE_CEPH_ISCSI" = "True" ]; then
# Only Ubuntu Focal have the required packages. Using distro provided
# packages might be more stable for CI.
if [[ $os_CODENAME =~ focal ]]; then
CEPH_PACKAGES="${CEPH_PACKAGES} ceph-iscsi targetcli-fb "
else
die $LINENO "For ubuntu, ceph iscsi only support focal now"
fi
fi
install_package ${CEPH_PACKAGES}
elif is_fedora; then
if ! [[ $os_VENDOR =~ Fedora ]] && [[ $os_RELEASE =~ (31|32) ]]; then
die $LINENO "Supported for Fedora 31 and 32. Not supported for other releases."
fi
# NOTE(lyarwood) Use the py3 el8 packages on Fedora
configure_repo_ceph "dnf" "$CEPH_RELEASE" "el8"
CEPH_PACKAGES="ceph"
if is_ceph_enabled_for_service manila; then
setup_packages_for_manila_on_fedora_family
fi
if [ "$ENABLE_CEPH_RGW" = "True" ]; then
CEPH_PACKAGES="${CEPH_PACKAGES} ceph-radosgw"
fi
if [ "$ENABLE_CEPH_ISCSI" = "True" ]; then
# TODO(xinliang): Install shaman ceph iscsi gateway packages like ceph-ansible:
# https://github.com/ceph/ceph-ansible/blob/6dd9b255656b7124f8963cf65a862930fa28d162/roles/ceph-iscsi-gw/tasks/non-container/prerequisites.yml#L2
die $LINENO "Ceph iscsi gateway is not supported for ${os_VENDOR} distro yet"
fi
install_package ${CEPH_PACKAGES}
else
die $LINENO "${os_VENDOR} is not supported by the Ceph plugin for Devstack"
fi
if is_ubuntu; then
sudo rm -rf /usr/lib/python3/dist-packages/logutils*.egg-info
fi
if is_fedora; then
sudo rm -rf /usr/lib64/python3*/site-packages/logutils*.egg-info
fi
}
# start_ceph() - Start running processes, including screen
function start_ceph {
sudo chown -R ceph. ${CEPH_DATA_DIR}
sudo systemctl start ceph-mon@$(hostname)
local ceph_version
ceph_version=$(_get_ceph_version mon)
sudo systemctl start ceph-mgr@${MGR_ID}
# use `tell mgr` as the mgr might not have been activated
# yet to register the python module commands.
if ! sudo ceph -c ${CEPH_CONF_FILE} tell mgr restful create-self-signed-cert; then
echo MGR Restful is not working, perhaps the package is not installed?
fi
for id in $(sudo ceph -c ${CEPH_CONF_FILE} osd ls); do
sudo systemctl start ceph-osd@$id
done
if is_ceph_enabled_for_service manila; then
sudo systemctl start ceph-mds@${MDS_ID}
fi
if [ "$ENABLE_CEPH_ISCSI" = "True" ]; then
start_ceph_iscsi
fi
}
# stop_ceph() - Stop running processes (non-screen)
function stop_ceph {
if [ "$ENABLE_CEPH_RGW" = "True" ]; then
sudo systemctl stop ceph-radosgw@rgw.$(hostname)
sudo systemctl disable ceph-radosgw@rgw.$(hostname)
fi
if [ "$ENABLE_CEPH_ISCSI" = "True" ]; then
stop_ceph_iscsi
fi
if is_ceph_enabled_for_service manila; then
if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
sudo systemctl stop nfs-ganesha
sudo systemctl disable nfs-ganesha
fi
sudo systemctl stop ceph-mds@${MDS_ID}
sudo systemctl disable ceph-mds@${MDS_ID}
fi
# if mon is dead or unhealthy we won't get the list
# of osds but should continue anyways.
ids=$(sudo ceph -c ${CEPH_CONF_FILE} osd ls 2>/dev/null --connect-timeout 5)
for id in $ids; do
sudo systemctl stop ceph-osd@$id
sudo systemctl disable ceph-osd@$id
done
sudo systemctl stop ceph-mgr@${MGR_ID}
sudo systemctl disable ceph-mgr@${MGR_ID}
# In nautilus we have the new ceph-crash service for monitoring
# Try to stop it. If there is no service, stop/disable do nothing
sudo systemctl stop ceph-crash
sudo systemctl disable ceph-crash
sudo systemctl stop ceph-mon@$(hostname)
sudo systemctl disable ceph-mon@$(hostname)
# NOTE(vkmc) Cleanup any leftover unit files
sudo rm -f /etc/systemd/system/ceph*
# Clean up CEPH_DATA_DIR
sudo umount ${CEPH_DATA_DIR}
# devstack create_disk function adds an entry in /etc/fstab
# that doesn't always get cleaned up on restacks so clean it
# up here. Use a pattern that escapes the '/' path delimiters
# in the expresssion given to the sed command.
pattern=$(echo $CEPH_DATA_DIR | sed 's_/_\\/_g')
sudo sed -i "/$pattern/d" /etc/fstab
sudo rm -rf ${CEPH_DATA_DIR}/*
}
# Restore xtrace
$XTRACE
## Local variables:
## mode: shell-script
## End: