openstack/openstack-helm-infra
Code Issues Proposed changes

[ceph-osd] Fix the sync issue between osds when using shared disk for metadata

Browse Source 
This is to fix the sync between ceph osds when they are using shared
disk for metadata as they are having conflict while preparing the metadata disk.

we are adding a lock when first osd preparing the sahred metadata disk so that
other osd will wait for the lock, also adding udev settle in few places to get
latest tags on lvm devices.

Change-Id: I018bd12a3f02cf8cd3486b9c97e14b138b5dac76
This commit is contained in:
Chinasubbareddy Mallavarapu 2020-10-10 21:32:49 -05:00 committed by chinasubbareddy mallavarapu
parent 3e52027702
commit 6a0feecaef
3 changed files with 14 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ceph-osd/Chart.yaml
View File

@ -15,6 +15,6 @@ apiVersion: v1
appVersion: v1.0.0 appVersion: v1.0.0
description: OpenStack-Helm Ceph OSD description: OpenStack-Helm Ceph OSD
name: ceph-osd name: ceph-osd
version: 0.1.4 version: 0.1.5
home: https://github.com/ceph/ceph home: https://github.com/ceph/ceph
... ...

6
ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl
View File

@ -80,6 +80,12 @@ function locked() {
"$@" "$@"
flock -u "${lock_fd}" flock -u "${lock_fd}"
} }
function global_locked() {
exec {global_lock_fd}>/var/lib/ceph/tmp/init-osd-global.lock || exit 1
flock -w 600 --verbose "${global_lock_fd}"
"$@"
flock -u "${global_lock_fd}"
}
function crush_create_or_move { function crush_create_or_move {
local crush_location=${1} local crush_location=${1}

11
ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl
View File

@ -185,6 +185,7 @@ function prep_device {
local device_type=$3 local device_type=$3
local data_disk=$4 local data_disk=$4
local vg_name lv_name VG DEVICE_OSD_ID logical_devices logical_volume local vg_name lv_name VG DEVICE_OSD_ID logical_devices logical_volume
udev_settle
vg_name=$(get_vg_name_from_device ${BLOCK_DEVICE}) vg_name=$(get_vg_name_from_device ${BLOCK_DEVICE})
lv_name=$(get_lv_name_from_device ${data_disk} ${device_type}) lv_name=$(get_lv_name_from_device ${data_disk} ${device_type})
VG=$(vgs --noheadings -o vg_name -S "vg_name=${vg_name}" | tr -d '[:space:]') VG=$(vgs --noheadings -o vg_name -S "vg_name=${vg_name}" | tr -d '[:space:]')
@ -219,6 +220,7 @@ function prep_device {
elif [[ "${device_type}" == "wal" ]]; then elif [[ "${device_type}" == "wal" ]]; then
BLOCK_WAL="${VG}/${lv_name}" BLOCK_WAL="${VG}/${lv_name}"
fi fi
udev_settle
} }
function osd_disk_prepare { function osd_disk_prepare {
@ -379,6 +381,7 @@ function osd_disk_prepare {
fi fi
if [[ ${CEPH_DISK_USED} -eq 1 ]]; then if [[ ${CEPH_DISK_USED} -eq 1 ]]; then
udev_settle
CLI_OPTS="${CLI_OPTS} --data ${OSD_DEVICE}" CLI_OPTS="${CLI_OPTS} --data ${OSD_DEVICE}"
ceph-volume simple scan --force ${OSD_DEVICE}$(sgdisk --print ${OSD_DEVICE} | grep "F800" | awk '{print $1}') ceph-volume simple scan --force ${OSD_DEVICE}$(sgdisk --print ${OSD_DEVICE} | grep "F800" | awk '{print $1}')
elif [[ ${CEPH_LVM_PREPARE} -eq 1 ]] || [[ ${DISK_ZAPPED} -eq 1 ]]; then elif [[ ${CEPH_LVM_PREPARE} -eq 1 ]] || [[ ${DISK_ZAPPED} -eq 1 ]]; then
@ -411,12 +414,12 @@ function osd_disk_prepare {
block_wal_string=$(echo ${BLOCK_WAL} | awk -F "/" '{print $2 "-" $3}') block_wal_string=$(echo ${BLOCK_WAL} | awk -F "/" '{print $2 "-" $3}')
fi fi
if [[ ${BLOCK_DB} && ${BLOCK_WAL} ]]; then if [[ ${BLOCK_DB} && ${BLOCK_WAL} ]]; then
prep_device "${BLOCK_DB}" "${BLOCK_DB_SIZE}" "db" "${OSD_DEVICE}" global_locked prep_device "${BLOCK_DB}" "${BLOCK_DB_SIZE}" "db" "${OSD_DEVICE}"
prep_device "${BLOCK_WAL}" "${BLOCK_WAL_SIZE}" "wal" "${OSD_DEVICE}" global_locked prep_device "${BLOCK_WAL}" "${BLOCK_WAL_SIZE}" "wal" "${OSD_DEVICE}"
elif [[ -z ${BLOCK_DB} && ${BLOCK_WAL} ]]; then elif [[ -z ${BLOCK_DB} && ${BLOCK_WAL} ]]; then
prep_device "${BLOCK_WAL}" "${BLOCK_WAL_SIZE}" "wal" "${OSD_DEVICE}" global_locked prep_device "${BLOCK_WAL}" "${BLOCK_WAL_SIZE}" "wal" "${OSD_DEVICE}"
elif [[ ${BLOCK_DB} && -z ${BLOCK_WAL} ]]; then elif [[ ${BLOCK_DB} && -z ${BLOCK_WAL} ]]; then
prep_device "${BLOCK_DB}" "${BLOCK_DB_SIZE}" "db" "${OSD_DEVICE}" global_locked prep_device "${BLOCK_DB}" "${BLOCK_DB_SIZE}" "db" "${OSD_DEVICE}"
fi fi
else else
if pvdisplay ${OSD_DEVICE} | awk '/VG Name/{print $3}' | grep "ceph"; then if pvdisplay ${OSD_DEVICE} | awk '/VG Name/{print $3}' | grep "ceph"; then