From 6a0feecaef478c2764cf138a5c0cdc4e46cce080 Mon Sep 17 00:00:00 2001
From: Chinasubbareddy Mallavarapu <cr3938@att.com>
Date: Sat, 10 Oct 2020 21:32:49 -0500
Subject: [PATCH] [ceph-osd] Fix the sync issue between osds when using shared
 disk for metadata

This is to fix the sync between ceph osds when they are using shared
disk for metadata as they are having conflict while preparing the metadata disk.

we are adding a lock when first osd preparing the sahred metadata disk so that
other osd will wait for the lock, also adding udev settle in few places to get
latest tags on lvm devices.

Change-Id: I018bd12a3f02cf8cd3486b9c97e14b138b5dac76
---
 ceph-osd/Chart.yaml                                   |  2 +-
 ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl |  6 ++++++
 .../bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl | 11 +++++++----
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/ceph-osd/Chart.yaml b/ceph-osd/Chart.yaml
index 0646ccb73..abd9da7e3 100644
--- a/ceph-osd/Chart.yaml
+++ b/ceph-osd/Chart.yaml
@@ -15,6 +15,6 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Ceph OSD
 name: ceph-osd
-version: 0.1.4
+version: 0.1.5
 home: https://github.com/ceph/ceph
 ...
diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl
index 029050289..8476e9a9e 100644
--- a/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl
+++ b/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl
@@ -80,6 +80,12 @@ function locked() {
   "$@"
   flock -u "${lock_fd}"
 }
+function global_locked() {
+  exec {global_lock_fd}>/var/lib/ceph/tmp/init-osd-global.lock || exit 1
+  flock -w 600 --verbose "${global_lock_fd}"
+  "$@"
+  flock -u "${global_lock_fd}"
+}
 
 function crush_create_or_move {
   local crush_location=${1}
diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl
index 6cd9d5b8f..4bde7157c 100644
--- a/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl
+++ b/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl
@@ -185,6 +185,7 @@ function prep_device {
   local device_type=$3
   local data_disk=$4
   local vg_name lv_name VG DEVICE_OSD_ID logical_devices logical_volume
+  udev_settle
   vg_name=$(get_vg_name_from_device ${BLOCK_DEVICE})
   lv_name=$(get_lv_name_from_device ${data_disk} ${device_type})
   VG=$(vgs --noheadings -o vg_name -S "vg_name=${vg_name}" | tr -d '[:space:]')
@@ -219,6 +220,7 @@ function prep_device {
   elif [[ "${device_type}" == "wal" ]]; then
     BLOCK_WAL="${VG}/${lv_name}"
   fi
+  udev_settle
 }
 
 function osd_disk_prepare {
@@ -379,6 +381,7 @@ function osd_disk_prepare {
   fi
 
   if [[ ${CEPH_DISK_USED} -eq 1 ]]; then
+    udev_settle
     CLI_OPTS="${CLI_OPTS} --data ${OSD_DEVICE}"
     ceph-volume simple scan --force ${OSD_DEVICE}$(sgdisk --print ${OSD_DEVICE} | grep "F800" | awk '{print $1}')
   elif [[ ${CEPH_LVM_PREPARE} -eq 1 ]] || [[ ${DISK_ZAPPED} -eq 1 ]]; then
@@ -411,12 +414,12 @@ function osd_disk_prepare {
       block_wal_string=$(echo ${BLOCK_WAL} | awk -F "/" '{print $2 "-" $3}')
     fi
     if [[ ${BLOCK_DB} && ${BLOCK_WAL} ]]; then
-      prep_device "${BLOCK_DB}" "${BLOCK_DB_SIZE}" "db" "${OSD_DEVICE}"
-      prep_device "${BLOCK_WAL}" "${BLOCK_WAL_SIZE}" "wal" "${OSD_DEVICE}"
+      global_locked prep_device "${BLOCK_DB}" "${BLOCK_DB_SIZE}" "db" "${OSD_DEVICE}"
+      global_locked prep_device "${BLOCK_WAL}" "${BLOCK_WAL_SIZE}" "wal" "${OSD_DEVICE}"
     elif [[ -z ${BLOCK_DB} && ${BLOCK_WAL} ]]; then
-      prep_device "${BLOCK_WAL}" "${BLOCK_WAL_SIZE}" "wal" "${OSD_DEVICE}"
+      global_locked prep_device "${BLOCK_WAL}" "${BLOCK_WAL_SIZE}" "wal" "${OSD_DEVICE}"
     elif [[ ${BLOCK_DB} && -z ${BLOCK_WAL} ]]; then
-      prep_device "${BLOCK_DB}" "${BLOCK_DB_SIZE}" "db" "${OSD_DEVICE}"
+      global_locked prep_device "${BLOCK_DB}" "${BLOCK_DB_SIZE}" "db" "${OSD_DEVICE}"
     fi
   else
     if pvdisplay ${OSD_DEVICE} | awk '/VG Name/{print $3}' | grep "ceph"; then