From c20b2f8463e962734222382a77f26b91a53fd264 Mon Sep 17 00:00:00 2001
From: Stanislav Makar <smakar@mirantis.com>
Date: Tue, 31 Mar 2015 09:07:18 +0000
Subject: [PATCH] Fix floating problem with OSD down

* Add the OSD activation and check if osd process for each OSD is started.
* Change the process of adding OSDs to cluster. Now Ceph OSDs are added one by
 one instead of by all together. This allows to check the every OSD status
 during adding to cluster.

Change-Id: I8e64c1b15ed92e6fb5939b6f41728efacae64319
Closes-bug: #1419884
---
 deployment/puppet/ceph/manifests/init.pp     |  4 +--
 deployment/puppet/ceph/manifests/osd.pp      | 31 --------------------
 deployment/puppet/ceph/manifests/osds.pp     | 15 ++++++++++
 deployment/puppet/ceph/manifests/osds/osd.pp | 29 ++++++++++++++++++
 4 files changed, 46 insertions(+), 33 deletions(-)
 delete mode 100644 deployment/puppet/ceph/manifests/osd.pp
 create mode 100644 deployment/puppet/ceph/manifests/osds.pp
 create mode 100644 deployment/puppet/ceph/manifests/osds/osd.pp

diff --git a/deployment/puppet/ceph/manifests/init.pp b/deployment/puppet/ceph/manifests/init.pp
index 1a1c42fe6a..6bd9bbeb4e 100644
--- a/deployment/puppet/ceph/manifests/init.pp
+++ b/deployment/puppet/ceph/manifests/init.pp
@@ -132,8 +132,8 @@ class ceph (
 
     'ceph-osd': {
       if ! empty($osd_devices) {
-        include ceph::osd
-        Class['ceph::conf'] -> Class['ceph::osd'] ~> Service['ceph']
+        include ceph::osds
+        Class['ceph::conf'] -> Class['ceph::osds'] ~> Service['ceph']
       }
     }
 
diff --git a/deployment/puppet/ceph/manifests/osd.pp b/deployment/puppet/ceph/manifests/osd.pp
deleted file mode 100644
index ca149496b2..0000000000
--- a/deployment/puppet/ceph/manifests/osd.pp
+++ /dev/null
@@ -1,31 +0,0 @@
-# prepare and bring online the devices listed in $::ceph::osd_devices
-class ceph::osd (
-  $devices = join(prefix($::ceph::osd_devices, "${::hostname}:"), ' '),
-){
-  firewall {'011 ceph-osd allow':
-    chain   => 'INPUT',
-    dport   => '6800-7100',
-    proto   => 'tcp',
-    action  => accept,
-  }
-
-  exec { 'ceph-deploy osd prepare':
-    # ceph-deploy osd prepare is ensuring there is a filesystem on the
-    # disk according to the args passed to ceph.conf (above).
-    #
-    # It has a long timeout because of the format taking forever. A
-    # resonable amount of time would be around 300 times the length of
-    # $osd_nodes. Right now its 0 to prevent puppet from aborting it.
-
-    command   => "ceph-deploy osd prepare ${devices}",
-    returns   => 0,
-    timeout   => 0, # TODO: make this something reasonable
-    tries     => 2, # This is necessary because of race for mon creating keys
-    try_sleep => 1,
-    logoutput => true,
-    unless    => "grep -q '^${ $::ceph::osd_devices[0] }' /proc/mounts",
-  }
-
-  Firewall['011 ceph-osd allow'] ->
-  Exec['ceph-deploy osd prepare']
-}
diff --git a/deployment/puppet/ceph/manifests/osds.pp b/deployment/puppet/ceph/manifests/osds.pp
new file mode 100644
index 0000000000..32814156c8
--- /dev/null
+++ b/deployment/puppet/ceph/manifests/osds.pp
@@ -0,0 +1,15 @@
+# prepare and bring online the devices listed in $::ceph::osd_devices
+class ceph::osds (
+  $devices = $::ceph::osd_devices,
+){
+
+  firewall { '011 ceph-osd allow':
+    chain  => 'INPUT',
+    dport  => '6800-7100',
+    proto  => 'tcp',
+    action => accept,
+  } ->
+
+  ceph::osds::osd{ $devices: }
+
+}
diff --git a/deployment/puppet/ceph/manifests/osds/osd.pp b/deployment/puppet/ceph/manifests/osds/osd.pp
new file mode 100644
index 0000000000..61495bcc0b
--- /dev/null
+++ b/deployment/puppet/ceph/manifests/osds/osd.pp
@@ -0,0 +1,29 @@
+define ceph::osds::osd () {
+
+  $deploy_device_name = "${::hostname}:${name}"
+
+  exec { "ceph-deploy osd prepare ${deploy_device_name}":
+    # ceph-deploy osd prepare is ensuring there is a filesystem on the
+    # disk according to the args passed to ceph.conf (above).
+    #
+    # It has a long timeout because of the format taking forever. A
+    # resonable amount of time would be around 300 times the length of
+    # $osd_nodes. Right now its 0 to prevent puppet from aborting it.
+    command   => "ceph-deploy osd prepare ${deploy_device_name}",
+    returns   => 0,
+    timeout   => 0, # TODO: make this something reasonable
+    tries     => 2, # This is necessary because of race for mon creating keys
+    try_sleep => 1,
+    logoutput => true,
+    unless    => "grep -q ${name} /proc/mounts",
+  } ->
+
+  exec { "ceph-deploy osd activate ${deploy_device_name}":
+    command   => "ceph-deploy osd activate ${deploy_device_name}",
+    try_sleep => 10,
+    tries     => 6,
+    logoutput => true,
+    unless    => "ceph osd dump | grep -q \"osd.$(sed -nEe 's|${name}\ .*ceph-([0-9]+).*$|\1|p' /proc/mounts)\ up\ .*\ in\ \"",
+  }
+
+}