Support Ceph OSD devices with existing data set

Partition preservation feature allows to preserve data on Ceph OSD device,
but later on Puppet will run 'ceph-deploy prepare' on every Ceph device in
a system. This call destroys data set on those devices.

To preserve data on Ceph OSD devices through deployment process, we need to
check if the device has Ceph data and if so, skip execution of 'ceph-deploy 
prepare' on that device. Only prepared devices must be activated to avoid
deployment failure.

Following steps are made in the strict sequence to ensure protection of 
the existing data:

* execute 'udevadm trigger' to force create links for devices in /dev/disk
* execute 'ceph-disk activate-all' to force activation of all devices
  identified as ceph osds
* in ceph::osds::osd class for every ceph osd device check if it is prepared
  or active, if not, run 'ceph-deploy prepare' on it
* for every osd device, check if it is prepared (not active) and activate it

Change-Id: I667fa6aab9d6f46c73bfb8ca0e267afede6049fb
Implements: blueprint upgrade-redeploy-node

deployment/puppet/ceph/manifests/osds.pp

@@ -1,8 +1,28 @@
-# prepare and bring online the devices listed in $::ceph::osd_devices
+# == Class: ceph::osd
+#
+# Prepare and bring online the OSD devices
+#
+# ==== Parameters
+#
+# [*devices*]
+# (optional) Array. This is the list of OSD devices identified by the facter.
+#
 class ceph::osds (
   $devices = $::ceph::osd_devices,
 ){
 
+  exec { 'udevadm trigger':
+    command     => 'udevadm trigger',
+    returns     => 0,
+    logoutput   => true,
+  } ->
+
+  exec {'ceph-disk activate-all':
+    command     => 'ceph-disk activate-all',
+    returns     => 0,
+    logoutput   => true,
+  } ->
+
   firewall { '011 ceph-osd allow':
     chain  => 'INPUT',
     dport  => '6800-7100',
@@ -11,5 +31,4 @@ class ceph::osds (
   } ->
 
   ceph::osds::osd{ $devices: }
-
 }

deployment/puppet/ceph/manifests/osds/osd.pp

@@ -1,3 +1,7 @@
+# == Define: ceph::osds::osd
+#
+# Prepare and activate OSD nodes on the node
+#
 define ceph::osds::osd () {
 
   # ${name} format is DISK[:JOURNAL]
@@ -18,8 +22,8 @@ define ceph::osds::osd () {
     tries     => 2, # This is necessary because of race for mon creating keys
     try_sleep => 1,
     logoutput => true,
-    unless    => "grep -q ${data_device_name} /proc/mounts",
-  } ->
+    unless    => "ceph-disk list | fgrep -q -e '${data_device_name} ceph data, active' -e '${data_device_name} ceph data, prepared'",
+  } -> Exec["ceph-deploy osd activate ${deploy_device_name}"]
 
   exec { "ceph-deploy osd activate ${deploy_device_name}":
     command   => "ceph-deploy osd activate ${deploy_device_name}",
@@ -27,7 +31,7 @@ define ceph::osds::osd () {
     tries     => 3,
     logoutput => true,
     timeout   => 0,
-    unless    => "ceph osd dump | grep -q \"osd.$(sed -nEe 's|${data_device_name}\\ .*ceph-([0-9]+).*$|\\1|p' /proc/mounts)\\ up\\ .*\\ in\\ \"",
+    onlyif    => "ceph-disk list | fgrep -q -e '${data_device_name} ceph data, prepared'",
   }
 
 }

deployment/puppet/ceph/spec/classes/osds__spec.rb

@@ -5,9 +5,11 @@ describe 'ceph::osds', :type => :class do
   context 'Simple ceph::osds class test' do
     let (:params) {{ :devices => ['/dev/vdc', '/dev/vdd' ] }}
 
-    it { should contain_firewall('011 ceph-osd allow') }
-    it { should contain_ceph__osds__osd('/dev/vdc') }
-    it { should contain_ceph__osds__osd('/dev/vdd') }
+    it { should contain_exec('udevadm trigger') }
+    it { should contain_exec('ceph-disk activate-all').that_requires('Exec[udevadm trigger]') }
+    it { should contain_firewall('011 ceph-osd allow').that_requires('Exec[ceph-disk activate-all]') }
+    it { should contain_ceph__osds__osd('/dev/vdc').that_requires('Firewall[011 ceph-osd allow]') }
+    it { should contain_ceph__osds__osd('/dev/vdd').that_requires('Firewall[011 ceph-osd allow]') }
   end
 
 
@@ -19,7 +21,7 @@ describe 'ceph::osds', :type => :class do
   end
 
   context 'Class ceph::osds with devices and journals' do
-    let (:params) {{ :devices => ['/dev/sdc1:/dev/sdc2', '/dev/sdd1:/dev/sdd2' ] }}
+    let (:params) {{ :devices => ['/dev/sdc1:/dev/sdc2', '/dev/sdd1:/dev/sdd2'] }}
 
     it { should contain_firewall('011 ceph-osd allow') }
     it { should contain_ceph__osds__osd('/dev/sdc1:/dev/sdc2') }

deployment/puppet/ceph/spec/defines/osd__spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 describe 'ceph::osds::osd', :type => :define do
   let :facts do
-    { :hostname => 'test.example', }
+    { :hostname => 'test.example' }
   end
 
   context 'Simple test' do
@@ -15,7 +15,7 @@ describe 'ceph::osds::osd', :type => :define do
       'tries'     => 2,
       'try_sleep' => 1,
       'logoutput' => true,
-      'unless'    => "grep -q /dev/svv /proc/mounts",
+      'unless'    => "ceph-disk list | fgrep -q -e '/dev/svv ceph data, active' -e '/dev/svv ceph data, prepared'",
       )
     }
     it { should contain_exec("ceph-deploy osd activate test.example:/dev/svv").with(
@@ -24,7 +24,7 @@ describe 'ceph::osds::osd', :type => :define do
       'tries'     => 3,
       'logoutput' => true,
       'timeout'   => 0,
-      'unless'    => "ceph osd dump | grep -q \"osd.$(sed -nEe 's|/dev/svv\\ .*ceph-([0-9]+).*$|\\1|p' /proc/mounts)\\ up\\ .*\\ in\\ \"",
+      'onlyif'    => "ceph-disk list | fgrep -q -e '/dev/svv ceph data, prepared'",
     )
     }
   end
@@ -38,7 +38,7 @@ describe 'ceph::osds::osd', :type => :define do
       'tries'     => 2,
       'try_sleep' => 1,
       'logoutput' => true,
-      'unless'    => "grep -q /dev/sdd /proc/mounts",
+      'unless'    => "ceph-disk list | fgrep -q -e '/dev/sdd ceph data, active' -e '/dev/sdd ceph data, prepared'",
       )
     }
     it { should contain_exec("ceph-deploy osd activate test.example:/dev/sdd:/dev/journal").with(
@@ -47,7 +47,7 @@ describe 'ceph::osds::osd', :type => :define do
       'tries'     => 3,
       'logoutput' => true,
       'timeout'   => 0,
-      'unless'    => "ceph osd dump | grep -q \"osd.$(sed -nEe 's|/dev/sdd\\ .*ceph-([0-9]+).*$|\\1|p' /proc/mounts)\\ up\\ .*\\ in\\ \"",
+      'onlyif'    => "ceph-disk list | fgrep -q -e '/dev/sdd ceph data, prepared'",
       )
     }
   end