From 8efd03917ab17986c268f2c0d75c28df2ff42262 Mon Sep 17 00:00:00 2001
From: Ricardo Rocha <rocha.porto@gmail.com>
Date: Tue, 2 Sep 2014 13:16:31 +1200
Subject: [PATCH] separate ceph-disk prepare / activate on ceph::osd

separates the ceph-disk prepare and activate parts, so that we
properly handle osds that are not active but have already been prepared
(as in an unmounted partition after a reboot).

uses ceph-disk output to detect status of partitions.

Closes-Bug: #1361446

Change-Id: I201817623c3162c7247893af44cad58b20989042
---
 manifests/osd.pp              | 28 ++++++++++---
 spec/defines/ceph_osd_spec.rb |  8 +++-
 spec/system/ceph_osd_spec.rb  | 75 +++++++++++++++++++++++++++++++++++
 3 files changed, 104 insertions(+), 7 deletions(-)

diff --git a/manifests/osd.pp b/manifests/osd.pp
index 2a759937..55f09515 100644
--- a/manifests/osd.pp
+++ b/manifests/osd.pp
@@ -50,19 +50,37 @@ define ceph::osd (
 
     if $ensure == present {
 
-      $ceph_mkfs = "ceph-osd-mkfs-${name}"
+      $ceph_prepare = "ceph-osd-prepare-${name}"
+      $ceph_activate = "ceph-osd-activate-${name}"
 
-      Ceph_Config<||> -> Exec[$ceph_mkfs]
-      Ceph::Mon<||> -> Exec[$ceph_mkfs]
-      Ceph::Key<||> -> Exec[$ceph_mkfs]
+      Ceph_Config<||> -> Exec[$ceph_prepare]
+      Ceph::Mon<||> -> Exec[$ceph_prepare]
+      Ceph::Key<||> -> Exec[$ceph_prepare]
       # ceph-disk: prepare should be idempotent http://tracker.ceph.com/issues/7475
-      exec { $ceph_mkfs:
+      exec { $ceph_prepare:
         command   => "/bin/true  # comment to satisfy puppet syntax requirements
 set -ex
 if ! test -b ${data} ; then
   mkdir -p ${data}
 fi
 ceph-disk prepare ${cluster_option} ${data} ${journal}
+",
+        unless    => "/bin/true  # comment to satisfy puppet syntax requirements
+set -ex
+ceph-disk list | grep ' *${data}.*ceph data, prepared' ||
+ceph-disk list | grep ' *${data}.*ceph data, active' ||
+ls -l /var/lib/ceph/osd/${cluster_name}-* | grep ' ${data}'
+",
+        logoutput => true,
+      }
+
+      Exec[$ceph_prepare] -> Exec[$ceph_activate]
+      exec { $ceph_activate:
+        command   => "/bin/true  # comment to satisfy puppet syntax requirements
+set -ex
+if ! test -b ${data} ; then
+  mkdir -p ${data}
+fi
 # activate happens via udev when using the entire device
 if ! test -b ${data} || ! test -b ${data}1 ; then
   ceph-disk activate ${data} || true
diff --git a/spec/defines/ceph_osd_spec.rb b/spec/defines/ceph_osd_spec.rb
index bdf6dcac..57a8c787 100644
--- a/spec/defines/ceph_osd_spec.rb
+++ b/spec/defines/ceph_osd_spec.rb
@@ -34,7 +34,9 @@ describe 'ceph::osd' do
         '/tmp'
       end
 
-      it { should contain_exec('ceph-osd-mkfs-/tmp') }
+      it { should contain_exec('ceph-osd-prepare-/tmp') }
+
+      it { should contain_exec('ceph-osd-activate-/tmp') }
 
     end
   end
@@ -53,7 +55,9 @@ describe 'ceph::osd' do
         '/tmp'
       end
 
-      it { should contain_exec('ceph-osd-mkfs-/tmp') }
+      it { should contain_exec('ceph-osd-prepare-/tmp') }
+
+      it { should contain_exec('ceph-osd-activate-/tmp') }
 
     end
   end
diff --git a/spec/system/ceph_osd_spec.rb b/spec/system/ceph_osd_spec.rb
index 77a01f8e..4950d665 100644
--- a/spec/system/ceph_osd_spec.rb
+++ b/spec/system/ceph_osd_spec.rb
@@ -321,6 +321,81 @@ describe 'ceph::osd' do
           shell 'ceph-disk zap /dev/sdb'
         end
 
+        it 'should install one OSD no cephx on partition and activate after umount' do
+          shell 'sgdisk --delete=1 /dev/sdb || true; sgdisk --largest-new=1 --change-name="1:ceph data" --partition-guid=1:7aebb13f-d4a5-4b94-8622-355d2b5401f1 --typecode=1:4fbd7e29-9d25-41b8-afd0-062c0ceff05d -- /dev/sdb' do |r|
+            r.exit_code.should be_zero
+          end
+
+          pp = <<-EOS
+            class { 'ceph':
+              fsid => '#{fsid}',
+              mon_host => #{mon_host},
+              authentication_type => 'none',
+            }
+            ceph_config {
+             'global/osd_journal_size': value => '100';
+            }
+            ceph::mon { 'a':
+              public_addr => #{mon_host},
+              authentication_type => 'none',
+            }
+            ceph::osd { '/dev/sdb1': }
+          EOS
+
+          puppet_apply(pp) do |r|
+            r.exit_code.should_not == 1
+            r.refresh
+            r.exit_code.should_not == 1
+          end
+
+          shell 'ceph osd tree' do |r|
+            r.stdout.should =~ /osd.0/
+            r.stderr.should be_empty
+            r.exit_code.should be_zero
+          end
+
+          # stop and umount (but leave it prepared)
+          shell 'stop ceph-osd id=0 || /etc/init.d/ceph stop osd.0; umount /dev/sdb1' do |r|
+            r.exit_code.should be_zero
+          end
+
+          # rerun puppet (should activate but not prepare)
+          puppet_apply(pp) do |r|
+            r.exit_code.should_not == 1
+            r.refresh
+            r.exit_code.should_not == 1
+          end
+
+          # check osd up and same osd.id
+          shell 'ceph osd tree' do |r|
+            r.stdout.should =~ /osd.0\s*up/
+            r.stderr.should be_empty
+            r.exit_code.should be_zero
+          end
+
+        end
+
+        it 'should uninstall one osd' do
+          shell 'ceph osd tree | grep DNE' do |r|
+            r.exit_code.should_not be_zero
+          end
+
+          pp = <<-EOS
+            ceph::osd { '/dev/sdb1':
+              ensure => absent,
+            }
+          EOS
+
+          puppet_apply(pp) do |r|
+            r.exit_code.should_not == 1
+          end
+
+          shell 'ceph osd tree | grep DNE' do |r|
+            r.exit_code.should be_zero
+          end
+          shell 'ceph-disk zap /dev/sdb'
+        end
+
         it 'should uninstall one monitor and all packages' do
           puppet_apply(purge) do |r|
             r.exit_code.should_not == 1