From 8f05952ddf5d6152ae40097f9e808ae6a9c04cb8 Mon Sep 17 00:00:00 2001 From: Oleg Gelbukh Date: Mon, 20 Jul 2015 13:29:09 +0000 Subject: [PATCH] Support Ceph OSD devices with existing data set Partition preservation feature allows to preserve data on Ceph OSD device, but later on Puppet will run 'ceph-deploy prepare' on every Ceph device in a system. This call destroys data set on those devices. To preserve data on Ceph OSD devices through deployment process, we need to check if the device has Ceph data and if so, skip execution of 'ceph-deploy prepare' on that device. Only prepared devices must be activated to avoid deployment failure. Following steps are made in the strict sequence to ensure protection of the existing data: * execute 'udevadm trigger' to force create links for devices in /dev/disk * execute 'ceph-disk activate-all' to force activation of all devices identified as ceph osds * in ceph::osds::osd class for every ceph osd device check if it is prepared or active, if not, run 'ceph-deploy prepare' on it * for every osd device, check if it is prepared (not active) and activate it Change-Id: I667fa6aab9d6f46c73bfb8ca0e267afede6049fb Implements: blueprint upgrade-redeploy-node --- deployment/puppet/ceph/manifests/osds.pp | 23 +++++++++++++++++-- deployment/puppet/ceph/manifests/osds/osd.pp | 10 +++++--- .../puppet/ceph/spec/classes/osds__spec.rb | 10 ++++---- .../puppet/ceph/spec/defines/osd__spec.rb | 10 ++++---- 4 files changed, 39 insertions(+), 14 deletions(-) diff --git a/deployment/puppet/ceph/manifests/osds.pp b/deployment/puppet/ceph/manifests/osds.pp index 32814156c8..19ce2b58ce 100644 --- a/deployment/puppet/ceph/manifests/osds.pp +++ b/deployment/puppet/ceph/manifests/osds.pp @@ -1,8 +1,28 @@ -# prepare and bring online the devices listed in $::ceph::osd_devices +# == Class: ceph::osd +# +# Prepare and bring online the OSD devices +# +# ==== Parameters +# +# [*devices*] +# (optional) Array. This is the list of OSD devices identified by the facter. +# class ceph::osds ( $devices = $::ceph::osd_devices, ){ + exec { 'udevadm trigger': + command => 'udevadm trigger', + returns => 0, + logoutput => true, + } -> + + exec {'ceph-disk activate-all': + command => 'ceph-disk activate-all', + returns => 0, + logoutput => true, + } -> + firewall { '011 ceph-osd allow': chain => 'INPUT', dport => '6800-7100', @@ -11,5 +31,4 @@ class ceph::osds ( } -> ceph::osds::osd{ $devices: } - } diff --git a/deployment/puppet/ceph/manifests/osds/osd.pp b/deployment/puppet/ceph/manifests/osds/osd.pp index b8fd18ea47..153b84d4c0 100644 --- a/deployment/puppet/ceph/manifests/osds/osd.pp +++ b/deployment/puppet/ceph/manifests/osds/osd.pp @@ -1,3 +1,7 @@ +# == Define: ceph::osds::osd +# +# Prepare and activate OSD nodes on the node +# define ceph::osds::osd () { # ${name} format is DISK[:JOURNAL] @@ -18,8 +22,8 @@ define ceph::osds::osd () { tries => 2, # This is necessary because of race for mon creating keys try_sleep => 1, logoutput => true, - unless => "grep -q ${data_device_name} /proc/mounts", - } -> + unless => "ceph-disk list | fgrep -q -e '${data_device_name} ceph data, active' -e '${data_device_name} ceph data, prepared'", + } -> Exec["ceph-deploy osd activate ${deploy_device_name}"] exec { "ceph-deploy osd activate ${deploy_device_name}": command => "ceph-deploy osd activate ${deploy_device_name}", @@ -27,7 +31,7 @@ define ceph::osds::osd () { tries => 3, logoutput => true, timeout => 0, - unless => "ceph osd dump | grep -q \"osd.$(sed -nEe 's|${data_device_name}\\ .*ceph-([0-9]+).*$|\\1|p' /proc/mounts)\\ up\\ .*\\ in\\ \"", + onlyif => "ceph-disk list | fgrep -q -e '${data_device_name} ceph data, prepared'", } } diff --git a/deployment/puppet/ceph/spec/classes/osds__spec.rb b/deployment/puppet/ceph/spec/classes/osds__spec.rb index b4b7c1b008..d04d47f071 100644 --- a/deployment/puppet/ceph/spec/classes/osds__spec.rb +++ b/deployment/puppet/ceph/spec/classes/osds__spec.rb @@ -5,9 +5,11 @@ describe 'ceph::osds', :type => :class do context 'Simple ceph::osds class test' do let (:params) {{ :devices => ['/dev/vdc', '/dev/vdd' ] }} - it { should contain_firewall('011 ceph-osd allow') } - it { should contain_ceph__osds__osd('/dev/vdc') } - it { should contain_ceph__osds__osd('/dev/vdd') } + it { should contain_exec('udevadm trigger') } + it { should contain_exec('ceph-disk activate-all').that_requires('Exec[udevadm trigger]') } + it { should contain_firewall('011 ceph-osd allow').that_requires('Exec[ceph-disk activate-all]') } + it { should contain_ceph__osds__osd('/dev/vdc').that_requires('Firewall[011 ceph-osd allow]') } + it { should contain_ceph__osds__osd('/dev/vdd').that_requires('Firewall[011 ceph-osd allow]') } end @@ -19,7 +21,7 @@ describe 'ceph::osds', :type => :class do end context 'Class ceph::osds with devices and journals' do - let (:params) {{ :devices => ['/dev/sdc1:/dev/sdc2', '/dev/sdd1:/dev/sdd2' ] }} + let (:params) {{ :devices => ['/dev/sdc1:/dev/sdc2', '/dev/sdd1:/dev/sdd2'] }} it { should contain_firewall('011 ceph-osd allow') } it { should contain_ceph__osds__osd('/dev/sdc1:/dev/sdc2') } diff --git a/deployment/puppet/ceph/spec/defines/osd__spec.rb b/deployment/puppet/ceph/spec/defines/osd__spec.rb index b510da3b5e..9c54569b50 100644 --- a/deployment/puppet/ceph/spec/defines/osd__spec.rb +++ b/deployment/puppet/ceph/spec/defines/osd__spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' describe 'ceph::osds::osd', :type => :define do let :facts do - { :hostname => 'test.example', } + { :hostname => 'test.example' } end context 'Simple test' do @@ -15,7 +15,7 @@ describe 'ceph::osds::osd', :type => :define do 'tries' => 2, 'try_sleep' => 1, 'logoutput' => true, - 'unless' => "grep -q /dev/svv /proc/mounts", + 'unless' => "ceph-disk list | fgrep -q -e '/dev/svv ceph data, active' -e '/dev/svv ceph data, prepared'", ) } it { should contain_exec("ceph-deploy osd activate test.example:/dev/svv").with( @@ -24,7 +24,7 @@ describe 'ceph::osds::osd', :type => :define do 'tries' => 3, 'logoutput' => true, 'timeout' => 0, - 'unless' => "ceph osd dump | grep -q \"osd.$(sed -nEe 's|/dev/svv\\ .*ceph-([0-9]+).*$|\\1|p' /proc/mounts)\\ up\\ .*\\ in\\ \"", + 'onlyif' => "ceph-disk list | fgrep -q -e '/dev/svv ceph data, prepared'", ) } end @@ -38,7 +38,7 @@ describe 'ceph::osds::osd', :type => :define do 'tries' => 2, 'try_sleep' => 1, 'logoutput' => true, - 'unless' => "grep -q /dev/sdd /proc/mounts", + 'unless' => "ceph-disk list | fgrep -q -e '/dev/sdd ceph data, active' -e '/dev/sdd ceph data, prepared'", ) } it { should contain_exec("ceph-deploy osd activate test.example:/dev/sdd:/dev/journal").with( @@ -47,7 +47,7 @@ describe 'ceph::osds::osd', :type => :define do 'tries' => 3, 'logoutput' => true, 'timeout' => 0, - 'unless' => "ceph osd dump | grep -q \"osd.$(sed -nEe 's|/dev/sdd\\ .*ceph-([0-9]+).*$|\\1|p' /proc/mounts)\\ up\\ .*\\ in\\ \"", + 'onlyif' => "ceph-disk list | fgrep -q -e '/dev/sdd ceph data, prepared'", ) } end