openstack/fuel-astute
Code Issues Proposed changes

Exception handling in case of unconfigured or not-installed ceph mons

Browse Source 
Change-Id: I5371cc9085f0d501d23f8f91b352ed0e08598880
Closes-Bug: #1453975
This commit is contained in:
Vladimir Sharshov (warpc) 2015-05-12 00:56:11 +03:00
parent d9d488c4c6
commit 6be8322593
2 changed files with 165 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
lib/astute/pre_delete.rb
View File

@ -32,7 +32,7 @@ module Astute
end
if result[:data][:exit_code] != 0
Astute.logger.debug "Ceph has been not found or has not been configured propertly." \
Astute.logger.debug "Ceph has not been found or has not been configured properly" \
" Safely removing nodes..."
return answer
end
@ -85,8 +85,21 @@ module Astute
return answer if ceph_mon_nodes.empty?
#Get the list of mon nodes
shell = MClient.new(ctx, "execute_shell_command", [ceph_mon_nodes[0]["id"]], timeout=120, retries=1)
result = shell.execute(:cmd => "ceph -f json mon dump").first.results
result = {}
shell = nil
ceph_mon_nodes.each do |ceph_mon_node|
shell = MClient.new(ctx, "execute_shell_command", [ceph_mon_node["id"]], timeout=120, retries=1)
result = shell.execute(:cmd => "ceph -f json mon dump").first.results
break if result[:data][:exit_code] == 0
end
if result[:data][:exit_code] != 0
Astute.logger.debug "Ceph mon has not been found or has not been configured properly" \
" Safely removing nodes..."
return answer
end
mon_dump = JSON.parse(result[:data][:stdout])
left_mons = mon_dump['mons'].select { | n | n if ! ceph_mons.include? n['name'] }
left_mon_names = left_mons.collect { |n| n['name'] }
@ -94,14 +107,14 @@ module Astute
#Remove nodes from ceph cluster
Astute.logger.info("Removing ceph mons #{ceph_mons} from cluster")
ceph_mon_nodes.each do | node |
ceph_mon_nodes.each do |node|
shell = MClient.new(ctx, "execute_shell_command", [node["id"]], timeout=120, retries=1)
#remove node from ceph mon list
shell.execute(:cmd => "ceph mon remove #{node["slave_name"]}").first.results
end
#Fix the ceph.conf on the left mon nodes
left_mon_names.each do | node |
left_mon_names.each do |node|
mon_initial_members_cmd = "sed -i \"s/mon_initial_members.*/mon_initial_members\ = #{left_mon_names.join(" ")}/g\" /etc/ceph/ceph.conf"
mon_host_cmd = "sed -i \"s/mon_host.*/mon_host\ = #{left_mon_ips.join(" ")}/g\" /etc/ceph/ceph.conf"
shell = MClient.new(ctx, "execute_shell_command", [node.split('-')[1]], timeout=120, retries=1)

220
spec/unit/pre_delete_spec.rb
View File

@ -15,9 +15,8 @@
require File.join(File.dirname(__FILE__), '../spec_helper')
describe '#check_ceph_osds' do
describe Astute::PreDelete do
include SpecHelpers
let(:ctx) { mock_ctx }
let(:success_result) { {"status"=>"ready"} }
@ -38,89 +37,164 @@ describe '#check_ceph_osds' do
[mock_result]
end
context "no ceph-osd nodes" do
describe '#check_ceph_osds' do
context "no ceph-osd nodes" do
let(:nodes) { [
{"id" => "1", "roles" => ["controller"]},
{"id" => "2", "roles" => ["compute"]}
]
}
it "should do nothing if no nodes have ceph-osd role" do
expect(Astute::PreDelete.check_ceph_osds(ctx, nodes)).to eq(success_result)
end
end
context "nodes with ceph-osd role" do
let(:nodes) { [
{"id" => "1", "roles" => ["primary-controller"]},
{"id" => "2", "roles" => ["compute", "ceph-osd"],
"slave_name" => "node-2"}
]
}
let(:pg_cmd) {
cmd = "ceph pg dump 2>/dev/null | " \
"awk '//{print $14, $16}' | " \
"egrep -o '\\<(1|2)\\>' | " \
"sort -un"
}
let(:osd_cmd) { "ceph -f json osd tree" }
let(:json_resp) { '{"nodes": [{"name": "node-2", "children": [1,2]}]}'}
let(:error_result) do
msg = "Ceph data still exists on: node-2. You must manually " \
"remove the OSDs from the cluster and allow Ceph to " \
"rebalance before deleting these nodes."
{"status" => "error", "error" => msg}
end
it "should raise error if OSDs contain data" do
mclient.expects(:execute).with({:cmd => osd_cmd})
.returns(build_mcresult(stdout=json_resp))
mclient.expects(:execute).with({:cmd => pg_cmd})
.returns(build_mcresult(stdout="1\n2"))
expect(Astute::PreDelete.check_ceph_osds(ctx, nodes)).to eq(error_result)
end
it 'should ignore nodes with unconfigured or failed ceph' do
mclient.expects(:execute).with({:cmd => osd_cmd}).twice
.returns(build_mcresult(stdout="", "2", 42))
.then.returns(build_mcresult(stdout=json_resp, "3", 1))
mclient.expects(:execute).with({:cmd => pg_cmd}).never
all_nodes = nodes + [{
"id" => "3",
"roles" => ["compute", "ceph-osd"],
"slave_name" => "node-3"}
]
expect(Astute::PreDelete.check_ceph_osds(ctx, all_nodes)).to eq(success_result)
end
it 'should find live ceph installation' do
mclient.expects(:execute).with({:cmd => osd_cmd}).twice
.returns(build_mcresult(stdout="", "2", 42))
.then.returns(build_mcresult(stdout=json_resp, "3", 0))
mclient.expects(:execute).with({:cmd => pg_cmd})
.returns(build_mcresult(stdout="1\n2"))
all_nodes = nodes + [{
"id" => "3",
"roles" => ["compute", "ceph-osd"],
"slave_name" => "node-3"}
]
expect(Astute::PreDelete.check_ceph_osds(ctx, all_nodes)).to eq(error_result)
end
it "should succeed with no pgs placed on node" do
mclient.expects(:execute).with({:cmd => osd_cmd})
.returns(build_mcresult(stdout=json_resp))
mclient.expects(:execute).with({:cmd => pg_cmd})
.returns(build_mcresult(stdout="3\n4"))
expect(Astute::PreDelete.check_ceph_osds(ctx, nodes)).to eq(success_result)
end
end
end # check_ceph_osds
describe '#remove_ceph_mons' do
let(:mon_cmd) { "ceph -f json mon dump" }
let(:json_resp) do
'{
"epoch": 5,
"mons": [
{"name":"node-1", "addr":"192.168.0.11:6789\/0"},
{"name":"node-2", "addr":"192.168.0.12:6789\/0"},
{"name":"node-3", "addr":"192.168.0.13:6789\/0"}
]
}'
end
def mon_rm_cmd(slave_name)
"ceph mon remove #{slave_name}"
end
let(:nodes) { [
{"id" => "1", "roles" => ["controller"]},
{"id" => "2", "roles" => ["compute"]}
{"id" => "1", "roles" => ["controller"], "slave_name" => "node-1"},
{"id" => "2", "roles" => ["controller"], "slave_name" => "node-2"}
]
}
it "should do nothing if no nodes have ceph-osd role" do
expect(Astute::PreDelete.check_ceph_osds(ctx, nodes)).to eq(success_result)
end
end
context "no ceph-mon nodes" do
let(:nodes) { [
{"id" => "3", "roles" => ["cinder"]},
{"id" => "4", "roles" => ["compute"]}
]
}
context "nodes with ceph-osd role" do
let(:nodes) { [
{"id" => "1", "roles" => ["primary-controller"]},
{"id" => "2", "roles" => ["compute", "ceph-osd"],
"slave_name" => "node-2"}
]
}
let(:pg_cmd) {
cmd = "ceph pg dump 2>/dev/null | " \
"awk '//{print $14, $16}' | " \
"egrep -o '\\<(1|2)\\>' | " \
"sort -un"
}
let(:osd_cmd) { "ceph -f json osd tree" }
let(:json_resp) { '{"nodes": [{"name": "node-2", "children": [1,2]}]}'}
let(:error_result) do
msg = "Ceph data still exists on: node-2. You must manually " \
"remove the OSDs from the cluster and allow Ceph to " \
"rebalance before deleting these nodes."
{"status" => "error", "error" => msg}
it "should do nothing if no nodes have ceph-osd role" do
expect(Astute::PreDelete.remove_ceph_mons(ctx, nodes)).to eq(success_result)
end
end
it "should raise error if OSDs contain data" do
mclient.expects(:execute).with({:cmd => osd_cmd})
.returns(build_mcresult(stdout=json_resp))
it 'should ignore nodes with unconfigured or failed ceph mons' do
mclient.expects(:execute).with({:cmd => mon_cmd}).twice
.returns(build_mcresult(stdout="", "1", 42))
.then.returns(build_mcresult(stdout=json_resp, "2", 1))
mclient.expects(:execute).with({:cmd => pg_cmd})
.returns(build_mcresult(stdout="1\n2"))
nodes.each do |node|
mclient.expects(:execute).with({:cmd => mon_rm_cmd(node['slave_name'])}).never
end
expect(Astute::PreDelete.check_ceph_osds(ctx, nodes)).to eq(error_result)
expect(Astute::PreDelete.remove_ceph_mons(ctx, nodes)).to eq(success_result)
end
it 'should ignore nodes with unconfigured or failed ceph' do
mclient.expects(:execute).with({:cmd => osd_cmd}).twice
.returns(build_mcresult(stdout="","2", 42))
.then.returns(build_mcresult(stdout=json_resp,"3", 1))
it 'should find and delete live ceph mon installation' do
mclient.expects(:execute).with({:cmd => mon_cmd}).twice
.returns(build_mcresult(stdout="", "1", 42))
.then.returns(build_mcresult(stdout=json_resp, "2", 0))
mclient.expects(:execute).with({:cmd => pg_cmd}).never
all_nodes = nodes + [{
"id" => "3",
"roles" => ["compute", "ceph-osd"],
"slave_name" => "node-3"}
]
expect(Astute::PreDelete.check_ceph_osds(ctx, all_nodes)).to eq(success_result)
nodes.each do |node|
mclient.expects(:execute).with({:cmd => mon_rm_cmd(node['slave_name'])}).once
.returns(build_mcresult(stdout="", node['id'], 0))
end
mclient.expects(:execute).with({:cmd =>
"sed -i \"s/mon_initial_members.*/mon_initial_members = node-3/g\" /etc/ceph/ceph.conf"})
.returns(build_mcresult(stdout="", "3", 0))
mclient.expects(:execute).with({:cmd =>
"sed -i \"s/mon_host.*/mon_host = 192.168.0.13/g\" /etc/ceph/ceph.conf"})
.returns(build_mcresult(stdout="", "3", 0))
expect(Astute::PreDelete.remove_ceph_mons(ctx, nodes)).to eq(success_result)
end
it 'should find live ceph installation' do
mclient.expects(:execute).with({:cmd => osd_cmd}).twice
.returns(build_mcresult(stdout="","2", 42))
.then.returns(build_mcresult(stdout=json_resp,"3", 0))
end # remove_ceph_mons
mclient.expects(:execute).with({:cmd => pg_cmd})
.returns(build_mcresult(stdout="1\n2"))
all_nodes = nodes + [{
"id" => "3",
"roles" => ["compute", "ceph-osd"],
"slave_name" => "node-3"}
]
expect(Astute::PreDelete.check_ceph_osds(ctx, all_nodes)).to eq(error_result)
end
it "should succeed with no pgs placed on node" do
mclient.expects(:execute).with({:cmd => osd_cmd})
.returns(build_mcresult(stdout=json_resp))
mclient.expects(:execute).with({:cmd => pg_cmd})
.returns(build_mcresult(stdout="3\n4"))
expect(Astute::PreDelete.check_ceph_osds(ctx, nodes)).to eq(success_result)
end
end
end # describe
end