openstack/cookbook-openstack-object-storage
Code Issues Proposed changes

Update providers to be rubocop compliant

Browse Source 
- Update rubocop config to include providers/**
- Update providers to comply with rubocop

Change-Id: Ia28e8b091895cadfe8eac2f722fc19a5218d2519
Implements: blueprint rubocop-for-object-storage
This commit is contained in:
Christopher H. Laco
2014-01-28 18:05:37 -05:00
committed by invsblduck
parent 2659e5f3e1
commit a399e066b9
4 changed files with 159 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.rubocop.yml
View File

@@ -3,10 +3,10 @@ AllCops:
- metadata.rb - metadata.rb
- Gemfile - Gemfile
- attributes/** - attributes/**
- providers/**
- recipes/** - recipes/**
Excludes: Excludes:
- libraries/** - libraries/**
- providers/**
- resources/** - resources/**
- spec/** - spec/**

138
providers/disk.rb
View File

@@ -1,3 +1,4 @@
# encoding: UTF-8
# #
# Copyright 2011, Dell # Copyright 2011, Dell
# #
@@ -19,44 +20,43 @@
require 'chef/mixin/shell_out' require 'chef/mixin/shell_out'
include Chef::Mixin::ShellOut include Chef::Mixin::ShellOut
# rubocop:disable CyclomaticComplexity, MethodLength
def load_current_resource def load_current_resource
dev_name = @new_resource.name dev_name = @new_resource.name
@current = Chef::Resource::OpenstackObjectStorageDisk.new(dev_name) @current = Chef::Resource::OpenstackObjectStorageDisk.new(dev_name)
parted_partition_parse dev_name parted_partition_parse dev_name
parts = @current.part() parts = @current.part
if not @current.blocks unless @current.blocks
# parted didn't return anything -- empty disk. # parted didn't return anything -- empty disk.
# get size from sfdisk # get size from sfdisk
sfdisk_get_size(dev_name) sfdisk_get_size(dev_name)
end end
Chef::Log.info("About to print partition table") Chef::Log.info('About to print partition table')
s = <<EOF s = <<EOF
current state for dev #{dev_name} current state for dev #{dev_name}
Size in 1K blocks: #{@current.blocks} Size in 1K blocks: #{@current.blocks}
EOF EOF
Chef::Log.info("Printing partition table") Chef::Log.info('Printing partition table')
num = 0 num = 0
parts.each { | p | parts.each do |p|
s << "partition " << num s << 'partition ' << num
s << " start/end/size (1k): #{p[:start]}/#{p[:end]}/#{p[:size]}" s << " start/end/size (1k): #{p[:start]}/#{p[:end]}/#{p[:size]}"
s << " type: #{p[:type]}" s << " type: #{p[:type]}"
s << "\n" s << "\n"
num+=1 num += 1
} if !parts.nil? end unless parts.nil?
Chef::Log.info(s) Chef::Log.info(s)
end end
=begin # sample output
sample output
# sfdisk /dev/sdb -g # sfdisk /dev/sdb -g
/dev/sdb: 261 cylinders, 255 heads, 63 sectors/track # /dev/sdb: 261 cylinders, 255 heads, 63 sectors/track
=end
def sfdisk_get_size(dev_name) def sfdisk_get_size(dev_name)
out = Mixlib::ShellOut.new("sfdisk #{dev_name} -s").run_command.stdout out = Mixlib::ShellOut.new("sfdisk #{dev_name} -s").run_command.stdout
Chef::Log.info("updating geo using sfdisk: #{out}") Chef::Log.info("updating geo using sfdisk: #{out}")
@@ -67,54 +67,53 @@ end
def parted_partition_parse(dev_name) def parted_partition_parse(dev_name)
Chef::Log.debug("reading partition table for #{dev_name}") Chef::Log.debug("reading partition table for #{dev_name}")
=begin
Run parted to get basic info about the disk # Run parted to get basic info about the disk
sample output: # sample output:
~# parted -m -s /dev/sda unit b print # # parted -m -s /dev/sda unit b print
BYT; # BYT;
/dev/vda:8589934592B:virtblk:512:512:msdos:Virtio Block Device; # /dev/vda:8589934592B:virtblk:512:512:msdos:Virtio Block Device;
1:1048576B:8589934591B:8588886016B:ext3::; # 1:1048576B:8589934591B:8588886016B:ext3::;
=end pipe = IO.popen("parted -m -s #{dev_name} unit b print") # this can return 1, but it's ok (if no partition table present, we'll create it)
pipe= IO.popen("parted -m -s #{dev_name} unit b print") # this can return 1, but it's ok (if no partition table present, we'll create it)
result = pipe.readlines result = pipe.readlines
parted_parse_results result parted_parse_results result
end end
def parted_parse_results(input) def parted_parse_results(input)
Chef::Log.debug("read:" + input.inspect) Chef::Log.debug('read:' + input.inspect)
input = input.to_a input = input.to_a
part_tab = [] part_tab = []
catch (:parse_error) do catch :parse_error do
line = input.shift # Error or BYT; line = input.shift # Error or BYT;
throw :parse_error if line =~ /^Error:/ throw :parse_error if line =~ /^Error:/
line = input.shift line = input.shift
throw :parse_error unless line =~ /\/dev\/([^\/]+):([0-9]+)B:(.*):.*$/ throw :parse_error unless line =~ %r{/dev/([^/]+):([0-9]+)B:(.*):.*$}
dev = Regexp.last_match(1)
blocks = Regexp.last_match(2).to_i / 1024 blocks = Regexp.last_match(2).to_i / 1024
if(@current.blocks and @current.blocks != blocks) if @current.blocks && @current.blocks != blocks
throw "Our disk size changed. Expecting: #{@current.blocks}, got #{blocks}" throw "Our disk size changed. Expecting: #{@current.blocks}, got #{blocks}"
end end
@current.blocks(blocks) @current.blocks(blocks)
input.each { |line| input.each do |input_line|
# 1:1048576B:8589934591B:8588886016B:ext3::; # 1:1048576B:8589934591B:8588886016B:ext3::;
throw :parse_error unless line =~ /([0-9]):([0-9]+)B:([0-9]+)B:([0-9]+)B:(.*):(.*):(.*);$/ throw :parse_error unless input_line =~ /([0-9]):([0-9]+)B:([0-9]+)B:([0-9]+)B:(.*):(.*):(.*);$/
part_num = Regexp.last_match(1).to_i part_num = Regexp.last_match(1).to_i
part_info = { part_info = {
:num => part_num, num: part_num,
:start => Regexp.last_match(2).to_i / 1024, start: Regexp.last_match(2).to_i / 1024,
:end => Regexp.last_match(3).to_i / 1024, end: Regexp.last_match(3).to_i / 1024,
:size => Regexp.last_match(4).to_i / 1024, size: Regexp.last_match(4).to_i / 1024,
:type => Regexp.last_match(5), type: Regexp.last_match(5),
:system => Regexp.last_match(6), system: Regexp.last_match(6),
:flags => Regexp.last_match(7) } flags: Regexp.last_match(7)
}
part_tab << part_info part_tab << part_info
} end
end end
@current.part(part_tab) @current.part(part_tab)
@@ -139,34 +138,34 @@ end
# Plus, then parted doesn't bitch every time you run it. # Plus, then parted doesn't bitch every time you run it.
action :ensure_exists do action :ensure_exists do
Chef::Log.info("Entering :ensure_exists") Chef::Log.info('Entering :ensure_exists')
req = @new_resource.part req = @new_resource.part
cur = @current.part cur = @current.part
dev_name = @new_resource.name dev_name = @new_resource.name
update = false update = false
recreate, delete_existing = false recreate = false
disk_blocks = @current.blocks #1k blocks disk_blocks = @current.blocks # 1k blocks
if (cur.nil?) if cur.nil?
recreate = true; recreate = true
else else
idx = 0 idx = 0
current_block=0 current_block = 0
Chef::Log.info("Checking partition #{idx}") Chef::Log.info("Checking partition #{idx}")
req.each { |params| req.each do |params|
if (cur[idx].nil?) if cur[idx].nil?
recreate = true recreate = true
Chef::Log.info("no current #{idx}") Chef::Log.info("no current #{idx}")
next next
end end
req_size = params[:size] # size in Mb - convert to blocks req_size = params[:size] # size in Mb - convert to blocks
if (req_size == :remaining) if req_size == :remaining
req_size = disk_blocks - current_block req_size = disk_blocks - current_block
else else
req_size = req_size * 1024 req_size = req_size * 1024
@@ -174,68 +173,64 @@ action :ensure_exists do
cur_size = cur[idx][:size] cur_size = cur[idx][:size]
cur_min, cur_max = req_size*0.9, req_size*1.1 cur_min, cur_max = (req_size * 0.9), (req_size * 1.1)
if !(cur_size > cur_min and cur_size < cur_max) recreate = true unless (cur_size > cur_min) && (cur_size < cur_max)
recreate = true
end
current_block += cur[idx][:size] current_block += cur[idx][:size]
Chef::Log.info("partition #{idx} #{(recreate ? 'differs' : 'is same')}: #{cur_size}/#{req_size}") Chef::Log.info("partition #{idx} #{(recreate ? 'differs' : 'is same')}: #{cur_size}/#{req_size}")
idx+=1 idx += 1
} end
end end
if !recreate if !recreate
Chef::Log.info("partition table matches - not recreating") Chef::Log.info('partition table matches - not recreating')
else else
### make sure to ensure that there are no mounted ### make sure to ensure that there are no mounted
### filesystems on the device ### filesystems on the device
re = /^(#{Regexp.escape(dev_name)}[0-9]+)/ re = /^(#{Regexp.escape(dev_name)}[0-9]+)/
mounted = [] mounted = []
shell_out!("mount").stdout.each_line { |line| shell_out!('mount').stdout.each_line do |line|
md = re.match(line) md = re.match(line)
next unless md next unless md
mounted << md[1] mounted << md[1]
} end
mounted.each { |m| mounted.each do |m|
Chef::Log.info("unmounting #{m}") Chef::Log.info("unmounting #{m}")
shell_out!("umount #{m}") shell_out!("umount #{m}")
} end
# Nuke current partition table. # Nuke current partition table.
execute "create new partition table" do execute 'create new partition table' do
command "parted -s -m #{dev_name} mktable gpt" command "parted -s -m #{dev_name} mktable gpt"
end end
# create new partitions # create new partitions
idx = 0 idx = 0
req.each { | params | req.each do |params|
start_block = 0 start_block = 0
if idx == 0 start_block = '1M' if idx == 0
start_block = "1M"
end
if (params[:size] == :remaining) if params[:size] == :remaining
requested_size = "100%" requested_size = '100%'
else else
requested_size = "#{params[:size]}M" requested_size = "#{params[:size]}M"
end end
s = "parted -m -s #{dev_name} " s = "parted -m -s #{dev_name} "
s << "mkpart #{idx} #{start_block} #{requested_size}" # #{params[:type]} s << "mkpart #{idx} #{start_block} #{requested_size}" # #{params[:type]}
Chef::Log.info("creating new partition #{idx+1} with:" + s) Chef::Log.info("creating new partition #{idx + 1} with:" + s)
execute "creating partition #{idx}" do execute "creating partition #{idx}" do
command s command s
end end
idx+=1 idx += 1
} end
update = true update = true
end end
# walk through the partitions and enforce disk format # walk through the partitions and enforce disk format
idx=1 idx = 1
req.each do |params| req.each do |params|
device = "#{dev_name}#{idx}" device = "#{dev_name}#{idx}"
Chef::Log.info("Checking #{device}") Chef::Log.info("Checking #{device}")
@@ -247,13 +242,13 @@ action :ensure_exists do
Chef::Log.info("Testing file system on #{device} for type #{params[:type]}") Chef::Log.info("Testing file system on #{device} for type #{params[:type]}")
case params[:type] case params[:type]
when "xfs" when 'xfs'
if not Mixlib::ShellOut.new("xfs_admin -l #{device}").run_command.status unless Mixlib::ShellOut.new("xfs_admin -l #{device}").run_command.status
Mixlib::ShellOut.new("mkfs.xfs -f -i size=512 #{device}").run_command Mixlib::ShellOut.new("mkfs.xfs -f -i size=512 #{device}").run_command
update = true update = true
end end
when "ext4" when 'ext4'
if not Mixlib::ShellOut.new("tune2fs -l #{device} | grep \"Filesystem volume name:\" | awk \'{print $4}\' | grep -v \"<none>\"").run_command.status unless Mixlib::ShellOut.new("tune2fs -l #{device} | grep \"Filesystem volume name:\" | awk \'{print $4}\' | grep -v \"<none>\"").run_command.status
Mixlib::ShellOut.new("mkfs.ext4 #{device}").run_command Mixlib::ShellOut.new("mkfs.ext4 #{device}").run_command
update = true update = true
end end
@@ -262,4 +257,3 @@ action :ensure_exists do
end end
new_resource.updated_by_last_action(update) new_resource.updated_by_last_action(update)
end end

101
providers/mounts.rb
View File

@@ -1,3 +1,4 @@
# encoding: UTF-8
# #
# Cookbook Name:: openstack-object-storage # Cookbook Name:: openstack-object-storage
# Provider:: mounts # Provider:: mounts
@@ -19,57 +20,58 @@
# Author: Ron Pedde <ron.pedde@rackspace.com> # Author: Ron Pedde <ron.pedde@rackspace.com>
# #
require "chef/util/file_edit" require 'chef/util/file_edit'
action :ensure_exists do action :ensure_exists do
proposed_devices = @new_resource.devices proposed_devices = @new_resource.devices
path = @new_resource.name path = @new_resource.name
dev_info = {} dev_info = {}
Chef::Log.info("IN MOUNTS") Chef::Log.info('IN MOUNTS')
new_resource.updated_by_last_action(false) new_resource.updated_by_last_action(false)
# walk through the devices, gathering information # walk through the devices, gathering information
proposed_devices.each do |device| proposed_devices.each do |device|
next if !::File.exists?("/dev/#{device}") next unless ::File.exists?("/dev/#{device}")
info = {} info = {}
info["device"] = device info['device'] = device
info["ip"] = @new_resource.ip info['ip'] = @new_resource.ip
info["format"] = @new_resource.format info['format'] = @new_resource.format
info["uuid"] = Mixlib::ShellOut.new("blkid /dev/#{device} -s UUID -o value").run_command.stdout.strip info['uuid'] = Mixlib::ShellOut.new("blkid /dev/#{device} -s UUID -o value").run_command.stdout.strip
info["mountpoint"] = info["uuid"].split("-").join("") info['mountpoint'] = info['uuid'].split('-').join('')
info["mounted"] = Mixlib::ShellOut.new("mount | grep '#{path}/#{info["mountpoint"]}\'").run_command.status info['mounted'] = Mixlib::ShellOut.new("mount | grep '#{path}/#{info["mountpoint"]}\'").run_command.status
info["size"] = Mixlib::ShellOut.new("sfdisk -s /dev/#{device}").run_command.stdout.to_i / 1024 info['size'] = Mixlib::ShellOut.new("sfdisk -s /dev/#{device}").run_command.stdout.to_i / 1024
next if (info["uuid"] == '') next if info['uuid'] == ''
dev_info[info["uuid"]] = info dev_info[info['uuid']] = info
end end
Chef::Log.info("Physical Inventory:") Chef::Log.info('Physical Inventory:')
dev_info.each do |_,v| dev_info.each do |_, v|
Chef::Log.info("Device: #{v['device']}, UUID: #{v['uuid']}, Mounted: #{v['mounted']}, Format: #{v['format']}") Chef::Log.info("Device: #{v['device']}, UUID: #{v['uuid']}, Mounted: #{v['mounted']}, Format: #{v['format']}")
end end
# make sure we have a "path" # make sure we have a "path"
Directory(path) do Directory(path) do
group "swift" group 'swift'
owner "swift" owner 'swift'
recursive true recursive true
end.run_action(:create) end.run_action(:create)
# find what should be mounted, and what IS mounted # find what should be mounted, and what IS mounted
mounts=node["filesystem"].inject({}) { |hsh, (k,v)| hsh.merge(v["mount"] => k) } mounts = node['filesystem'].reduce({}) { |hsh, (k, v)| hsh.merge(v['mount'] => k) }
valid_mounts = dev_info.inject([]) {|ary, (_,v)| ary << "#{path}/#{v['mountpoint']}"} valid_mounts = dev_info.reduce([]) { |ary, (_, v)| ary << "#{path}/#{v['mountpoint']}" }
mountpoints = Dir.new(path).reject {|x| x[/^\./] }.collect { |d| "#{path}/#{d}" } mountpoints = Dir.new(path).reject { |x| x[/^\./] }.map { |d| "#{path}/#{d}" }
inverted_mounts = dev_info.inject({}) {|hsh,(k,v)| hsh.merge({v["mountpoint"] => v.merge("uuid" => k)})} inverted_mounts = dev_info.reduce({}) { |hsh, (k, v)| hsh.merge(v['mountpoint'] => v.merge('uuid' => k)) }
fstabs=::File.readlines("/etc/fstab").inject({}) do |hash,line| fstabs = ::File.readlines('/etc/fstab').reduce({}) do |hash, line|
line = line.split("#")[0].split() line = line.split('#')[1].split
Chef::Log.info("#{line[0]} ... #{line[1]}") Chef::Log.info("#{line[0]} ... #{line[1]}")
hash.merge(line[1] => line[0]) hash.merge(line[1] => line[0])
end.reject { |k,v| !k or !v or !k.length or !v.length } end
fstabs.reject! { |k, v| !k || !v || !k.length || !v.length }
Chef::Log.info("Mounts: #{mounts}") Chef::Log.info("Mounts: #{mounts}")
Chef::Log.info("Valid Mounts: #{valid_mounts}") Chef::Log.info("Valid Mounts: #{valid_mounts}")
@@ -77,16 +79,16 @@ action :ensure_exists do
Chef::Log.info("Fstabs: #{fstabs}") Chef::Log.info("Fstabs: #{fstabs}")
# mounts in /srv/node that shouldn't be there # mounts in /srv/node that shouldn't be there
(mounts.keys.select{|x| x and x[/^#{path}/]} - valid_mounts).each do |dev| (mounts.keys.select { |x| x && x[/^#{path}/] } - valid_mounts).each do |dev|
Chef::Log.info("Unmounting #{dev}") Chef::Log.info("Unmounting #{dev}")
Mixlib::ShellOut.new("umount #{dev}").run_command if Mixlib::ShellOut.new("mount | grep '#{dev}'").run_command.status Mixlib::ShellOut.new("umount #{dev}").run_command if Mixlib::ShellOut.new("mount | grep '#{dev}'").run_command.status
new_resource.updated_by_last_action(true) new_resource.updated_by_last_action(true)
end end
# fstab entries that don't need to be there anymore # fstab entries that don't need to be there anymore
(fstabs.keys.select {|k| k.start_with? path} - valid_mounts).each do |dev| (fstabs.keys.select { |k| k.start_with?(path) } - valid_mounts).each do |dev|
fe = Chef::Util::FileEdit.new("/etc/fstab") fe = Chef::Util::FileEdit.new('/etc/fstab')
fe.search_file_delete_line(Regexp.new(dev.gsub("/","\/"))) fe.search_file_delete_line(Regexp.new(dev.gsub('/', '\/')))
fe.write_file fe.write_file
new_resource.updated_by_last_action(true) new_resource.updated_by_last_action(true)
end end
@@ -104,27 +106,27 @@ action :ensure_exists do
# new, unmounted devices # new, unmounted devices
(valid_mounts - mounts.keys).each do |mountpoint| (valid_mounts - mounts.keys).each do |mountpoint|
info = inverted_mounts[mountpoint.gsub("#{path}/","")] info = inverted_mounts[mountpoint.gsub("#{path}/", '')]
Chef::Log.info("mounting #{mountpoint} (#{info['device']})") Chef::Log.info("mounting #{mountpoint} (#{info['device']})")
mount_path = "#{path}/#{info['mountpoint']}" mount_path = "#{path}/#{info['mountpoint']}"
Directory(mount_path) do Directory(mount_path) do
group "swift" group 'swift'
owner "swift" owner 'swift'
recursive true recursive true
end.run_action(:create) end.run_action(:create)
case info['format'] case info['format']
when 'ext4' when 'ext4'
mount_options = "noatime,nodiratime,nobarrier,user_xattr" mount_options = 'noatime,nodiratime,nobarrier,user_xattr'
when 'xfs' when 'xfs'
case node["platform"] case node['platform']
when "ubuntu","debian" when 'ubuntu', 'debian'
mount_options = "noatime,nodiratime,nobarrier,logbufs=8,nobootwait" mount_options = 'noatime,nodiratime,nobarrier,logbufs=8,nobootwait'
else else
mount_options = "noatime,nodiratime,nobarrier,logbufs=8" mount_options = 'noatime,nodiratime,nobarrier,logbufs=8'
end end
end end
@@ -137,7 +139,7 @@ action :ensure_exists do
action :nothing action :nothing
end end
if not fstabs.has_key?(mount_path) unless fstabs.key?(mount_path)
# then its a brand-new drive, freshly formatted # then its a brand-new drive, freshly formatted
Chef::Log.info("Mounting new device #{info['mountpoint']}") Chef::Log.info("Mounting new device #{info['mountpoint']}")
mt.run_action(:enable) mt.run_action(:enable)
@@ -147,22 +149,21 @@ action :ensure_exists do
new_resource.updated_by_last_action(true) new_resource.updated_by_last_action(true)
end end
dev_info.reject { |k,v| v["mounted"] }.keys.each do |uuid| dev_info.reject { |k, v| v['mounted'] }.keys.each do |uuid|
dev_info[uuid]["mounted"] = Mixlib::ShellOut.new("mount | grep '#{path}/#{dev_info[uuid]["mountpoint"]}\'").run_command.status dev_info[uuid]['mounted'] = Mixlib::ShellOut.new("mount | grep '#{path}/#{dev_info[uuid]["mountpoint"]}\'").run_command.status
end end
if @new_resource.publish_attributes and dev_info != {} if @new_resource.publish_attributes && dev_info != {}
dev_info.each do |k,v| dev_info.each do |k, v|
node.set["swift"]["state"]["devs"][k] = { node.set['swift']['state']['devs'][k] = {
:device => v["device"], device: v['device'],
:size => v["size"], size: v['size'],
:uuid => v["uuid"], uuid: v['uuid'],
:mounted => v["mounted"], mounted: v['mounted'],
:format => v["format"], format: v['format'],
:mountpoint => v["mountpoint"], mountpoint: v['mountpoint'],
:ip => v["ip"] ip: v['ip']
} }
end end
end end
end end

80
providers/ring_script.rb
View File

@@ -1,3 +1,4 @@
# encoding: UTF-8
# #
# Cookbook Name:: openstack-object-storage # Cookbook Name:: openstack-object-storage
# Resource:: ring_script # Resource:: ring_script
@@ -19,19 +20,20 @@
# Author: Ron Pedde <ron.pedde@rackspace.com> # Author: Ron Pedde <ron.pedde@rackspace.com>
# #
require "pp" require 'pp'
# rubocop:disable PerlBackrefs, CyclomaticComplexity, MethodLength
def generate_script def generate_script
# need to load and parse the existing rings. # need to load and parse the existing rings.
ports = { "object" => "6000", "container" => "6001", "account" => "6002" } ports = { 'object' => '6000', 'container' => '6001', 'account' => '6002' }
must_rebalance = false must_rebalance = false
ring_path = @new_resource.ring_path ring_path = @new_resource.ring_path
ring_data = { :raw => {}, :parsed => {}, :in_use => {} } ring_data = { raw: {}, parsed: {}, in_use: {} }
disk_data = {} disk_data = {}
dirty_cluster_reasons = [] dirty_cluster_reasons = []
[ "account", "container", "object" ].each do |which| ['account', 'container', 'object'].each do |which|
ring_data[:raw][which] = nil ring_data[:raw][which] = nil
if ::File.exist?("#{ring_path}/#{which}.builder") if ::File.exist?("#{ring_path}/#{which}.builder")
@@ -40,7 +42,7 @@ def generate_script
# Chef::Log.debug("#{ which.capitalize } Ring data: #{ring_data[:raw][which]}") # Chef::Log.debug("#{ which.capitalize } Ring data: #{ring_data[:raw][which]}")
ring_data[:parsed][which] = parse_ring_output(ring_data[:raw][which]) ring_data[:parsed][which] = parse_ring_output(ring_data[:raw][which])
node.set["swift"]["state"]["ring"][which] = ring_data[:parsed][which] node.set['swift']['state']['ring'][which] = ring_data[:parsed][which]
end end
else else
Chef::Log.info("#{which.capitalize} ring builder files do not exist!") Chef::Log.info("#{which.capitalize} ring builder files do not exist!")
@@ -58,41 +60,41 @@ def generate_script
end end
end end
Chef::Log.debug("#{which.capitalize} Ring - In use: #{PP.pp(ring_data[:in_use][which],dump='')}") Chef::Log.debug("#{which.capitalize} Ring - In use: #{PP.pp(ring_data[:in_use][which], '')}")
# figure out what's present in the cluster # figure out what's present in the cluster
disk_data[which] = {} disk_data[which] = {}
role = node["swift"]["#{which}_server_chef_role"] role = node['swift']["#{which}_server_chef_role"]
disk_state,_,_ = Chef::Search::Query.new.search(:node,"chef_environment:#{node.chef_environment} AND roles:#{role}") disk_state, _, _ = Chef::Search::Query.new.search(:node, "chef_environment:#{node.chef_environment} AND roles:#{role}")
# for a running track of available disks # for a running track of available disks
disk_data[:available] ||= {} disk_data[:available] ||= {}
disk_data[:available][which] ||= {} disk_data[:available][which] ||= {}
disk_state.each do |swiftnode| disk_state.each do |swiftnode|
if swiftnode[:swift][:state] and swiftnode[:swift][:state][:devs] if swiftnode[:swift][:state] && swiftnode[:swift][:state][:devs]
swiftnode[:swift][:state][:devs].each do |k,v| swiftnode[:swift][:state][:devs].each do |k, v|
disk_data[which][v[:ip]] = disk_data[which][v[:ip]] || {} disk_data[which][v[:ip]] = disk_data[which][v[:ip]] || {}
disk_data[which][v[:ip]][k] = {} disk_data[which][v[:ip]][k] = {}
v.keys.each { |x| disk_data[which][v[:ip]][k].store(x,v[x]) } v.keys.each { |x| disk_data[which][v[:ip]][k].store(x, v[x]) }
if swiftnode[:swift].has_key?("#{which}-zone") if swiftnode[:swift].key?("#{which}-zone")
disk_data[which][v[:ip]][k]["zone"]=swiftnode[:swift]["#{which}-zone"] disk_data[which][v[:ip]][k]['zone'] = swiftnode[:swift]["#{which}-zone"]
elsif swiftnode[:swift].has_key?("zone") elsif swiftnode[:swift].key?('zone')
disk_data[which][v[:ip]][k]["zone"]=swiftnode[:swift]["zone"] disk_data[which][v[:ip]][k]['zone'] = swiftnode[:swift]['zone']
else else
raise "Node #{swiftnode[:hostname]} has no zone assigned" fail "Node #{swiftnode[:hostname]} has no zone assigned"
end end
disk_data[:available][which][v[:mountpoint]] = v[:ip] disk_data[:available][which][v[:mountpoint]] = v[:ip]
if not v[:mounted] unless v[:mounted]
dirty_cluster_reasons << "Disk #{v[:name]} (#{v[:uuid]}) is not mounted on host #{v[:ip]} (#{swiftnode[:hostname]})" dirty_cluster_reasons << "Disk #{v[:name]} (#{v[:uuid]}) is not mounted on host #{v[:ip]} (#{swiftnode[:hostname]})"
end end
end end
end end
end end
Chef::Log.debug("#{which.capitalize} Ring - Avail: #{PP.pp(disk_data[:available][which],dump='')}") Chef::Log.debug("#{which.capitalize} Ring - Avail: #{PP.pp(disk_data[:available][which], '')}")
end end
# Have the raw data, now bump it together and drop the script # Have the raw data, now bump it together and drop the script
@@ -100,7 +102,7 @@ def generate_script
s = "#!/bin/bash\n\n# This script is automatically generated.\n" s = "#!/bin/bash\n\n# This script is automatically generated.\n"
s << "# Running it will likely blow up your system if you don't review it carefully.\n" s << "# Running it will likely blow up your system if you don't review it carefully.\n"
s << "# You have been warned.\n\n" s << "# You have been warned.\n\n"
if not node["swift"]["auto_rebuild_rings"] unless node['swift']['auto_rebuild_rings']
s << "if [ \"$1\" != \"--force\" ]; then\n" s << "if [ \"$1\" != \"--force\" ]; then\n"
s << " echo \"Auto rebuild rings is disabled, so you must use --force to generate rings\"\n" s << " echo \"Auto rebuild rings is disabled, so you must use --force to generate rings\"\n"
s << " exit 0\n" s << " exit 0\n"
@@ -113,24 +115,24 @@ def generate_script
missing_disks = {} missing_disks = {}
new_servers = [] new_servers = []
[ "account", "container", "object" ].each do |which| ['account', 'container', 'object'].each do |which|
# remove available disks that are already in the ring # remove available disks that are already in the ring
new_disks[which] = disk_data[:available][which].reject{ |k,v| ring_data[:in_use][which].has_key?(k) } new_disks[which] = disk_data[:available][which].reject { |k, v| ring_data[:in_use][which].key?(k) }
# find all in-ring disks that are not in the cluster # find all in-ring disks that are not in the cluster
missing_disks[which] = ring_data[:in_use][which].reject{ |k,v| disk_data[:available][which].has_key?(k) } missing_disks[which] = ring_data[:in_use][which].reject { |k, v| disk_data[:available][which].key?(k) }
Chef::Log.debug("#{which.capitalize} Ring - Missing: #{PP.pp(missing_disks[which],dump='')}") Chef::Log.debug("#{which.capitalize} Ring - Missing: #{PP.pp(missing_disks[which], '')}")
Chef::Log.debug("#{which.capitalize} Ring - New: #{PP.pp(new_disks[which],dump='')}") Chef::Log.debug("#{which.capitalize} Ring - New: #{PP.pp(new_disks[which], '')}")
s << "\n# -- #{which.capitalize} Servers --\n\n" s << "\n# -- #{which.capitalize} Servers --\n\n"
disk_data[which].keys.sort.each do |ip| disk_data[which].keys.sort.each do |ip|
s << "# #{ip}\n" s << "# #{ip}\n"
disk_data[which][ip].keys.sort.each do |k| disk_data[which][ip].keys.sort.each do |k|
v = disk_data[which][ip][k] v = disk_data[which][ip][k]
s << "# " + v.keys.sort.select{|x| ["ip", "device", "uuid"].include?(x)}.collect{|x| v[x] }.join(", ") s << '# ' + v.keys.sort.select { |x| ['ip', 'device', 'uuid'].include?(x) }.map { |x| v[x] }.join(', ')
if new_disks[which].has_key?(v["mountpoint"]) if new_disks[which].key?(v['mountpoint'])
s << " (NEW!)" s << ' (NEW!)'
new_servers << ip unless new_servers.include?(ip) new_servers << ip unless new_servers.include?(ip)
end end
s << "\n" s << "\n"
@@ -147,7 +149,7 @@ def generate_script
disk_data[which].keys.sort.each do |ip| disk_data[which].keys.sort.each do |ip|
disk_data[which][ip].keys.sort.each do |uuid| disk_data[which][ip].keys.sort.each do |uuid|
v = disk_data[which][ip][uuid] v = disk_data[which][ip][uuid]
if new_disks[which].has_key?(v['mountpoint']) if new_disks[which].key?(v['mountpoint'])
s << "swift-ring-builder #{ring_path}/#{which}.builder add z#{v['zone']}-#{v['ip']}:#{ports[which]}/#{v['mountpoint']} #{v['size']}\n" s << "swift-ring-builder #{ring_path}/#{which}.builder add z#{v['zone']}-#{v['ip']}:#{ports[which]}/#{v['mountpoint']} #{v['size']}\n"
must_rebalance = true must_rebalance = true
end end
@@ -156,9 +158,9 @@ def generate_script
# remove the disks -- sort to ensure consistent order # remove the disks -- sort to ensure consistent order
missing_disks[which].keys.sort.each do |mountpoint| missing_disks[which].keys.sort.each do |mountpoint|
diskinfo=ring_data[:parsed][which][:hosts].select{|k,v| v.has_key?(mountpoint)}.collect{|_,v| v[mountpoint]}[0] diskinfo = ring_data[:parsed][which][:hosts].select { |k, v| v.key?(mountpoint) }.map { |_, v| v[mountpoint] }[0]
Chef::Log.debug("Missing diskinfo: #{PP.pp(diskinfo,dump='')}") Chef::Log.debug("Missing diskinfo: #{PP.pp(diskinfo, '')}")
description = Hash[diskinfo.select{|k,v| [:zone, :ip, :device].include?(k)}].collect{|k,v| "#{k}: #{v}" }.join(", ") description = Hash[diskinfo.select { |k, v| [:zone, :ip, :device].include?(k) }].map { |k, v| "#{k}: #{v}" }.join(', ')
s << "# #{description}\n" s << "# #{description}\n"
s << "swift-ring-builder #{ring_path}/#{which}.builder remove d#{missing_disks[which][mountpoint]}\n" s << "swift-ring-builder #{ring_path}/#{which}.builder remove d#{missing_disks[which][mountpoint]}\n"
must_rebalance = true must_rebalance = true
@@ -166,26 +168,26 @@ def generate_script
s << "\n" s << "\n"
if(must_rebalance) if must_rebalance
s << "swift-ring-builder #{ring_path}/#{which}.builder rebalance\n\n\n" s << "swift-ring-builder #{ring_path}/#{which}.builder rebalance\n\n\n"
else else
s << "# #{which.capitalize} ring has no outstanding changes!\n\n" s << "# #{which.capitalize} ring has no outstanding changes!\n\n"
end end
# we'll only rebalance if we meet the minimums for new adds # we'll only rebalance if we meet the minimums for new adds
if node["swift"].has_key?("wait_for") if node['swift'].key?('wait_for')
if node["swift"]["wait_for"] > new_servers.count if node['swift']['wait_for'] > new_servers.count
Chef::Log.debug("New servers, but not enough to force a rebalance") Chef::Log.debug('New servers, but not enough to force a rebalance')
must_rebalance = false must_rebalance = false
end end
end end
end end
[ s, must_rebalance ] [s, must_rebalance]
end end
# Parse the raw output of swift-ring-builder # Parse the raw output of swift-ring-builder
def parse_ring_output(ring_data) def parse_ring_output(ring_data)
output = { :state => {} } output = { state: {} }
ring_data.each do |line| ring_data.each do |line|
if line =~ /build version ([0-9]+)/ if line =~ /build version ([0-9]+)/
@@ -256,7 +258,7 @@ def parse_ring_output(ring_data)
elsif line =~ /^The minimum number of hours before a partition can be reassigned is (\d+)$/ elsif line =~ /^The minimum number of hours before a partition can be reassigned is (\d+)$/
output[:state][:min_part_hours] = $1 output[:state][:min_part_hours] = $1
else else
raise "Cannot parse ring builder output for #{line}" fail "Cannot parse ring builder output for #{line}"
end end
end end
@@ -266,7 +268,7 @@ end
action :ensure_exists do action :ensure_exists do
Chef::Log.debug("Ensuring #{new_resource.name}") Chef::Log.debug("Ensuring #{new_resource.name}")
new_resource.updated_by_last_action(false) new_resource.updated_by_last_action(false)
s,must_update = generate_script s, must_update = generate_script
script_file = File new_resource.name do script_file = File new_resource.name do
owner new_resource.owner owner new_resource.owner