diff --git a/.rubocop.yml b/.rubocop.yml index 2ad2045..17b48ac 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -3,10 +3,10 @@ AllCops: - metadata.rb - Gemfile - attributes/** + - providers/** - recipes/** Excludes: - libraries/** - - providers/** - resources/** - spec/** diff --git a/providers/disk.rb b/providers/disk.rb index 58cb62b..b14756c 100644 --- a/providers/disk.rb +++ b/providers/disk.rb @@ -1,3 +1,4 @@ +# encoding: UTF-8 # # Copyright 2011, Dell # @@ -19,44 +20,43 @@ require 'chef/mixin/shell_out' include Chef::Mixin::ShellOut +# rubocop:disable CyclomaticComplexity, MethodLength def load_current_resource dev_name = @new_resource.name @current = Chef::Resource::OpenstackObjectStorageDisk.new(dev_name) parted_partition_parse dev_name - parts = @current.part() + parts = @current.part - if not @current.blocks + unless @current.blocks # parted didn't return anything -- empty disk. # get size from sfdisk sfdisk_get_size(dev_name) end - Chef::Log.info("About to print partition table") + Chef::Log.info('About to print partition table') s = < part_num, - :start => Regexp.last_match(2).to_i / 1024, - :end => Regexp.last_match(3).to_i / 1024, - :size => Regexp.last_match(4).to_i / 1024, - :type => Regexp.last_match(5), - :system => Regexp.last_match(6), - :flags => Regexp.last_match(7) } + num: part_num, + start: Regexp.last_match(2).to_i / 1024, + end: Regexp.last_match(3).to_i / 1024, + size: Regexp.last_match(4).to_i / 1024, + type: Regexp.last_match(5), + system: Regexp.last_match(6), + flags: Regexp.last_match(7) + } part_tab << part_info - } + end end @current.part(part_tab) @@ -139,34 +138,34 @@ end # Plus, then parted doesn't bitch every time you run it. action :ensure_exists do - Chef::Log.info("Entering :ensure_exists") + Chef::Log.info('Entering :ensure_exists') req = @new_resource.part cur = @current.part dev_name = @new_resource.name update = false - recreate, delete_existing = false + recreate = false - disk_blocks = @current.blocks #1k blocks + disk_blocks = @current.blocks # 1k blocks - if (cur.nil?) - recreate = true; + if cur.nil? + recreate = true else idx = 0 - current_block=0 + current_block = 0 Chef::Log.info("Checking partition #{idx}") - req.each { |params| - if (cur[idx].nil?) + req.each do |params| + if cur[idx].nil? recreate = true Chef::Log.info("no current #{idx}") next end req_size = params[:size] # size in Mb - convert to blocks - if (req_size == :remaining) + if req_size == :remaining req_size = disk_blocks - current_block else req_size = req_size * 1024 @@ -174,68 +173,64 @@ action :ensure_exists do cur_size = cur[idx][:size] - cur_min, cur_max = req_size*0.9, req_size*1.1 - if !(cur_size > cur_min and cur_size < cur_max) - recreate = true - end + cur_min, cur_max = (req_size * 0.9), (req_size * 1.1) + recreate = true unless (cur_size > cur_min) && (cur_size < cur_max) current_block += cur[idx][:size] Chef::Log.info("partition #{idx} #{(recreate ? 'differs' : 'is same')}: #{cur_size}/#{req_size}") - idx+=1 - } + idx += 1 + end end if !recreate - Chef::Log.info("partition table matches - not recreating") + Chef::Log.info('partition table matches - not recreating') else ### make sure to ensure that there are no mounted ### filesystems on the device re = /^(#{Regexp.escape(dev_name)}[0-9]+)/ mounted = [] - shell_out!("mount").stdout.each_line { |line| + shell_out!('mount').stdout.each_line do |line| md = re.match(line) next unless md mounted << md[1] - } - mounted.each { |m| + end + mounted.each do |m| Chef::Log.info("unmounting #{m}") shell_out!("umount #{m}") - } + end # Nuke current partition table. - execute "create new partition table" do + execute 'create new partition table' do command "parted -s -m #{dev_name} mktable gpt" end # create new partitions idx = 0 - req.each { | params | + req.each do |params| start_block = 0 - if idx == 0 - start_block = "1M" - end + start_block = '1M' if idx == 0 - if (params[:size] == :remaining) - requested_size = "100%" + if params[:size] == :remaining + requested_size = '100%' else requested_size = "#{params[:size]}M" end s = "parted -m -s #{dev_name} " s << "mkpart #{idx} #{start_block} #{requested_size}" # #{params[:type]} - Chef::Log.info("creating new partition #{idx+1} with:" + s) + Chef::Log.info("creating new partition #{idx + 1} with:" + s) execute "creating partition #{idx}" do command s end - idx+=1 + idx += 1 - } + end update = true end # walk through the partitions and enforce disk format - idx=1 + idx = 1 req.each do |params| device = "#{dev_name}#{idx}" Chef::Log.info("Checking #{device}") @@ -247,13 +242,13 @@ action :ensure_exists do Chef::Log.info("Testing file system on #{device} for type #{params[:type]}") case params[:type] - when "xfs" - if not Mixlib::ShellOut.new("xfs_admin -l #{device}").run_command.status + when 'xfs' + unless Mixlib::ShellOut.new("xfs_admin -l #{device}").run_command.status Mixlib::ShellOut.new("mkfs.xfs -f -i size=512 #{device}").run_command update = true end - when "ext4" - if not Mixlib::ShellOut.new("tune2fs -l #{device} | grep \"Filesystem volume name:\" | awk \'{print $4}\' | grep -v \"\"").run_command.status + when 'ext4' + unless Mixlib::ShellOut.new("tune2fs -l #{device} | grep \"Filesystem volume name:\" | awk \'{print $4}\' | grep -v \"\"").run_command.status Mixlib::ShellOut.new("mkfs.ext4 #{device}").run_command update = true end @@ -262,4 +257,3 @@ action :ensure_exists do end new_resource.updated_by_last_action(update) end - diff --git a/providers/mounts.rb b/providers/mounts.rb index 6481c5a..98fae02 100644 --- a/providers/mounts.rb +++ b/providers/mounts.rb @@ -1,3 +1,4 @@ +# encoding: UTF-8 # # Cookbook Name:: openstack-object-storage # Provider:: mounts @@ -19,57 +20,58 @@ # Author: Ron Pedde # -require "chef/util/file_edit" +require 'chef/util/file_edit' action :ensure_exists do proposed_devices = @new_resource.devices path = @new_resource.name dev_info = {} - Chef::Log.info("IN MOUNTS") + Chef::Log.info('IN MOUNTS') new_resource.updated_by_last_action(false) # walk through the devices, gathering information proposed_devices.each do |device| - next if !::File.exists?("/dev/#{device}") + next unless ::File.exists?("/dev/#{device}") info = {} - info["device"] = device - info["ip"] = @new_resource.ip - info["format"] = @new_resource.format - info["uuid"] = Mixlib::ShellOut.new("blkid /dev/#{device} -s UUID -o value").run_command.stdout.strip - info["mountpoint"] = info["uuid"].split("-").join("") - info["mounted"] = Mixlib::ShellOut.new("mount | grep '#{path}/#{info["mountpoint"]}\'").run_command.status - info["size"] = Mixlib::ShellOut.new("sfdisk -s /dev/#{device}").run_command.stdout.to_i / 1024 + info['device'] = device + info['ip'] = @new_resource.ip + info['format'] = @new_resource.format + info['uuid'] = Mixlib::ShellOut.new("blkid /dev/#{device} -s UUID -o value").run_command.stdout.strip + info['mountpoint'] = info['uuid'].split('-').join('') + info['mounted'] = Mixlib::ShellOut.new("mount | grep '#{path}/#{info["mountpoint"]}\'").run_command.status + info['size'] = Mixlib::ShellOut.new("sfdisk -s /dev/#{device}").run_command.stdout.to_i / 1024 - next if (info["uuid"] == '') + next if info['uuid'] == '' - dev_info[info["uuid"]] = info + dev_info[info['uuid']] = info end - Chef::Log.info("Physical Inventory:") - dev_info.each do |_,v| + Chef::Log.info('Physical Inventory:') + dev_info.each do |_, v| Chef::Log.info("Device: #{v['device']}, UUID: #{v['uuid']}, Mounted: #{v['mounted']}, Format: #{v['format']}") end # make sure we have a "path" Directory(path) do - group "swift" - owner "swift" + group 'swift' + owner 'swift' recursive true end.run_action(:create) # find what should be mounted, and what IS mounted - mounts=node["filesystem"].inject({}) { |hsh, (k,v)| hsh.merge(v["mount"] => k) } - valid_mounts = dev_info.inject([]) {|ary, (_,v)| ary << "#{path}/#{v['mountpoint']}"} - mountpoints = Dir.new(path).reject {|x| x[/^\./] }.collect { |d| "#{path}/#{d}" } - inverted_mounts = dev_info.inject({}) {|hsh,(k,v)| hsh.merge({v["mountpoint"] => v.merge("uuid" => k)})} - fstabs=::File.readlines("/etc/fstab").inject({}) do |hash,line| - line = line.split("#")[0].split() + mounts = node['filesystem'].reduce({}) { |hsh, (k, v)| hsh.merge(v['mount'] => k) } + valid_mounts = dev_info.reduce([]) { |ary, (_, v)| ary << "#{path}/#{v['mountpoint']}" } + mountpoints = Dir.new(path).reject { |x| x[/^\./] }.map { |d| "#{path}/#{d}" } + inverted_mounts = dev_info.reduce({}) { |hsh, (k, v)| hsh.merge(v['mountpoint'] => v.merge('uuid' => k)) } + fstabs = ::File.readlines('/etc/fstab').reduce({}) do |hash, line| + line = line.split('#')[1].split Chef::Log.info("#{line[0]} ... #{line[1]}") hash.merge(line[1] => line[0]) - end.reject { |k,v| !k or !v or !k.length or !v.length } + end + fstabs.reject! { |k, v| !k || !v || !k.length || !v.length } Chef::Log.info("Mounts: #{mounts}") Chef::Log.info("Valid Mounts: #{valid_mounts}") @@ -77,16 +79,16 @@ action :ensure_exists do Chef::Log.info("Fstabs: #{fstabs}") # mounts in /srv/node that shouldn't be there - (mounts.keys.select{|x| x and x[/^#{path}/]} - valid_mounts).each do |dev| + (mounts.keys.select { |x| x && x[/^#{path}/] } - valid_mounts).each do |dev| Chef::Log.info("Unmounting #{dev}") Mixlib::ShellOut.new("umount #{dev}").run_command if Mixlib::ShellOut.new("mount | grep '#{dev}'").run_command.status new_resource.updated_by_last_action(true) end # fstab entries that don't need to be there anymore - (fstabs.keys.select {|k| k.start_with? path} - valid_mounts).each do |dev| - fe = Chef::Util::FileEdit.new("/etc/fstab") - fe.search_file_delete_line(Regexp.new(dev.gsub("/","\/"))) + (fstabs.keys.select { |k| k.start_with?(path) } - valid_mounts).each do |dev| + fe = Chef::Util::FileEdit.new('/etc/fstab') + fe.search_file_delete_line(Regexp.new(dev.gsub('/', '\/'))) fe.write_file new_resource.updated_by_last_action(true) end @@ -104,27 +106,27 @@ action :ensure_exists do # new, unmounted devices (valid_mounts - mounts.keys).each do |mountpoint| - info = inverted_mounts[mountpoint.gsub("#{path}/","")] + info = inverted_mounts[mountpoint.gsub("#{path}/", '')] Chef::Log.info("mounting #{mountpoint} (#{info['device']})") mount_path = "#{path}/#{info['mountpoint']}" Directory(mount_path) do - group "swift" - owner "swift" + group 'swift' + owner 'swift' recursive true end.run_action(:create) case info['format'] when 'ext4' - mount_options = "noatime,nodiratime,nobarrier,user_xattr" + mount_options = 'noatime,nodiratime,nobarrier,user_xattr' when 'xfs' - case node["platform"] - when "ubuntu","debian" - mount_options = "noatime,nodiratime,nobarrier,logbufs=8,nobootwait" - else - mount_options = "noatime,nodiratime,nobarrier,logbufs=8" + case node['platform'] + when 'ubuntu', 'debian' + mount_options = 'noatime,nodiratime,nobarrier,logbufs=8,nobootwait' + else + mount_options = 'noatime,nodiratime,nobarrier,logbufs=8' end end @@ -137,7 +139,7 @@ action :ensure_exists do action :nothing end - if not fstabs.has_key?(mount_path) + unless fstabs.key?(mount_path) # then its a brand-new drive, freshly formatted Chef::Log.info("Mounting new device #{info['mountpoint']}") mt.run_action(:enable) @@ -147,22 +149,21 @@ action :ensure_exists do new_resource.updated_by_last_action(true) end - dev_info.reject { |k,v| v["mounted"] }.keys.each do |uuid| - dev_info[uuid]["mounted"] = Mixlib::ShellOut.new("mount | grep '#{path}/#{dev_info[uuid]["mountpoint"]}\'").run_command.status + dev_info.reject { |k, v| v['mounted'] }.keys.each do |uuid| + dev_info[uuid]['mounted'] = Mixlib::ShellOut.new("mount | grep '#{path}/#{dev_info[uuid]["mountpoint"]}\'").run_command.status end - if @new_resource.publish_attributes and dev_info != {} - dev_info.each do |k,v| - node.set["swift"]["state"]["devs"][k] = { - :device => v["device"], - :size => v["size"], - :uuid => v["uuid"], - :mounted => v["mounted"], - :format => v["format"], - :mountpoint => v["mountpoint"], - :ip => v["ip"] + if @new_resource.publish_attributes && dev_info != {} + dev_info.each do |k, v| + node.set['swift']['state']['devs'][k] = { + device: v['device'], + size: v['size'], + uuid: v['uuid'], + mounted: v['mounted'], + format: v['format'], + mountpoint: v['mountpoint'], + ip: v['ip'] } end end end - diff --git a/providers/ring_script.rb b/providers/ring_script.rb index 6f8289f..a6305ee 100644 --- a/providers/ring_script.rb +++ b/providers/ring_script.rb @@ -1,3 +1,4 @@ +# encoding: UTF-8 # # Cookbook Name:: openstack-object-storage # Resource:: ring_script @@ -19,19 +20,20 @@ # Author: Ron Pedde # -require "pp" +require 'pp' +# rubocop:disable PerlBackrefs, CyclomaticComplexity, MethodLength def generate_script # need to load and parse the existing rings. - ports = { "object" => "6000", "container" => "6001", "account" => "6002" } + ports = { 'object' => '6000', 'container' => '6001', 'account' => '6002' } must_rebalance = false ring_path = @new_resource.ring_path - ring_data = { :raw => {}, :parsed => {}, :in_use => {} } + ring_data = { raw: {}, parsed: {}, in_use: {} } disk_data = {} dirty_cluster_reasons = [] - [ "account", "container", "object" ].each do |which| + ['account', 'container', 'object'].each do |which| ring_data[:raw][which] = nil if ::File.exist?("#{ring_path}/#{which}.builder") @@ -40,7 +42,7 @@ def generate_script # Chef::Log.debug("#{ which.capitalize } Ring data: #{ring_data[:raw][which]}") ring_data[:parsed][which] = parse_ring_output(ring_data[:raw][which]) - node.set["swift"]["state"]["ring"][which] = ring_data[:parsed][which] + node.set['swift']['state']['ring'][which] = ring_data[:parsed][which] end else Chef::Log.info("#{which.capitalize} ring builder files do not exist!") @@ -58,41 +60,41 @@ def generate_script end end - Chef::Log.debug("#{which.capitalize} Ring - In use: #{PP.pp(ring_data[:in_use][which],dump='')}") + Chef::Log.debug("#{which.capitalize} Ring - In use: #{PP.pp(ring_data[:in_use][which], '')}") # figure out what's present in the cluster disk_data[which] = {} - role = node["swift"]["#{which}_server_chef_role"] - disk_state,_,_ = Chef::Search::Query.new.search(:node,"chef_environment:#{node.chef_environment} AND roles:#{role}") + role = node['swift']["#{which}_server_chef_role"] + disk_state, _, _ = Chef::Search::Query.new.search(:node, "chef_environment:#{node.chef_environment} AND roles:#{role}") # for a running track of available disks disk_data[:available] ||= {} disk_data[:available][which] ||= {} disk_state.each do |swiftnode| - if swiftnode[:swift][:state] and swiftnode[:swift][:state][:devs] - swiftnode[:swift][:state][:devs].each do |k,v| + if swiftnode[:swift][:state] && swiftnode[:swift][:state][:devs] + swiftnode[:swift][:state][:devs].each do |k, v| disk_data[which][v[:ip]] = disk_data[which][v[:ip]] || {} disk_data[which][v[:ip]][k] = {} - v.keys.each { |x| disk_data[which][v[:ip]][k].store(x,v[x]) } + v.keys.each { |x| disk_data[which][v[:ip]][k].store(x, v[x]) } - if swiftnode[:swift].has_key?("#{which}-zone") - disk_data[which][v[:ip]][k]["zone"]=swiftnode[:swift]["#{which}-zone"] - elsif swiftnode[:swift].has_key?("zone") - disk_data[which][v[:ip]][k]["zone"]=swiftnode[:swift]["zone"] + if swiftnode[:swift].key?("#{which}-zone") + disk_data[which][v[:ip]][k]['zone'] = swiftnode[:swift]["#{which}-zone"] + elsif swiftnode[:swift].key?('zone') + disk_data[which][v[:ip]][k]['zone'] = swiftnode[:swift]['zone'] else - raise "Node #{swiftnode[:hostname]} has no zone assigned" + fail "Node #{swiftnode[:hostname]} has no zone assigned" end disk_data[:available][which][v[:mountpoint]] = v[:ip] - if not v[:mounted] + unless v[:mounted] dirty_cluster_reasons << "Disk #{v[:name]} (#{v[:uuid]}) is not mounted on host #{v[:ip]} (#{swiftnode[:hostname]})" end end end end - Chef::Log.debug("#{which.capitalize} Ring - Avail: #{PP.pp(disk_data[:available][which],dump='')}") + Chef::Log.debug("#{which.capitalize} Ring - Avail: #{PP.pp(disk_data[:available][which], '')}") end # Have the raw data, now bump it together and drop the script @@ -100,7 +102,7 @@ def generate_script s = "#!/bin/bash\n\n# This script is automatically generated.\n" s << "# Running it will likely blow up your system if you don't review it carefully.\n" s << "# You have been warned.\n\n" - if not node["swift"]["auto_rebuild_rings"] + unless node['swift']['auto_rebuild_rings'] s << "if [ \"$1\" != \"--force\" ]; then\n" s << " echo \"Auto rebuild rings is disabled, so you must use --force to generate rings\"\n" s << " exit 0\n" @@ -113,24 +115,24 @@ def generate_script missing_disks = {} new_servers = [] - [ "account", "container", "object" ].each do |which| + ['account', 'container', 'object'].each do |which| # remove available disks that are already in the ring - new_disks[which] = disk_data[:available][which].reject{ |k,v| ring_data[:in_use][which].has_key?(k) } + new_disks[which] = disk_data[:available][which].reject { |k, v| ring_data[:in_use][which].key?(k) } # find all in-ring disks that are not in the cluster - missing_disks[which] = ring_data[:in_use][which].reject{ |k,v| disk_data[:available][which].has_key?(k) } + missing_disks[which] = ring_data[:in_use][which].reject { |k, v| disk_data[:available][which].key?(k) } - Chef::Log.debug("#{which.capitalize} Ring - Missing: #{PP.pp(missing_disks[which],dump='')}") - Chef::Log.debug("#{which.capitalize} Ring - New: #{PP.pp(new_disks[which],dump='')}") + Chef::Log.debug("#{which.capitalize} Ring - Missing: #{PP.pp(missing_disks[which], '')}") + Chef::Log.debug("#{which.capitalize} Ring - New: #{PP.pp(new_disks[which], '')}") s << "\n# -- #{which.capitalize} Servers --\n\n" disk_data[which].keys.sort.each do |ip| s << "# #{ip}\n" disk_data[which][ip].keys.sort.each do |k| v = disk_data[which][ip][k] - s << "# " + v.keys.sort.select{|x| ["ip", "device", "uuid"].include?(x)}.collect{|x| v[x] }.join(", ") - if new_disks[which].has_key?(v["mountpoint"]) - s << " (NEW!)" + s << '# ' + v.keys.sort.select { |x| ['ip', 'device', 'uuid'].include?(x) }.map { |x| v[x] }.join(', ') + if new_disks[which].key?(v['mountpoint']) + s << ' (NEW!)' new_servers << ip unless new_servers.include?(ip) end s << "\n" @@ -147,7 +149,7 @@ def generate_script disk_data[which].keys.sort.each do |ip| disk_data[which][ip].keys.sort.each do |uuid| v = disk_data[which][ip][uuid] - if new_disks[which].has_key?(v['mountpoint']) + if new_disks[which].key?(v['mountpoint']) s << "swift-ring-builder #{ring_path}/#{which}.builder add z#{v['zone']}-#{v['ip']}:#{ports[which]}/#{v['mountpoint']} #{v['size']}\n" must_rebalance = true end @@ -156,9 +158,9 @@ def generate_script # remove the disks -- sort to ensure consistent order missing_disks[which].keys.sort.each do |mountpoint| - diskinfo=ring_data[:parsed][which][:hosts].select{|k,v| v.has_key?(mountpoint)}.collect{|_,v| v[mountpoint]}[0] - Chef::Log.debug("Missing diskinfo: #{PP.pp(diskinfo,dump='')}") - description = Hash[diskinfo.select{|k,v| [:zone, :ip, :device].include?(k)}].collect{|k,v| "#{k}: #{v}" }.join(", ") + diskinfo = ring_data[:parsed][which][:hosts].select { |k, v| v.key?(mountpoint) }.map { |_, v| v[mountpoint] }[0] + Chef::Log.debug("Missing diskinfo: #{PP.pp(diskinfo, '')}") + description = Hash[diskinfo.select { |k, v| [:zone, :ip, :device].include?(k) }].map { |k, v| "#{k}: #{v}" }.join(', ') s << "# #{description}\n" s << "swift-ring-builder #{ring_path}/#{which}.builder remove d#{missing_disks[which][mountpoint]}\n" must_rebalance = true @@ -166,26 +168,26 @@ def generate_script s << "\n" - if(must_rebalance) + if must_rebalance s << "swift-ring-builder #{ring_path}/#{which}.builder rebalance\n\n\n" else s << "# #{which.capitalize} ring has no outstanding changes!\n\n" end # we'll only rebalance if we meet the minimums for new adds - if node["swift"].has_key?("wait_for") - if node["swift"]["wait_for"] > new_servers.count - Chef::Log.debug("New servers, but not enough to force a rebalance") + if node['swift'].key?('wait_for') + if node['swift']['wait_for'] > new_servers.count + Chef::Log.debug('New servers, but not enough to force a rebalance') must_rebalance = false end end end - [ s, must_rebalance ] + [s, must_rebalance] end # Parse the raw output of swift-ring-builder def parse_ring_output(ring_data) - output = { :state => {} } + output = { state: {} } ring_data.each do |line| if line =~ /build version ([0-9]+)/ @@ -256,7 +258,7 @@ def parse_ring_output(ring_data) elsif line =~ /^The minimum number of hours before a partition can be reassigned is (\d+)$/ output[:state][:min_part_hours] = $1 else - raise "Cannot parse ring builder output for #{line}" + fail "Cannot parse ring builder output for #{line}" end end @@ -266,7 +268,7 @@ end action :ensure_exists do Chef::Log.debug("Ensuring #{new_resource.name}") new_resource.updated_by_last_action(false) - s,must_update = generate_script + s, must_update = generate_script script_file = File new_resource.name do owner new_resource.owner