openstack
/
swift
Code Issues Proposed changes

Merge "Check swift.conf MD5 with recon"

This commit is contained in:
Jenkins 2014-04-23 03:23:47 +00:00 committed by Gerrit Code Review
parent c20a8377c6 31dac18625
commit fc7f6dd924
4 changed files with 246 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

249
swift/cli/recon.py
View File

@ -16,8 +16,10 @@
cmdline utility to perform cluster reconnaissance cmdline utility to perform cluster reconnaissance
""" """
from __future__ import print_function
from eventlet.green import urllib2 from eventlet.green import urllib2
from swift.common.utils import SWIFT_CONF_FILE
from swift.common.ring import Ring from swift.common.ring import Ring
from urlparse import urlparse from urlparse import urlparse
try: try:
@ -82,16 +84,16 @@ class Scout(object):
body = urllib2.urlopen(url, timeout=self.timeout).read() body = urllib2.urlopen(url, timeout=self.timeout).read()
content = json.loads(body) content = json.loads(body)
if self.verbose: if self.verbose:
print "-> %s: %s" % (url, content) print("-> %s: %s" % (url, content))
status = 200 status = 200
except urllib2.HTTPError as err: except urllib2.HTTPError as err:
if not self.suppress_errors or self.verbose: if not self.suppress_errors or self.verbose:
print "-> %s: %s" % (url, err) print("-> %s: %s" % (url, err))
content = err content = err
status = err.code status = err.code
except urllib2.URLError as err: except urllib2.URLError as err:
if not self.suppress_errors or self.verbose: if not self.suppress_errors or self.verbose:
print "-> %s: %s" % (url, err) print("-> %s: %s" % (url, err))
content = err content = err
status = -1 status = -1
return url, content, status return url, content, status
@ -143,10 +145,10 @@ class SwiftRecon(object):
:param stats: dict of stats generated by _gen_stats :param stats: dict of stats generated by _gen_stats
""" """
print '[%(name)s] low: %(low)d, high: %(high)d, avg: ' \ print('[%(name)s] low: %(low)d, high: %(high)d, avg: '
'%(average).1f, total: %(total)d, ' \ '%(average).1f, total: %(total)d, '
'Failed: %(perc_none).1f%%, no_result: %(number_none)d, ' \ 'Failed: %(perc_none).1f%%, no_result: %(number_none)d, '
'reported: %(reported)d' % stats 'reported: %(reported)d' % stats)
def _ptime(self, timev=None): def _ptime(self, timev=None):
""" """
@ -158,6 +160,21 @@ class SwiftRecon(object):
else: else:
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
def _md5_file(self, path):
"""
Get the MD5 checksum of a file.
:param path: path to file
:returns: MD5 checksum, hex encoded
"""
md5sum = md5()
with open(path, 'rb') as f:
block = f.read(4096)
while block:
md5sum.update(block)
block = f.read(4096)
return md5sum.hexdigest()
def get_devices(self, zone_filter, swift_dir, ring_name): def get_devices(self, zone_filter, swift_dir, ring_name):
""" """
Get a list of hosts in the ring Get a list of hosts in the ring
@ -183,36 +200,59 @@ class SwiftRecon(object):
set([('127.0.0.1', 6020), ('127.0.0.2', 6030)]) set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
:param ringfile: The local ring file to compare the md5sum with. :param ringfile: The local ring file to compare the md5sum with.
""" """
stats = {}
matches = 0 matches = 0
errors = 0 errors = 0
md5sum = md5() ring_sum = self._md5_file(ringfile)
with open(ringfile, 'rb') as f:
block = f.read(4096)
while block:
md5sum.update(block)
block = f.read(4096)
ring_sum = md5sum.hexdigest()
recon = Scout("ringmd5", self.verbose, self.suppress_errors, recon = Scout("ringmd5", self.verbose, self.suppress_errors,
self.timeout) self.timeout)
print "[%s] Checking ring md5sums" % self._ptime() print("[%s] Checking ring md5sums" % self._ptime())
if self.verbose: if self.verbose:
print "-> On disk %s md5sum: %s" % (ringfile, ring_sum) print("-> On disk %s md5sum: %s" % (ringfile, ring_sum))
for url, response, status in self.pool.imap(recon.scout, hosts): for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200: if status == 200:
stats[url] = response[ringfile]
if response[ringfile] != ring_sum: if response[ringfile] != ring_sum:
print "!! %s (%s) doesn't match on disk md5sum" % \ print("!! %s (%s) doesn't match on disk md5sum" %
(url, response[ringfile]) (url, response[ringfile]))
else: else:
matches = matches + 1 matches = matches + 1
if self.verbose: if self.verbose:
print "-> %s matches." % url print("-> %s matches." % url)
else: else:
errors = errors + 1 errors = errors + 1
print "%s/%s hosts matched, %s error[s] while checking hosts." \ print("%s/%s hosts matched, %s error[s] while checking hosts."
% (matches, len(hosts), errors) % (matches, len(hosts), errors))
print "=" * 79 print("=" * 79)
def get_swiftconfmd5(self, hosts, printfn=print):
"""
Compare swift.conf md5sum with that on remote hosts
:param hosts: set of hosts to check. in the format of:
set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
:param printfn: function to print text; defaults to print()
"""
matches = 0
errors = 0
conf_sum = self._md5_file(SWIFT_CONF_FILE)
recon = Scout("swiftconfmd5", self.verbose, self.suppress_errors,
self.timeout)
printfn("[%s] Checking swift.conf md5sum" % self._ptime())
if self.verbose:
printfn("-> On disk swift.conf md5sum: %s" % (conf_sum,))
for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200:
if response[SWIFT_CONF_FILE] != conf_sum:
printfn("!! %s (%s) doesn't match on disk md5sum" %
(url, response[SWIFT_CONF_FILE]))
else:
matches = matches + 1
if self.verbose:
printfn("-> %s matches." % url)
else:
errors = errors + 1
printfn("%s/%s hosts matched, %s error[s] while checking hosts."
% (matches, len(hosts), errors))
printfn("=" * 79)
def async_check(self, hosts): def async_check(self, hosts):
""" """
@ -224,7 +264,7 @@ class SwiftRecon(object):
scan = {} scan = {}
recon = Scout("async", self.verbose, self.suppress_errors, recon = Scout("async", self.verbose, self.suppress_errors,
self.timeout) self.timeout)
print "[%s] Checking async pendings" % self._ptime() print("[%s] Checking async pendings" % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts): for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200: if status == 200:
scan[url] = response['async_pending'] scan[url] = response['async_pending']
@ -232,8 +272,8 @@ class SwiftRecon(object):
if stats['reported'] > 0: if stats['reported'] > 0:
self._print_stats(stats) self._print_stats(stats)
else: else:
print "[async_pending] - No hosts returned valid data." print("[async_pending] - No hosts returned valid data.")
print "=" * 79 print("=" * 79)
def umount_check(self, hosts): def umount_check(self, hosts):
""" """
@ -246,8 +286,8 @@ class SwiftRecon(object):
errors = {} errors = {}
recon = Scout("unmounted", self.verbose, self.suppress_errors, recon = Scout("unmounted", self.verbose, self.suppress_errors,
self.timeout) self.timeout)
print "[%s] Getting unmounted drives from %s hosts..." % \ print("[%s] Getting unmounted drives from %s hosts..." %
(self._ptime(), len(hosts)) (self._ptime(), len(hosts)))
for url, response, status in self.pool.imap(recon.scout, hosts): for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200: if status == 200:
unmounted[url] = [] unmounted[url] = []
@ -260,12 +300,12 @@ class SwiftRecon(object):
for host in unmounted: for host in unmounted:
node = urlparse(host).netloc node = urlparse(host).netloc
for entry in unmounted[host]: for entry in unmounted[host]:
print "Not mounted: %s on %s" % (entry, node) print("Not mounted: %s on %s" % (entry, node))
for host in errors: for host in errors:
node = urlparse(host).netloc node = urlparse(host).netloc
for entry in errors[host]: for entry in errors[host]:
print "Device errors: %s on %s" % (entry, node) print("Device errors: %s on %s" % (entry, node))
print "=" * 79 print("=" * 79)
def expirer_check(self, hosts): def expirer_check(self, hosts):
""" """
@ -277,7 +317,7 @@ class SwiftRecon(object):
stats = {'object_expiration_pass': [], 'expired_last_pass': []} stats = {'object_expiration_pass': [], 'expired_last_pass': []}
recon = Scout("expirer/%s" % self.server_type, self.verbose, recon = Scout("expirer/%s" % self.server_type, self.verbose,
self.suppress_errors, self.timeout) self.suppress_errors, self.timeout)
print "[%s] Checking on expirers" % self._ptime() print("[%s] Checking on expirers" % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts): for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200: if status == 200:
stats['object_expiration_pass'].append( stats['object_expiration_pass'].append(
@ -290,10 +330,10 @@ class SwiftRecon(object):
if computed['reported'] > 0: if computed['reported'] > 0:
self._print_stats(computed) self._print_stats(computed)
else: else:
print "[%s] - No hosts returned valid data." % k print("[%s] - No hosts returned valid data." % k)
else: else:
print "[%s] - No hosts returned valid data." % k print("[%s] - No hosts returned valid data." % k)
print "=" * 79 print("=" * 79)
def replication_check(self, hosts): def replication_check(self, hosts):
""" """
@ -306,7 +346,7 @@ class SwiftRecon(object):
'attempted': []} 'attempted': []}
recon = Scout("replication/%s" % self.server_type, self.verbose, recon = Scout("replication/%s" % self.server_type, self.verbose,
self.suppress_errors, self.timeout) self.suppress_errors, self.timeout)
print "[%s] Checking on replication" % self._ptime() print("[%s] Checking on replication" % self._ptime())
least_recent_time = 9999999999 least_recent_time = 9999999999
least_recent_url = None least_recent_url = None
most_recent_time = 0 most_recent_time = 0
@ -336,29 +376,29 @@ class SwiftRecon(object):
if computed['reported'] > 0: if computed['reported'] > 0:
self._print_stats(computed) self._print_stats(computed)
else: else:
print "[%s] - No hosts returned valid data." % k print("[%s] - No hosts returned valid data." % k)
else: else:
print "[%s] - No hosts returned valid data." % k print("[%s] - No hosts returned valid data." % k)
if least_recent_url is not None: if least_recent_url is not None:
host = urlparse(least_recent_url).netloc host = urlparse(least_recent_url).netloc
if not least_recent_time: if not least_recent_time:
print 'Oldest completion was NEVER by %s.' % host print('Oldest completion was NEVER by %s.' % host)
else: else:
elapsed = time.time() - least_recent_time elapsed = time.time() - least_recent_time
elapsed, elapsed_unit = seconds2timeunit(elapsed) elapsed, elapsed_unit = seconds2timeunit(elapsed)
print 'Oldest completion was %s (%d %s ago) by %s.' % ( print('Oldest completion was %s (%d %s ago) by %s.' % (
time.strftime('%Y-%m-%d %H:%M:%S', time.strftime('%Y-%m-%d %H:%M:%S',
time.gmtime(least_recent_time)), time.gmtime(least_recent_time)),
elapsed, elapsed_unit, host) elapsed, elapsed_unit, host))
if most_recent_url is not None: if most_recent_url is not None:
host = urlparse(most_recent_url).netloc host = urlparse(most_recent_url).netloc
elapsed = time.time() - most_recent_time elapsed = time.time() - most_recent_time
elapsed, elapsed_unit = seconds2timeunit(elapsed) elapsed, elapsed_unit = seconds2timeunit(elapsed)
print 'Most recent completion was %s (%d %s ago) by %s.' % ( print('Most recent completion was %s (%d %s ago) by %s.' % (
time.strftime('%Y-%m-%d %H:%M:%S', time.strftime('%Y-%m-%d %H:%M:%S',
time.gmtime(most_recent_time)), time.gmtime(most_recent_time)),
elapsed, elapsed_unit, host) elapsed, elapsed_unit, host))
print "=" * 79 print("=" * 79)
def object_replication_check(self, hosts): def object_replication_check(self, hosts):
""" """
@ -370,7 +410,7 @@ class SwiftRecon(object):
stats = {} stats = {}
recon = Scout("replication", self.verbose, self.suppress_errors, recon = Scout("replication", self.verbose, self.suppress_errors,
self.timeout) self.timeout)
print "[%s] Checking on replication" % self._ptime() print("[%s] Checking on replication" % self._ptime())
least_recent_time = 9999999999 least_recent_time = 9999999999
least_recent_url = None least_recent_url = None
most_recent_time = 0 most_recent_time = 0
@ -391,29 +431,29 @@ class SwiftRecon(object):
if computed['reported'] > 0: if computed['reported'] > 0:
self._print_stats(computed) self._print_stats(computed)
else: else:
print "[replication_time] - No hosts returned valid data." print("[replication_time] - No hosts returned valid data.")
else: else:
print "[replication_time] - No hosts returned valid data." print("[replication_time] - No hosts returned valid data.")
if least_recent_url is not None: if least_recent_url is not None:
host = urlparse(least_recent_url).netloc host = urlparse(least_recent_url).netloc
if not least_recent_time: if not least_recent_time:
print 'Oldest completion was NEVER by %s.' % host print('Oldest completion was NEVER by %s.' % host)
else: else:
elapsed = time.time() - least_recent_time elapsed = time.time() - least_recent_time
elapsed, elapsed_unit = seconds2timeunit(elapsed) elapsed, elapsed_unit = seconds2timeunit(elapsed)
print 'Oldest completion was %s (%d %s ago) by %s.' % ( print('Oldest completion was %s (%d %s ago) by %s.' % (
time.strftime('%Y-%m-%d %H:%M:%S', time.strftime('%Y-%m-%d %H:%M:%S',
time.gmtime(least_recent_time)), time.gmtime(least_recent_time)),
elapsed, elapsed_unit, host) elapsed, elapsed_unit, host))
if most_recent_url is not None: if most_recent_url is not None:
host = urlparse(most_recent_url).netloc host = urlparse(most_recent_url).netloc
elapsed = time.time() - most_recent_time elapsed = time.time() - most_recent_time
elapsed, elapsed_unit = seconds2timeunit(elapsed) elapsed, elapsed_unit = seconds2timeunit(elapsed)
print 'Most recent completion was %s (%d %s ago) by %s.' % ( print('Most recent completion was %s (%d %s ago) by %s.' % (
time.strftime('%Y-%m-%d %H:%M:%S', time.strftime('%Y-%m-%d %H:%M:%S',
time.gmtime(most_recent_time)), time.gmtime(most_recent_time)),
elapsed, elapsed_unit, host) elapsed, elapsed_unit, host))
print "=" * 79 print("=" * 79)
def updater_check(self, hosts): def updater_check(self, hosts):
""" """
@ -425,7 +465,7 @@ class SwiftRecon(object):
stats = [] stats = []
recon = Scout("updater/%s" % self.server_type, self.verbose, recon = Scout("updater/%s" % self.server_type, self.verbose,
self.suppress_errors, self.timeout) self.suppress_errors, self.timeout)
print "[%s] Checking updater times" % self._ptime() print("[%s] Checking updater times" % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts): for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200: if status == 200:
if response['%s_updater_sweep' % self.server_type]: if response['%s_updater_sweep' % self.server_type]:
@ -436,10 +476,10 @@ class SwiftRecon(object):
if computed['reported'] > 0: if computed['reported'] > 0:
self._print_stats(computed) self._print_stats(computed)
else: else:
print "[updater_last_sweep] - No hosts returned valid data." print("[updater_last_sweep] - No hosts returned valid data.")
else: else:
print "[updater_last_sweep] - No hosts returned valid data." print("[updater_last_sweep] - No hosts returned valid data.")
print "=" * 79 print("=" * 79)
def auditor_check(self, hosts): def auditor_check(self, hosts):
""" """
@ -455,12 +495,12 @@ class SwiftRecon(object):
asince = '%s_audits_since' % self.server_type asince = '%s_audits_since' % self.server_type
recon = Scout("auditor/%s" % self.server_type, self.verbose, recon = Scout("auditor/%s" % self.server_type, self.verbose,
self.suppress_errors, self.timeout) self.suppress_errors, self.timeout)
print "[%s] Checking auditor stats" % self._ptime() print("[%s] Checking auditor stats" % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts): for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200: if status == 200:
scan[url] = response scan[url] = response
if len(scan) < 1: if len(scan) < 1:
print "Error: No hosts available" print("Error: No hosts available")
return return
stats = {} stats = {}
stats[adone] = [scan[i][adone] for i in scan stats[adone] = [scan[i][adone] for i in scan
@ -473,7 +513,7 @@ class SwiftRecon(object):
if scan[i][asince] is not None] if scan[i][asince] is not None]
for k in stats: for k in stats:
if len(stats[k]) < 1: if len(stats[k]) < 1:
print "[%s] - No hosts returned valid data." % k print("[%s] - No hosts returned valid data." % k)
else: else:
if k != asince: if k != asince:
computed = self._gen_stats(stats[k], k) computed = self._gen_stats(stats[k], k)
@ -484,9 +524,9 @@ class SwiftRecon(object):
high = max(stats[asince]) high = max(stats[asince])
total = sum(stats[asince]) total = sum(stats[asince])
average = total / len(stats[asince]) average = total / len(stats[asince])
print '[last_pass] oldest: %s, newest: %s, avg: %s' % \ print('[last_pass] oldest: %s, newest: %s, avg: %s' %
(self._ptime(low), self._ptime(high), self._ptime(average)) (self._ptime(low), self._ptime(high), self._ptime(average)))
print "=" * 79 print("=" * 79)
def nested_get_value(self, key, recon_entry): def nested_get_value(self, key, recon_entry):
""" """
@ -522,7 +562,7 @@ class SwiftRecon(object):
quarantined = 'quarantined' quarantined = 'quarantined'
recon = Scout("auditor/object", self.verbose, self.suppress_errors, recon = Scout("auditor/object", self.verbose, self.suppress_errors,
self.timeout) self.timeout)
print "[%s] Checking auditor stats " % self._ptime() print("[%s] Checking auditor stats " % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts): for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200: if status == 200:
if response['object_auditor_stats_ALL']: if response['object_auditor_stats_ALL']:
@ -545,16 +585,16 @@ class SwiftRecon(object):
if None in stats[k]: if None in stats[k]:
stats[k] = [x for x in stats[k] if x is not None] stats[k] = [x for x in stats[k] if x is not None]
if len(stats[k]) < 1: if len(stats[k]) < 1:
print "[Auditor %s] - No hosts returned valid data." % k print("[Auditor %s] - No hosts returned valid data." % k)
else: else:
computed = self._gen_stats(stats[k], computed = self._gen_stats(stats[k],
name='ALL_%s_last_path' % k) name='ALL_%s_last_path' % k)
if computed['reported'] > 0: if computed['reported'] > 0:
self._print_stats(computed) self._print_stats(computed)
else: else:
print "[ALL_auditor] - No hosts returned valid data." print("[ALL_auditor] - No hosts returned valid data.")
else: else:
print "[ALL_auditor] - No hosts returned valid data." print("[ALL_auditor] - No hosts returned valid data.")
if len(zbf_scan) > 0: if len(zbf_scan) > 0:
stats = {} stats = {}
stats[atime] = [(self.nested_get_value(atime, zbf_scan[i])) stats[atime] = [(self.nested_get_value(atime, zbf_scan[i]))
@ -569,17 +609,17 @@ class SwiftRecon(object):
if None in stats[k]: if None in stats[k]:
stats[k] = [x for x in stats[k] if x is not None] stats[k] = [x for x in stats[k] if x is not None]
if len(stats[k]) < 1: if len(stats[k]) < 1:
print "[Auditor %s] - No hosts returned valid data." % k print("[Auditor %s] - No hosts returned valid data." % k)
else: else:
computed = self._gen_stats(stats[k], computed = self._gen_stats(stats[k],
name='ZBF_%s_last_path' % k) name='ZBF_%s_last_path' % k)
if computed['reported'] > 0: if computed['reported'] > 0:
self._print_stats(computed) self._print_stats(computed)
else: else:
print "[ZBF_auditor] - No hosts returned valid data." print("[ZBF_auditor] - No hosts returned valid data.")
else: else:
print "[ZBF_auditor] - No hosts returned valid data." print("[ZBF_auditor] - No hosts returned valid data.")
print "=" * 79 print("=" * 79)
def load_check(self, hosts): def load_check(self, hosts):
""" """
@ -593,7 +633,7 @@ class SwiftRecon(object):
load15 = {} load15 = {}
recon = Scout("load", self.verbose, self.suppress_errors, recon = Scout("load", self.verbose, self.suppress_errors,
self.timeout) self.timeout)
print "[%s] Checking load averages" % self._ptime() print("[%s] Checking load averages" % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts): for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200: if status == 200:
load1[url] = response['1m'] load1[url] = response['1m']
@ -606,8 +646,8 @@ class SwiftRecon(object):
name='%s_load_avg' % item) name='%s_load_avg' % item)
self._print_stats(computed) self._print_stats(computed)
else: else:
print "[%s_load_avg] - No hosts returned valid data." % item print("[%s_load_avg] - No hosts returned valid data." % item)
print "=" * 79 print("=" * 79)
def quarantine_check(self, hosts): def quarantine_check(self, hosts):
""" """
@ -621,7 +661,7 @@ class SwiftRecon(object):
acctq = {} acctq = {}
recon = Scout("quarantined", self.verbose, self.suppress_errors, recon = Scout("quarantined", self.verbose, self.suppress_errors,
self.timeout) self.timeout)
print "[%s] Checking quarantine" % self._ptime() print("[%s] Checking quarantine" % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts): for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200: if status == 200:
objq[url] = response['objects'] objq[url] = response['objects']
@ -634,8 +674,8 @@ class SwiftRecon(object):
name='quarantined_%s' % item) name='quarantined_%s' % item)
self._print_stats(computed) self._print_stats(computed)
else: else:
print "No hosts returned valid data." print("No hosts returned valid data.")
print "=" * 79 print("=" * 79)
def socket_usage(self, hosts): def socket_usage(self, hosts):
""" """
@ -651,7 +691,7 @@ class SwiftRecon(object):
orphan = {} orphan = {}
recon = Scout("sockstat", self.verbose, self.suppress_errors, recon = Scout("sockstat", self.verbose, self.suppress_errors,
self.timeout) self.timeout)
print "[%s] Checking socket usage" % self._ptime() print("[%s] Checking socket usage" % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts): for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200: if status == 200:
inuse4[url] = response['tcp_in_use'] inuse4[url] = response['tcp_in_use']
@ -667,8 +707,8 @@ class SwiftRecon(object):
computed = self._gen_stats(stats[item].values(), item) computed = self._gen_stats(stats[item].values(), item)
self._print_stats(computed) self._print_stats(computed)
else: else:
print "No hosts returned valid data." print("No hosts returned valid data.")
print "=" * 79 print("=" * 79)
def disk_usage(self, hosts, top=0, human_readable=False): def disk_usage(self, hosts, top=0, human_readable=False):
""" """
@ -686,14 +726,14 @@ class SwiftRecon(object):
top_percents = [(None, 0)] * top top_percents = [(None, 0)] * top
recon = Scout("diskusage", self.verbose, self.suppress_errors, recon = Scout("diskusage", self.verbose, self.suppress_errors,
self.timeout) self.timeout)
print "[%s] Checking disk usage now" % self._ptime() print("[%s] Checking disk usage now" % self._ptime())
for url, response, status in self.pool.imap(recon.scout, hosts): for url, response, status in self.pool.imap(recon.scout, hosts):
if status == 200: if status == 200:
hostusage = [] hostusage = []
for entry in response: for entry in response:
if not isinstance(entry['mounted'], bool): if not isinstance(entry['mounted'], bool):
print "-> %s/%s: Error: %s" % (url, entry['device'], print("-> %s/%s: Error: %s" % (url, entry['device'],
entry['mounted']) entry['mounted']))
elif entry['mounted']: elif entry['mounted']:
used = float(entry['used']) / float(entry['size']) \ used = float(entry['used']) / float(entry['size']) \
* 100.0 * 100.0
@ -719,17 +759,17 @@ class SwiftRecon(object):
for percent in stats[url]: for percent in stats[url]:
percents[int(percent)] = percents.get(int(percent), 0) + 1 percents[int(percent)] = percents.get(int(percent), 0) + 1
else: else:
print "-> %s: Error. No drive info available." % url print("-> %s: Error. No drive info available." % url)
if len(lows) > 0: if len(lows) > 0:
low = min(lows) low = min(lows)
high = max(highs) high = max(highs)
# dist graph shamelessly stolen from https://github.com/gholt/tcod # dist graph shamelessly stolen from https://github.com/gholt/tcod
print "Distribution Graph:" print("Distribution Graph:")
mul = 69.0 / max(percents.values()) mul = 69.0 / max(percents.values())
for percent in sorted(percents): for percent in sorted(percents):
print '% 3d%%%5d %s' % (percent, percents[percent], print('% 3d%%%5d %s' % (percent, percents[percent],
'*' * int(percents[percent] * mul)) '*' * int(percents[percent] * mul)))
raw_used = sum(raw_total_used) raw_used = sum(raw_total_used)
raw_avail = sum(raw_total_avail) raw_avail = sum(raw_total_avail)
raw_total = raw_used + raw_avail raw_total = raw_used + raw_avail
@ -738,26 +778,26 @@ class SwiftRecon(object):
raw_used = size_suffix(raw_used) raw_used = size_suffix(raw_used)
raw_avail = size_suffix(raw_avail) raw_avail = size_suffix(raw_avail)
raw_total = size_suffix(raw_total) raw_total = size_suffix(raw_total)
print "Disk usage: space used: %s of %s" % (raw_used, raw_total) print("Disk usage: space used: %s of %s" % (raw_used, raw_total))
print "Disk usage: space free: %s of %s" % (raw_avail, raw_total) print("Disk usage: space free: %s of %s" % (raw_avail, raw_total))
print "Disk usage: lowest: %s%%, highest: %s%%, avg: %s%%" % \ print("Disk usage: lowest: %s%%, highest: %s%%, avg: %s%%" %
(low, high, avg_used) (low, high, avg_used))
else: else:
print "No hosts returned valid data." print("No hosts returned valid data.")
print "=" * 79 print("=" * 79)
if top_percents: if top_percents:
print 'TOP %s' % top print('TOP %s' % top)
for ident, used in top_percents: for ident, used in top_percents:
if ident: if ident:
url, device = ident.split() url, device = ident.split()
host = urlparse(url).netloc.split(':')[0] host = urlparse(url).netloc.split(':')[0]
print '%.02f%% %s' % (used, '%-15s %s' % (host, device)) print('%.02f%% %s' % (used, '%-15s %s' % (host, device)))
def main(self): def main(self):
""" """
Retrieve and report cluster info from hosts running recon middleware. Retrieve and report cluster info from hosts running recon middleware.
""" """
print "=" * 79 print("=" * 79)
usage = ''' usage = '''
usage: %prog <server_type> [-v] [--suppress] [-a] [-r] [-u] [-d] usage: %prog <server_type> [-v] [--suppress] [-a] [-r] [-u] [-d]
[-l] [--md5] [--auditor] [--updater] [--expirer] [--sockstat] [-l] [--md5] [--auditor] [--updater] [--expirer] [--sockstat]
@ -820,7 +860,7 @@ class SwiftRecon(object):
if arguments[0] in self.check_types: if arguments[0] in self.check_types:
self.server_type = arguments[0] self.server_type = arguments[0]
else: else:
print "Invalid Server Type" print("Invalid Server Type")
args.print_help() args.print_help()
sys.exit(1) sys.exit(1)
else: else:
@ -837,8 +877,8 @@ class SwiftRecon(object):
else: else:
hosts = self.get_devices(None, swift_dir, self.server_type) hosts = self.get_devices(None, swift_dir, self.server_type)
print "--> Starting reconnaissance on %s hosts" % len(hosts) print("--> Starting reconnaissance on %s hosts" % len(hosts))
print "=" * 79 print("=" * 79)
if options.all: if options.all:
if self.server_type == 'object': if self.server_type == 'object':
@ -865,7 +905,7 @@ class SwiftRecon(object):
if self.server_type == 'object': if self.server_type == 'object':
self.async_check(hosts) self.async_check(hosts)
else: else:
print "Error: Can't check asyncs on non object servers." print("Error: Can't check asyncs on non object servers.")
if options.unmounted: if options.unmounted:
self.umount_check(hosts) self.umount_check(hosts)
if options.replication: if options.replication:
@ -880,20 +920,21 @@ class SwiftRecon(object):
self.auditor_check(hosts) self.auditor_check(hosts)
if options.updater: if options.updater:
if self.server_type == 'account': if self.server_type == 'account':
print "Error: Can't check updaters on account servers." print("Error: Can't check updaters on account servers.")
else: else:
self.updater_check(hosts) self.updater_check(hosts)
if options.expirer: if options.expirer:
if self.server_type == 'object': if self.server_type == 'object':
self.expirer_check(hosts) self.expirer_check(hosts)
else: else:
print "Error: Can't check expired on non object servers." print("Error: Can't check expired on non object servers.")
if options.loadstats: if options.loadstats:
self.load_check(hosts) self.load_check(hosts)
if options.diskusage: if options.diskusage:
self.disk_usage(hosts, options.top, options.human_readable) self.disk_usage(hosts, options.top, options.human_readable)
if options.md5: if options.md5:
self.get_ringmd5(hosts, ring_file) self.get_ringmd5(hosts, ring_file)
self.get_swiftconfmd5(hosts)
if options.quarantined: if options.quarantined:
self.quarantine_check(hosts) self.quarantine_check(hosts)
if options.sockstat: if options.sockstat:
@ -905,7 +946,7 @@ def main():
reconnoiter = SwiftRecon() reconnoiter = SwiftRecon()
reconnoiter.main() reconnoiter.main()
except KeyboardInterrupt: except KeyboardInterrupt:
print '\n' print('\n')
if __name__ == '__main__': if __name__ == '__main__':

22
swift/common/middleware/recon.py
View File

@ -19,7 +19,8 @@ from swift import gettext_ as _
from swift import __version__ as swiftver from swift import __version__ as swiftver
from swift.common.swob import Request, Response from swift.common.swob import Request, Response
from swift.common.utils import get_logger, config_true_value, json from swift.common.utils import get_logger, config_true_value, json, \
SWIFT_CONF_FILE
from swift.common.constraints import check_mount from swift.common.constraints import check_mount
from resource import getpagesize from resource import getpagesize
from hashlib import md5 from hashlib import md5
@ -244,6 +245,23 @@ class ReconMiddleware(object):
self.logger.exception(_('Error reading ringfile')) self.logger.exception(_('Error reading ringfile'))
return sums return sums
def get_swift_conf_md5(self, openr=open):
"""get md5 of swift.conf"""
md5sum = md5()
try:
with openr(SWIFT_CONF_FILE, 'r') as fh:
chunk = fh.read(4096)
while chunk:
md5sum.update(chunk)
chunk = fh.read(4096)
except IOError as err:
if err.errno != errno.ENOENT:
self.logger.exception(_('Error reading swift.conf'))
hexsum = None
else:
hexsum = md5sum.hexdigest()
return {SWIFT_CONF_FILE: hexsum}
def get_quarantine_count(self): def get_quarantine_count(self):
"""get obj/container/account quarantine counts""" """get obj/container/account quarantine counts"""
qcounts = {"objects": 0, "containers": 0, "accounts": 0} qcounts = {"objects": 0, "containers": 0, "accounts": 0}
@ -318,6 +336,8 @@ class ReconMiddleware(object):
content = self.get_diskusage() content = self.get_diskusage()
elif rcheck == "ringmd5": elif rcheck == "ringmd5":
content = self.get_ring_md5() content = self.get_ring_md5()
elif rcheck == "swiftconfmd5":
content = self.get_swift_conf_md5()
elif rcheck == "quarantined": elif rcheck == "quarantined":
content = self.get_quarantine_count() content = self.get_quarantine_count()
elif rcheck == "sockstat": elif rcheck == "sockstat":

69
test/unit/cli/test_recon.py
View File

@ -21,6 +21,7 @@ import string
import tempfile import tempfile
import time import time
import unittest import unittest
import urlparse
from eventlet.green import urllib2 from eventlet.green import urllib2
@ -146,3 +147,71 @@ class TestRecon(unittest.TestCase):
ips = self.recon_instance.get_devices( ips = self.recon_instance.get_devices(
1, self.swift_dir, self.ring_name) 1, self.swift_dir, self.ring_name)
self.assertEqual(set([('127.0.0.1', 10001)]), ips) self.assertEqual(set([('127.0.0.1', 10001)]), ips)
class TestReconCommands(unittest.TestCase):
def setUp(self):
self.recon = recon.SwiftRecon()
self.hosts = set([('127.0.0.1', 10000)])
def mock_responses(self, resps):
def fake_urlopen(url, timeout):
scheme, netloc, path, _, _, _ = urlparse.urlparse(url)
self.assertEqual(scheme, 'http') # can't handle anything else
self.assertTrue(path.startswith('/recon/'))
if ':' in netloc:
host, port = netloc.split(':', 1)
port = int(port)
else:
host = netloc
port = 80
response_body = resps[(host, port, path[7:])]
resp = mock.MagicMock()
resp.read = mock.MagicMock(side_effect=[response_body])
return resp
return mock.patch('eventlet.green.urllib2.urlopen', fake_urlopen)
def test_get_swiftconfmd5(self):
hosts = set([('10.1.1.1', 10000),
('10.2.2.2', 10000)])
cksum = '729cf900f2876dead617d088ece7fe8c'
responses = {
('10.1.1.1', 10000, 'swiftconfmd5'):
json.dumps({'/etc/swift/swift.conf': cksum}),
('10.2.2.2', 10000, 'swiftconfmd5'):
json.dumps({'/etc/swift/swift.conf': cksum})}
printed = []
with self.mock_responses(responses):
with mock.patch.object(self.recon, '_md5_file', lambda _: cksum):
self.recon.get_swiftconfmd5(hosts, printfn=printed.append)
output = '\n'.join(printed) + '\n'
self.assertTrue("2/2 hosts matched" in output)
def test_get_swiftconfmd5_mismatch(self):
hosts = set([('10.1.1.1', 10000),
('10.2.2.2', 10000)])
cksum = '29d5912b1fcfcc1066a7f51412769c1d'
responses = {
('10.1.1.1', 10000, 'swiftconfmd5'):
json.dumps({'/etc/swift/swift.conf': cksum}),
('10.2.2.2', 10000, 'swiftconfmd5'):
json.dumps({'/etc/swift/swift.conf': 'bogus'})}
printed = []
with self.mock_responses(responses):
with mock.patch.object(self.recon, '_md5_file', lambda _: cksum):
self.recon.get_swiftconfmd5(hosts, printfn=printed.append)
output = '\n'.join(printed) + '\n'
self.assertTrue("1/2 hosts matched" in output)
self.assertTrue("http://10.2.2.2:10000/recon/swiftconfmd5 (bogus) "
"doesn't match on disk md5sum" in output)

11
test/unit/common/middleware/test_recon.py
View File

@ -161,6 +161,9 @@ class FakeRecon(object):
def fake_ringmd5(self): def fake_ringmd5(self):
return {'ringmd5test': "1"} return {'ringmd5test': "1"}
def fake_swiftconfmd5(self):
return {'/etc/swift/swift.conf': "abcdef"}
def fake_quarantined(self): def fake_quarantined(self):
return {'quarantinedtest': "1"} return {'quarantinedtest': "1"}
@ -725,6 +728,7 @@ class TestReconMiddleware(unittest.TestCase):
self.app.get_unmounted = self.frecon.fake_unmounted self.app.get_unmounted = self.frecon.fake_unmounted
self.app.get_diskusage = self.frecon.fake_diskusage self.app.get_diskusage = self.frecon.fake_diskusage
self.app.get_ring_md5 = self.frecon.fake_ringmd5 self.app.get_ring_md5 = self.frecon.fake_ringmd5
self.app.get_swift_conf_md5 = self.frecon.fake_swiftconfmd5
self.app.get_quarantine_count = self.frecon.fake_quarantined self.app.get_quarantine_count = self.frecon.fake_quarantined
self.app.get_socket_info = self.frecon.fake_sockstat self.app.get_socket_info = self.frecon.fake_sockstat
@ -916,6 +920,13 @@ class TestReconMiddleware(unittest.TestCase):
resp = self.app(req.environ, start_response) resp = self.app(req.environ, start_response)
self.assertEquals(resp, get_ringmd5_resp) self.assertEquals(resp, get_ringmd5_resp)
def test_recon_get_swiftconfmd5(self):
get_swiftconfmd5_resp = ['{"/etc/swift/swift.conf": "abcdef"}']
req = Request.blank('/recon/swiftconfmd5',
environ={'REQUEST_METHOD': 'GET'})
resp = self.app(req.environ, start_response)
self.assertEquals(resp, get_swiftconfmd5_resp)
def test_recon_get_quarantined(self): def test_recon_get_quarantined(self):
get_quarantined_resp = ['{"quarantinedtest": "1"}'] get_quarantined_resp = ['{"quarantinedtest": "1"}']
req = Request.blank('/recon/quarantined', req = Request.blank('/recon/quarantined',