openstack/swift
Code Issues Proposed changes

Merge "Show each policy's information on quarantined files in recon"

Browse Source
This commit is contained in:
Jenkins 2015-02-13 00:17:53 +00:00 committed by Gerrit Code Review
commit 947f979dee
4 changed files with 84 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
swift/cli/recon.py
View File

@ -692,6 +692,7 @@ class SwiftRecon(object):
objq = {}
conq = {}
acctq = {}
stats = {}
recon = Scout("quarantined", self.verbose, self.suppress_errors,
self.timeout)
print("[%s] Checking quarantine" % self._ptime())
@ -700,7 +701,12 @@ class SwiftRecon(object):
objq[url] = response['objects']
conq[url] = response['containers']
acctq[url] = response['accounts']
stats = {"objects": objq, "containers": conq, "accounts": acctq}
if response['policies']:
for key in response['policies']:
pkey = "objects_%s" % key
stats.setdefault(pkey, {})
stats[pkey][url] = response['policies'][key]['objects']
stats.update({"objects": objq, "containers": conq, "accounts": acctq})
for item in stats:
if len(stats[item]) > 0:
computed = self._gen_stats(stats[item].values(),

23
swift/common/middleware/recon.py
View File

@ -266,15 +266,28 @@ class ReconMiddleware(object):
def get_quarantine_count(self):
"""get obj/container/account quarantine counts"""
qcounts = {"objects": 0, "containers": 0, "accounts": 0}
qcounts = {"objects": 0, "containers": 0, "accounts": 0,
"policies": {}}
qdir = "quarantined"
for device in os.listdir(self.devices):
for qtype in qcounts:
qtgt = os.path.join(self.devices, device, qdir, qtype)
if os.path.exists(qtgt):
qpath = os.path.join(self.devices, device, qdir)
if os.path.exists(qpath):
for qtype in os.listdir(qpath):
qtgt = os.path.join(qpath, qtype)
linkcount = os.lstat(qtgt).st_nlink
if linkcount > 2:
qcounts[qtype] += linkcount - 2
if qtype.startswith('objects'):
if '-' in qtype:
pkey = qtype.split('-', 1)[1]
else:
pkey = '0'
qcounts['policies'].setdefault(pkey,
{'objects': 0})
qcounts['policies'][pkey]['objects'] \
+= linkcount - 2
qcounts['objects'] += linkcount - 2
else:
qcounts[qtype] += linkcount - 2
return qcounts
def get_socket_info(self, openr=open):

50
test/unit/cli/test_recon.py
View File

@ -18,6 +18,7 @@ import json
import mock
import os
import random
import re
import string
from StringIO import StringIO
import tempfile
@ -211,6 +212,55 @@ class TestRecon(unittest.TestCase):
for ring in ('account', 'container', 'object', 'object-1'):
os.remove(os.path.join(self.swift_dir, "%s.ring.gz" % ring))
def test_quarantine_check(self):
hosts = [('127.0.0.1', 6010), ('127.0.0.1', 6020),
('127.0.0.1', 6030), ('127.0.0.1', 6040)]
# sample json response from http://<host>:<port>/recon/quarantined
responses = {6010: {'accounts': 0, 'containers': 0, 'objects': 1,
'policies': {'0': {'objects': 0},
'1': {'objects': 1}}},
6020: {'accounts': 1, 'containers': 1, 'objects': 3,
'policies': {'0': {'objects': 1},
'1': {'objects': 2}}},
6030: {'accounts': 2, 'containers': 2, 'objects': 5,
'policies': {'0': {'objects': 2},
'1': {'objects': 3}}},
6040: {'accounts': 3, 'containers': 3, 'objects': 7,
'policies': {'0': {'objects': 3},
'1': {'objects': 4}}}}
# <low> <high> <avg> <total> <Failed> <no_result> <reported>
expected = {'objects_0': (0, 3, 1.5, 6, 0.0, 0, 4),
'objects_1': (1, 4, 2.5, 10, 0.0, 0, 4),
'objects': (1, 7, 4.0, 16, 0.0, 0, 4),
'accounts': (0, 3, 1.5, 6, 0.0, 0, 4),
'containers': (0, 3, 1.5, 6, 0.0, 0, 4)}
def mock_scout_quarantine(app, host):
url = 'http://%s:%s/recon/quarantined' % host
response = responses[host[1]]
status = 200
return url, response, status
stdout = StringIO()
patches = [
mock.patch('swift.cli.recon.Scout.scout', mock_scout_quarantine),
mock.patch('sys.stdout', new=stdout),
]
with nested(*patches):
self.recon_instance.quarantine_check(hosts)
output = stdout.getvalue()
r = re.compile("\[quarantined_(.*)\](.*)")
for line in output.splitlines():
m = r.match(line)
if m:
ex = expected.pop(m.group(1))
self.assertEquals(m.group(2),
" low: %s, high: %s, avg: %s, total: %s,"
" Failed: %s%%, no_result: %s, reported: %s"
% ex)
self.assertFalse(expected)
class TestReconCommands(unittest.TestCase):
def setUp(self):

12
test/unit/common/middleware/test_recon.py
View File

@ -794,7 +794,7 @@ class TestReconSuccess(TestCase):
self.assertEquals(rv, du_resp)
def test_get_quarantine_count(self):
self.mockos.ls_output = ['sda']
dirs = [['sda'], ['accounts', 'containers', 'objects', 'objects-1']]
self.mockos.ismount_output = True
def fake_lstat(*args, **kwargs):
@ -806,10 +806,16 @@ class TestReconSuccess(TestCase):
def fake_exists(*args, **kwargs):
return True
def fake_listdir(*args, **kwargs):
return dirs.pop(0)
with mock.patch("os.lstat", fake_lstat):
with mock.patch("os.path.exists", fake_exists):
rv = self.app.get_quarantine_count()
self.assertEquals(rv, {'objects': 2, 'accounts': 2, 'containers': 2})
with mock.patch("os.listdir", fake_listdir):
rv = self.app.get_quarantine_count()
self.assertEquals(rv, {'objects': 4, 'accounts': 2, 'policies':
{'1': {'objects': 2}, '0': {'objects': 2}},
'containers': 2})
def test_get_socket_info(self):
sockstat_content = ['sockets: used 271',