openstack/swift
Code Issues Proposed changes

Merge "Adds storage policy option to recon"

Browse Source
This commit is contained in:
Jenkins
2016-03-07 20:31:10 +00:00
committed by Gerrit Code Review
parent 6cfb8557a8 81a4355c2d
commit ebdff38553
2 changed files with 197 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
swift/cli/recon.py
View File

@@ -21,6 +21,7 @@ from eventlet.green import urllib2, socket
from six.moves.urllib.parse import urlparse
from swift.common.utils import SWIFT_CONF_FILE
from swift.common.ring import Ring
from swift.common.storage_policy import POLICIES
from hashlib import md5
import eventlet
import json
@@ -202,18 +203,19 @@ class SwiftRecon(object):
block = f.read(4096)
return md5sum.hexdigest()
def get_devices(self, region_filter, zone_filter, swift_dir, ring_name):
def get_hosts(self, region_filter, zone_filter, swift_dir, ring_names):
"""
Get a list of hosts in the ring
Get a list of hosts in the rings.
:param region_filter: Only list regions matching given filter
:param zone_filter: Only list zones matching given filter
:param swift_dir: Directory of swift config, usually /etc/swift
:param ring_name: Name of the ring, such as 'object'
:param ring_names: Collection of ring names, such as
['object', 'object-2']
:returns: a set of tuples containing the ip and port of hosts
"""
ring_data = Ring(swift_dir, ring_name=ring_name)
devs = [d for d in ring_data.devs if d]
rings = [Ring(swift_dir, ring_name=n) for n in ring_names]
devs = [d for r in rings for d in r.devs if d]
if region_filter is not None:
devs = [d for d in devs if d['region'] == region_filter]
if zone_filter is not None:
@@ -913,6 +915,26 @@ class SwiftRecon(object):
matches, len(hosts), errors))
print("=" * 79)
def _get_ring_names(self, policy=None):
'''
Retrieve name of ring files.
If no policy is passed and the server type is object,
the ring names of all storage-policies are retrieved.
:param policy: name or index of storage policy, only applicable
with server_type==object.
:returns: list of ring names.
'''
if self.server_type == 'object':
ring_names = [p.ring_name for p in POLICIES if (
p.name == policy or not policy or (
policy.isdigit() and int(policy) == int(p)))]
else:
ring_names = [self.server_type]
return ring_names
def main(self):
"""
Retrieve and report cluster info from hosts running recon middleware.
@@ -982,6 +1004,9 @@ class SwiftRecon(object):
default=5)
args.add_option('--swiftdir', default="/etc/swift",
help="Default = /etc/swift")
args.add_option('--policy', '-p',
help='Only query object servers in specified '
'storage policy (specified as name or index).')
options, arguments = args.parse_args()
if len(sys.argv) <= 1 or len(arguments) > 1:
@@ -1003,8 +1028,14 @@ class SwiftRecon(object):
self.suppress_errors = options.suppress
self.timeout = options.timeout
hosts = self.get_devices(options.region, options.zone,
swift_dir, self.server_type)
ring_names = self._get_ring_names(options.policy)
if not ring_names:
print('Invalid Storage Policy')
args.print_help()
sys.exit(0)
hosts = self.get_hosts(options.region, options.zone,
swift_dir, ring_names)
print("--> Starting reconnaissance on %s hosts" % len(hosts))
print("=" * 79)

195
test/unit/cli/test_recon.py
View File

@@ -16,12 +16,12 @@
import json
import mock
import os
import random
import re
import string
import tempfile
import time
import unittest
import shutil
import sys
from eventlet.green import urllib2, socket
from six import StringIO
@@ -30,6 +30,9 @@ from six.moves import urllib
from swift.cli import recon
from swift.common import utils
from swift.common.ring import builder
from swift.common.ring import utils as ring_utils
from swift.common.storage_policy import StoragePolicy, POLICIES
from test.unit import patch_policies
class TestHelpers(unittest.TestCase):
@@ -135,22 +138,50 @@ class TestScout(unittest.TestCase):
self.assertEqual(status, -1)
@patch_policies
class TestRecon(unittest.TestCase):
def setUp(self, *_args, **_kwargs):
self.recon_instance = recon.SwiftRecon()
self.swift_dir = tempfile.gettempdir()
self.ring_name = "test_object_%s" % (
''.join(random.choice(string.digits) for x in range(6)))
self.tmpfile_name = "%s/%s.ring.gz" % (self.swift_dir, self.ring_name)
self.swift_dir = tempfile.mkdtemp()
self.ring_name = POLICIES.legacy.ring_name
self.tmpfile_name = os.path.join(
self.swift_dir, self.ring_name + '.ring.gz')
self.ring_name2 = POLICIES[1].ring_name
self.tmpfile_name2 = os.path.join(
self.swift_dir, self.ring_name2 + '.ring.gz')
utils.HASH_PATH_SUFFIX = 'endcap'
utils.HASH_PATH_PREFIX = 'startcap'
def tearDown(self, *_args, **_kwargs):
try:
os.remove(self.tmpfile_name)
except OSError:
pass
shutil.rmtree(self.swift_dir, ignore_errors=True)
def _make_object_rings(self):
ringbuilder = builder.RingBuilder(2, 3, 1)
devs = [
'r0z0-127.0.0.1:10000/sda1',
'r0z1-127.0.0.1:10001/sda1',
'r1z0-127.0.0.1:10002/sda1',
'r1z1-127.0.0.1:10003/sda1',
]
for raw_dev_str in devs:
dev = ring_utils.parse_add_value(raw_dev_str)
dev['weight'] = 1.0
ringbuilder.add_dev(dev)
ringbuilder.rebalance()
ringbuilder.get_ring().save(self.tmpfile_name)
ringbuilder = builder.RingBuilder(2, 2, 1)
devs = [
'r0z0-127.0.0.1:10000/sda1',
'r0z1-127.0.0.2:10004/sda1',
]
for raw_dev_str in devs:
dev = ring_utils.parse_add_value(raw_dev_str)
dev['weight'] = 1.0
ringbuilder.add_dev(dev)
ringbuilder.rebalance()
ringbuilder.get_ring().save(self.tmpfile_name2)
def test_gen_stats(self):
stats = self.recon_instance._gen_stats((1, 4, 10, None), 'Sample')
@@ -176,47 +207,56 @@ class TestRecon(unittest.TestCase):
self.assertEqual(timestamp2, "2013-12-17 10:00:00")
mock_gmtime.assert_called_with()
def test_get_devices(self):
ringbuilder = builder.RingBuilder(2, 3, 1)
ringbuilder.add_dev({'id': 0, 'zone': 0, 'weight': 1,
'ip': '127.0.0.1', 'port': 10000,
'device': 'sda1', 'region': 0})
ringbuilder.add_dev({'id': 1, 'zone': 1, 'weight': 1,
'ip': '127.0.0.1', 'port': 10001,
'device': 'sda1', 'region': 0})
ringbuilder.add_dev({'id': 2, 'zone': 0, 'weight': 1,
'ip': '127.0.0.1', 'port': 10002,
'device': 'sda1', 'region': 1})
ringbuilder.add_dev({'id': 3, 'zone': 1, 'weight': 1,
'ip': '127.0.0.1', 'port': 10003,
'device': 'sda1', 'region': 1})
ringbuilder.rebalance()
ringbuilder.get_ring().save(self.tmpfile_name)
def test_get_hosts(self):
self._make_object_rings()
ips = self.recon_instance.get_devices(
None, None, self.swift_dir, self.ring_name)
ips = self.recon_instance.get_hosts(
None, None, self.swift_dir, [self.ring_name])
self.assertEqual(
set([('127.0.0.1', 10000), ('127.0.0.1', 10001),
('127.0.0.1', 10002), ('127.0.0.1', 10003)]), ips)
ips = self.recon_instance.get_devices(
0, None, self.swift_dir, self.ring_name)
ips = self.recon_instance.get_hosts(
0, None, self.swift_dir, [self.ring_name])
self.assertEqual(
set([('127.0.0.1', 10000), ('127.0.0.1', 10001)]), ips)
ips = self.recon_instance.get_devices(
1, None, self.swift_dir, self.ring_name)
ips = self.recon_instance.get_hosts(
1, None, self.swift_dir, [self.ring_name])
self.assertEqual(
set([('127.0.0.1', 10002), ('127.0.0.1', 10003)]), ips)
ips = self.recon_instance.get_devices(
0, 0, self.swift_dir, self.ring_name)
ips = self.recon_instance.get_hosts(
0, 0, self.swift_dir, [self.ring_name])
self.assertEqual(set([('127.0.0.1', 10000)]), ips)
ips = self.recon_instance.get_devices(
1, 1, self.swift_dir, self.ring_name)
ips = self.recon_instance.get_hosts(
1, 1, self.swift_dir, [self.ring_name])
self.assertEqual(set([('127.0.0.1', 10003)]), ips)
ips = self.recon_instance.get_hosts(
None, None, self.swift_dir, [self.ring_name, self.ring_name2])
self.assertEqual(
set([('127.0.0.1', 10000), ('127.0.0.1', 10001),
('127.0.0.1', 10002), ('127.0.0.1', 10003),
('127.0.0.2', 10004)]), ips)
ips = self.recon_instance.get_hosts(
0, None, self.swift_dir, [self.ring_name, self.ring_name2])
self.assertEqual(
set([('127.0.0.1', 10000), ('127.0.0.1', 10001),
('127.0.0.2', 10004)]), ips)
ips = self.recon_instance.get_hosts(
1, None, self.swift_dir, [self.ring_name, self.ring_name2])
self.assertEqual(
set([('127.0.0.1', 10002), ('127.0.0.1', 10003)]), ips)
ips = self.recon_instance.get_hosts(
0, 1, self.swift_dir, [self.ring_name, self.ring_name2])
self.assertEqual(set([('127.0.0.1', 10001),
('127.0.0.2', 10004)]), ips)
def test_get_ringmd5(self):
for server_type in ('account', 'container', 'object', 'object-1'):
ring_name = '%s.ring.gz' % server_type
@@ -343,6 +383,89 @@ class TestRecon(unittest.TestCase):
" Failed: %s%%, no_result: %s, reported: %s"
% expected)
def test_get_ring_names(self):
self.recon_instance.server_type = 'not-object'
self.assertEqual(self.recon_instance._get_ring_names(), ['not-object'])
self.recon_instance.server_type = 'object'
with patch_policies([StoragePolicy(0, 'zero', is_default=True)]):
self.assertEqual(self.recon_instance._get_ring_names(),
['object'])
with patch_policies([StoragePolicy(0, 'zero', is_default=True),
StoragePolicy(1, 'one')]):
self.assertEqual(self.recon_instance._get_ring_names(),
['object', 'object-1'])
self.assertEqual(self.recon_instance._get_ring_names('0'),
['object'])
self.assertEqual(self.recon_instance._get_ring_names('zero'),
['object'])
self.assertEqual(self.recon_instance._get_ring_names('1'),
['object-1'])
self.assertEqual(self.recon_instance._get_ring_names('one'),
['object-1'])
self.assertEqual(self.recon_instance._get_ring_names('3'), [])
self.assertEqual(self.recon_instance._get_ring_names('wrong'),
[])
def test_main_object_hosts_default_all_policies(self):
self._make_object_rings()
discovered_hosts = set()
def server_type_check(hosts):
for h in hosts:
discovered_hosts.add(h)
self.recon_instance.server_type_check = server_type_check
with mock.patch.object(sys, 'argv', [
"prog", "object", "--swiftdir=%s" % self.swift_dir,
"--validate-servers"]):
self.recon_instance.main()
expected = set([
('127.0.0.1', 10000),
('127.0.0.1', 10001),
('127.0.0.1', 10002),
('127.0.0.1', 10003),
('127.0.0.2', 10004),
])
self.assertEqual(expected, discovered_hosts)
def test_main_object_hosts_default_unu(self):
self._make_object_rings()
discovered_hosts = set()
def server_type_check(hosts):
for h in hosts:
discovered_hosts.add(h)
self.recon_instance.server_type_check = server_type_check
with mock.patch.object(sys, 'argv', [
"prog", "object", "--swiftdir=%s" % self.swift_dir,
"--validate-servers", '--policy=unu']):
self.recon_instance.main()
expected = set([
('127.0.0.1', 10000),
('127.0.0.2', 10004),
])
self.assertEqual(expected, discovered_hosts)
def test_main_object_hosts_default_invalid(self):
self._make_object_rings()
stdout = StringIO()
with mock.patch.object(sys, 'argv', [
"prog", "object", "--swiftdir=%s" % self.swift_dir,
"--validate-servers", '--policy=invalid']),\
mock.patch('sys.stdout', stdout):
self.assertRaises(SystemExit, recon.main)
self.assertIn('Invalid Storage Policy', stdout.getvalue())
class TestReconCommands(unittest.TestCase):
def setUp(self):