openstack/deb-swift
Code Issues Proposed changes

Merge "Add policy support to dispersion tools"

Browse Source
This commit is contained in:
Jenkins 2015-06-25 09:42:00 +00:00 committed by Gerrit Code Review
commit f5d6f8c23a
3 changed files with 60 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
bin/swift-dispersion-populate
View File

@ -31,16 +31,17 @@ except ImportError:
from swift.common.internal_client import SimpleClient
from swift.common.ring import Ring
from swift.common.utils import compute_eta, get_time_units, config_true_value
from swift.common.storage_policy import POLICIES
insecure = False
def put_container(connpool, container, report):
def put_container(connpool, container, report, headers):
global retries_done
try:
with connpool.item() as conn:
conn.put_container(container)
conn.put_container(container, headers=headers)
retries_done += conn.attempts - 1
if report:
report(True)
@ -105,6 +106,9 @@ Usage: %%prog [options] [conf_file]
help='No overlap of partitions if running populate \
more than once. Will increase coverage by amount shown \
in dispersion.conf file')
parser.add_option('-P', '--policy-name', dest='policy_name',
help="Specify storage policy name")
options, args = parser.parse_args()
if args:
@ -114,6 +118,15 @@ Usage: %%prog [options] [conf_file]
if not c.read(conffile):
exit('Unable to read config file: %s' % conffile)
conf = dict(c.items('dispersion'))
if options.policy_name is None:
policy = POLICIES.default
else:
policy = POLICIES.get_by_name(options.policy_name)
if policy is None:
exit('Unable to find policy: %s' % options.policy_name)
print 'Using storage policy: %s ' % policy.name
swift_dir = conf.get('swift_dir', '/etc/swift')
dispersion_coverage = float(conf.get('dispersion_coverage', 1))
retries = int(conf.get('retries', 5))
@ -141,6 +154,8 @@ Usage: %%prog [options] [conf_file]
insecure=insecure)
account = url.rsplit('/', 1)[1]
connpool = Pool(max_size=concurrency)
headers = {}
headers['X-Storage-Policy'] = policy.name
connpool.create = lambda: SimpleClient(
url=url, token=token, retries=retries)
@ -152,7 +167,7 @@ Usage: %%prog [options] [conf_file]
if options.no_overlap:
with connpool.item() as conn:
containers = [cont['name'] for cont in conn.get_account(
prefix='dispersion_', full_listing=True)[1]]
prefix='dispersion_%d' % policy.idx, full_listing=True)[1]]
containers_listed = len(containers)
if containers_listed > 0:
for container in containers:
@ -170,11 +185,12 @@ Usage: %%prog [options] [conf_file]
next_report += 2
suffix = 0
while need_to_queue >= 1 and parts_left:
container = 'dispersion_%d' % suffix
container = 'dispersion_%d_%d' % (policy.idx, suffix)
part = container_ring.get_part(account, container)
if part in parts_left:
if suffix >= options.container_suffix_start:
coropool.spawn(put_container, connpool, container, report)
coropool.spawn(put_container, connpool, container, report,
headers)
sleep()
else:
report(True)
@ -195,9 +211,9 @@ Usage: %%prog [options] [conf_file]
stdout.flush()
if object_populate:
container = 'dispersion_objects'
put_container(connpool, container, None)
object_ring = Ring(swift_dir, ring_name='object')
container = 'dispersion_objects_%d' % policy.idx
put_container(connpool, container, None, headers)
object_ring = Ring(swift_dir, ring_name=policy.ring_name)
parts_left = dict((x, x) for x in xrange(object_ring.partition_count))
if options.no_overlap:

41
bin/swift-dispersion-report
View File

@ -36,6 +36,7 @@ from swift.common.internal_client import SimpleClient
from swift.common.ring import Ring
from swift.common.exceptions import ClientException
from swift.common.utils import compute_eta, get_time_units, config_true_value
from swift.common.storage_policy import POLICIES
unmounted = []
@ -73,10 +74,10 @@ def get_error_log(prefix):
def container_dispersion_report(coropool, connpool, account, container_ring,
retries, output_missing_partitions):
retries, output_missing_partitions, policy):
with connpool.item() as conn:
containers = [c['name'] for c in conn.get_account(
prefix='dispersion_', full_listing=True)[1]]
prefix='dispersion_%d' % policy.idx, full_listing=True)[1]]
containers_listed = len(containers)
if not containers_listed:
print >>stderr, 'No containers to query. Has ' \
@ -169,8 +170,8 @@ def container_dispersion_report(coropool, connpool, account, container_ring,
def object_dispersion_report(coropool, connpool, account, object_ring,
retries, output_missing_partitions):
container = 'dispersion_objects'
retries, output_missing_partitions, policy):
container = 'dispersion_objects_%d' % policy.idx
with connpool.item() as conn:
try:
objects = [o['name'] for o in conn.get_container(
@ -196,6 +197,11 @@ def object_dispersion_report(coropool, connpool, account, object_ring,
begun = time()
next_report = [time() + 2]
headers = None
if policy is not None:
headers = {}
headers['X-Backend-Storage-Policy-Index'] = int(policy)
def direct(obj, part, nodes):
found_count = 0
for node in nodes:
@ -203,7 +209,8 @@ def object_dispersion_report(coropool, connpool, account, object_ring,
try:
attempts, _junk = direct_client.retry(
direct_client.direct_head_object, node, part, account,
container, obj, error_log=error_log, retries=retries)
container, obj, error_log=error_log, retries=retries,
headers=headers)
retries_done[0] += attempts - 1
found_count += 1
except ClientException as err:
@ -290,9 +297,9 @@ def missing_string(partition_count, missing_copies, copy_count):
verb_string = 'were'
partition_string = 'partitions'
copy_string = 'copy'
if missing_copies > 1:
copy_string = 'copies'
copy_string = 'copies'
if missing_copies == 1:
copy_string = 'copy'
return '%sThere %s %d %s missing %s %s.' % (
exclamations, verb_string, partition_count, partition_string,
@ -323,6 +330,9 @@ Usage: %%prog [options] [conf_file]
parser.add_option('--insecure', action='store_true', default=False,
help='Allow accessing insecure keystone server. '
'The keystone\'s certificate will not be verified.')
parser.add_option('-P', '--policy-name', dest='policy_name',
help="Specify storage policy name")
options, args = parser.parse_args()
if args:
@ -332,6 +342,15 @@ Usage: %%prog [options] [conf_file]
if not c.read(conffile):
exit('Unable to read config file: %s' % conffile)
conf = dict(c.items('dispersion'))
if options.policy_name is None:
policy = POLICIES.default
else:
policy = POLICIES.get_by_name(options.policy_name)
if policy is None:
exit('Unable to find policy: %s' % options.policy_name)
print 'Using storage policy: %s ' % policy.name
swift_dir = conf.get('swift_dir', '/etc/swift')
retries = int(conf.get('retries', 5))
concurrency = int(conf.get('concurrency', 25))
@ -364,16 +383,16 @@ Usage: %%prog [options] [conf_file]
url=url, token=token, retries=retries)
container_ring = Ring(swift_dir, ring_name='container')
object_ring = Ring(swift_dir, ring_name='object')
object_ring = Ring(swift_dir, ring_name=policy.ring_name)
output = {}
if container_report:
output['container'] = container_dispersion_report(
coropool, connpool, account, container_ring, retries,
options.partitions)
options.partitions, policy)
if object_report:
output['object'] = object_dispersion_report(
coropool, connpool, account, object_ring, retries,
options.partitions)
options.partitions, policy)
if json_output:
print json.dumps(output)

6
doc/source/admin_guide.rst
View File

@ -339,6 +339,12 @@ allows it to be more easily consumed by third party utilities::
$ swift-dispersion-report -j
{"object": {"retries:": 0, "missing_two": 0, "copies_found": 7863, "missing_one": 0, "copies_expected": 7863, "pct_found": 100.0, "overlapping": 0, "missing_all": 0}, "container": {"retries:": 0, "missing_two": 0, "copies_found": 12534, "missing_one": 0, "copies_expected": 12534, "pct_found": 100.0, "overlapping": 15, "missing_all": 0}}
Note that you may select which storage policy to use by setting the option
'--policy-name silver' or '-P silver' (silver is the example policy name here).
If no policy is specified, the default will be used per the swift.conf file.
When you specify a policy the containers created also include the policy index,
thus even when running a container_only report, you will need to specify the
policy not using the default.
-----------------------------------
Geographically Distributed Clusters