Add "--no-overlap" option to swift-dispersion populate
This change allows the user to use a "--no-overlap" parameter when running the tool multiple times. It will increase the coverage by whatever is specified in the dispersion_coverage field of the conf file in a manner where existing container/objects are left in place and no partition is populated more than once. Related-Bug: #1233045 Change-Id: I139fed2f4c967ba18d073b7ecd1e946ed4da1271
This commit is contained in:
parent
8d02147d04
commit
4faf170270
|
@ -101,6 +101,10 @@ Usage: %%prog [options] [conf_file]
|
||||||
parser.add_option('--insecure', action='store_true', default=False,
|
parser.add_option('--insecure', action='store_true', default=False,
|
||||||
help='Allow accessing insecure keystone server. '
|
help='Allow accessing insecure keystone server. '
|
||||||
'The keystone\'s certificate will not be verified.')
|
'The keystone\'s certificate will not be verified.')
|
||||||
|
parser.add_option('--no-overlap', action='store_true', default=False,
|
||||||
|
help='No overlap of partitions if running populate \
|
||||||
|
more than once. Will increase coverage by amount shown \
|
||||||
|
in dispersion.conf file')
|
||||||
options, args = parser.parse_args()
|
options, args = parser.parse_args()
|
||||||
|
|
||||||
if args:
|
if args:
|
||||||
|
@ -144,6 +148,19 @@ Usage: %%prog [options] [conf_file]
|
||||||
container_ring = Ring(swift_dir, ring_name='container')
|
container_ring = Ring(swift_dir, ring_name='container')
|
||||||
parts_left = dict((x, x)
|
parts_left = dict((x, x)
|
||||||
for x in xrange(container_ring.partition_count))
|
for x in xrange(container_ring.partition_count))
|
||||||
|
|
||||||
|
if options.no_overlap:
|
||||||
|
with connpool.item() as conn:
|
||||||
|
containers = [cont['name'] for cont in conn.get_account(
|
||||||
|
prefix='dispersion_', full_listing=True)[1]]
|
||||||
|
containers_listed = len(containers)
|
||||||
|
if containers_listed > 0:
|
||||||
|
for container in containers:
|
||||||
|
partition, _junk = container_ring.get_nodes(account,
|
||||||
|
container)
|
||||||
|
if partition in parts_left:
|
||||||
|
del parts_left[partition]
|
||||||
|
|
||||||
item_type = 'containers'
|
item_type = 'containers'
|
||||||
created = 0
|
created = 0
|
||||||
retries_done = 0
|
retries_done = 0
|
||||||
|
@ -152,9 +169,9 @@ Usage: %%prog [options] [conf_file]
|
||||||
begun = next_report = time()
|
begun = next_report = time()
|
||||||
next_report += 2
|
next_report += 2
|
||||||
suffix = 0
|
suffix = 0
|
||||||
while need_to_queue >= 1:
|
while need_to_queue >= 1 and parts_left:
|
||||||
container = 'dispersion_%d' % suffix
|
container = 'dispersion_%d' % suffix
|
||||||
part, _junk = container_ring.get_nodes(account, container)
|
part = container_ring.get_part(account, container)
|
||||||
if part in parts_left:
|
if part in parts_left:
|
||||||
if suffix >= options.container_suffix_start:
|
if suffix >= options.container_suffix_start:
|
||||||
coropool.spawn(put_container, connpool, container, report)
|
coropool.spawn(put_container, connpool, container, report)
|
||||||
|
@ -168,7 +185,13 @@ Usage: %%prog [options] [conf_file]
|
||||||
elapsed, elapsed_unit = get_time_units(time() - begun)
|
elapsed, elapsed_unit = get_time_units(time() - begun)
|
||||||
print '\r\x1B[KCreated %d containers for dispersion reporting, ' \
|
print '\r\x1B[KCreated %d containers for dispersion reporting, ' \
|
||||||
'%d%s, %d retries' % \
|
'%d%s, %d retries' % \
|
||||||
(need_to_create, round(elapsed), elapsed_unit, retries_done)
|
((need_to_create - need_to_queue), round(elapsed), elapsed_unit,
|
||||||
|
retries_done)
|
||||||
|
if options.no_overlap:
|
||||||
|
con_coverage = container_ring.partition_count - len(parts_left)
|
||||||
|
print '\r\x1B[KTotal container coverage is now %.2f%%.' % \
|
||||||
|
((float(con_coverage) / container_ring.partition_count
|
||||||
|
* 100))
|
||||||
stdout.flush()
|
stdout.flush()
|
||||||
|
|
||||||
if object_populate:
|
if object_populate:
|
||||||
|
@ -176,6 +199,23 @@ Usage: %%prog [options] [conf_file]
|
||||||
put_container(connpool, container, None)
|
put_container(connpool, container, None)
|
||||||
object_ring = Ring(swift_dir, ring_name='object')
|
object_ring = Ring(swift_dir, ring_name='object')
|
||||||
parts_left = dict((x, x) for x in xrange(object_ring.partition_count))
|
parts_left = dict((x, x) for x in xrange(object_ring.partition_count))
|
||||||
|
|
||||||
|
if options.no_overlap:
|
||||||
|
with connpool.item() as conn:
|
||||||
|
obj_container = [cont_b['name'] for cont_b in conn.get_account(
|
||||||
|
prefix=container, full_listing=True)[1]]
|
||||||
|
if obj_container:
|
||||||
|
with connpool.item() as conn:
|
||||||
|
objects = [o['name'] for o in
|
||||||
|
conn.get_container(container,
|
||||||
|
prefix='dispersion_',
|
||||||
|
full_listing=True)[1]]
|
||||||
|
for my_object in objects:
|
||||||
|
partition = object_ring.get_part(account, container,
|
||||||
|
my_object)
|
||||||
|
if partition in parts_left:
|
||||||
|
del parts_left[partition]
|
||||||
|
|
||||||
item_type = 'objects'
|
item_type = 'objects'
|
||||||
created = 0
|
created = 0
|
||||||
retries_done = 0
|
retries_done = 0
|
||||||
|
@ -184,9 +224,9 @@ Usage: %%prog [options] [conf_file]
|
||||||
begun = next_report = time()
|
begun = next_report = time()
|
||||||
next_report += 2
|
next_report += 2
|
||||||
suffix = 0
|
suffix = 0
|
||||||
while need_to_queue >= 1:
|
while need_to_queue >= 1 and parts_left:
|
||||||
obj = 'dispersion_%d' % suffix
|
obj = 'dispersion_%d' % suffix
|
||||||
part, _junk = object_ring.get_nodes(account, container, obj)
|
part = object_ring.get_part(account, container, obj)
|
||||||
if part in parts_left:
|
if part in parts_left:
|
||||||
if suffix >= options.object_suffix_start:
|
if suffix >= options.object_suffix_start:
|
||||||
coropool.spawn(
|
coropool.spawn(
|
||||||
|
@ -201,5 +241,10 @@ Usage: %%prog [options] [conf_file]
|
||||||
elapsed, elapsed_unit = get_time_units(time() - begun)
|
elapsed, elapsed_unit = get_time_units(time() - begun)
|
||||||
print '\r\x1B[KCreated %d objects for dispersion reporting, ' \
|
print '\r\x1B[KCreated %d objects for dispersion reporting, ' \
|
||||||
'%d%s, %d retries' % \
|
'%d%s, %d retries' % \
|
||||||
(need_to_create, round(elapsed), elapsed_unit, retries_done)
|
((need_to_create - need_to_queue), round(elapsed), elapsed_unit,
|
||||||
|
retries_done)
|
||||||
|
if options.no_overlap:
|
||||||
|
obj_coverage = object_ring.partition_count - len(parts_left)
|
||||||
|
print '\r\x1B[KTotal object coverage is now %.2f%%.' % \
|
||||||
|
((float(obj_coverage) / object_ring.partition_count * 100))
|
||||||
stdout.flush()
|
stdout.flush()
|
||||||
|
|
|
@ -72,6 +72,8 @@ Only run object population
|
||||||
Only run container population
|
Only run container population
|
||||||
.IP "\fB--object-only\fR"
|
.IP "\fB--object-only\fR"
|
||||||
Only run object population
|
Only run object population
|
||||||
|
.IP "\fB--no-overlap\fR"
|
||||||
|
Increase coverage by amount in dispersion_coverage option with no overlap of existing partitions (if run more than once)
|
||||||
|
|
||||||
.SH CONFIGURATION
|
.SH CONFIGURATION
|
||||||
.PD 0
|
.PD 0
|
||||||
|
|
Loading…
Reference in New Issue