Shuffle download order (of containers and objects)

When downloading the same containers or objects with multiple
invocations of the swift command-line client, you'll get better
throughput and avoid "hot spots" if each client randomizes its download
order.

Note that the marker must be picked *before* shuffling the containers or
objects.

Change-Id: I7240eda57a80e1708c2483827c6329fd57d5fc51
This commit is contained in:
Darrell Bishop 2012-08-16 21:39:00 -07:00
parent 99b8253450
commit 9083beb41f

@ -22,6 +22,7 @@ from optparse import OptionParser
from os import environ, listdir, makedirs, utime, _exit as os_exit
from os.path import basename, dirname, getmtime, getsize, isdir, join
from Queue import Empty, Queue
from random import shuffle
from sys import argv, exc_info, exit, stderr, stdout
from threading import current_thread, enumerate as threading_enumerate, Thread
from time import sleep
@ -423,9 +424,10 @@ def st_download(parser, args, print_queue, error_queue):
conn.get_container(container, marker=marker)[1]]
if not objects:
break
marker = objects[-1]
shuffle(objects)
for obj in objects:
object_queue.put((container, obj))
marker = objects[-1]
except ClientException, err:
if err.http_status != 404:
raise
@ -450,9 +452,10 @@ def st_download(parser, args, print_queue, error_queue):
for c in conn.get_account(marker=marker)[1]]
if not containers:
break
marker = containers[-1]
shuffle(containers)
for container in containers:
container_queue.put(container)
marker = containers[-1]
except ClientException, err:
if err.http_status != 404:
raise