Shuffle download order (of containers and objects)

When downloading the same containers or objects with multiple
invocations of the swift command-line client, you'll get better
throughput and avoid "hot spots" if each client randomizes its download
order.

Note that the marker must be picked *before* shuffling the containers or
objects.

Change-Id: I7240eda57a80e1708c2483827c6329fd57d5fc51
This commit is contained in:
Darrell Bishop 2012-08-16 21:39:00 -07:00
parent 99b8253450
commit 9083beb41f

@ -22,6 +22,7 @@ from optparse import OptionParser
from os import environ, listdir, makedirs, utime, _exit as os_exit from os import environ, listdir, makedirs, utime, _exit as os_exit
from os.path import basename, dirname, getmtime, getsize, isdir, join from os.path import basename, dirname, getmtime, getsize, isdir, join
from Queue import Empty, Queue from Queue import Empty, Queue
from random import shuffle
from sys import argv, exc_info, exit, stderr, stdout from sys import argv, exc_info, exit, stderr, stdout
from threading import current_thread, enumerate as threading_enumerate, Thread from threading import current_thread, enumerate as threading_enumerate, Thread
from time import sleep from time import sleep
@ -423,9 +424,10 @@ def st_download(parser, args, print_queue, error_queue):
conn.get_container(container, marker=marker)[1]] conn.get_container(container, marker=marker)[1]]
if not objects: if not objects:
break break
marker = objects[-1]
shuffle(objects)
for obj in objects: for obj in objects:
object_queue.put((container, obj)) object_queue.put((container, obj))
marker = objects[-1]
except ClientException, err: except ClientException, err:
if err.http_status != 404: if err.http_status != 404:
raise raise
@ -450,9 +452,10 @@ def st_download(parser, args, print_queue, error_queue):
for c in conn.get_account(marker=marker)[1]] for c in conn.get_account(marker=marker)[1]]
if not containers: if not containers:
break break
marker = containers[-1]
shuffle(containers)
for container in containers: for container in containers:
container_queue.put(container) container_queue.put(container)
marker = containers[-1]
except ClientException, err: except ClientException, err:
if err.http_status != 404: if err.http_status != 404:
raise raise