Shuffle download order (of containers and objects)
When downloading the same containers or objects with multiple invocations of the swift command-line client, you'll get better throughput and avoid "hot spots" if each client randomizes its download order. Note that the marker must be picked *before* shuffling the containers or objects. Change-Id: I7240eda57a80e1708c2483827c6329fd57d5fc51
This commit is contained in:
parent
99b8253450
commit
9083beb41f
@ -22,6 +22,7 @@ from optparse import OptionParser
|
|||||||
from os import environ, listdir, makedirs, utime, _exit as os_exit
|
from os import environ, listdir, makedirs, utime, _exit as os_exit
|
||||||
from os.path import basename, dirname, getmtime, getsize, isdir, join
|
from os.path import basename, dirname, getmtime, getsize, isdir, join
|
||||||
from Queue import Empty, Queue
|
from Queue import Empty, Queue
|
||||||
|
from random import shuffle
|
||||||
from sys import argv, exc_info, exit, stderr, stdout
|
from sys import argv, exc_info, exit, stderr, stdout
|
||||||
from threading import current_thread, enumerate as threading_enumerate, Thread
|
from threading import current_thread, enumerate as threading_enumerate, Thread
|
||||||
from time import sleep
|
from time import sleep
|
||||||
@ -423,9 +424,10 @@ def st_download(parser, args, print_queue, error_queue):
|
|||||||
conn.get_container(container, marker=marker)[1]]
|
conn.get_container(container, marker=marker)[1]]
|
||||||
if not objects:
|
if not objects:
|
||||||
break
|
break
|
||||||
|
marker = objects[-1]
|
||||||
|
shuffle(objects)
|
||||||
for obj in objects:
|
for obj in objects:
|
||||||
object_queue.put((container, obj))
|
object_queue.put((container, obj))
|
||||||
marker = objects[-1]
|
|
||||||
except ClientException, err:
|
except ClientException, err:
|
||||||
if err.http_status != 404:
|
if err.http_status != 404:
|
||||||
raise
|
raise
|
||||||
@ -450,9 +452,10 @@ def st_download(parser, args, print_queue, error_queue):
|
|||||||
for c in conn.get_account(marker=marker)[1]]
|
for c in conn.get_account(marker=marker)[1]]
|
||||||
if not containers:
|
if not containers:
|
||||||
break
|
break
|
||||||
|
marker = containers[-1]
|
||||||
|
shuffle(containers)
|
||||||
for container in containers:
|
for container in containers:
|
||||||
container_queue.put(container)
|
container_queue.put(container)
|
||||||
marker = containers[-1]
|
|
||||||
except ClientException, err:
|
except ClientException, err:
|
||||||
if err.http_status != 404:
|
if err.http_status != 404:
|
||||||
raise
|
raise
|
||||||
|
Loading…
x
Reference in New Issue
Block a user