Merge "Shuffle download order (of containers and objects)"

This commit is contained in:
Jenkins
2012-08-17 22:41:58 +00:00
committed by Gerrit Code Review

View File

@@ -22,6 +22,7 @@ from optparse import OptionParser
from os import environ, listdir, makedirs, utime, _exit as os_exit
from os.path import basename, dirname, getmtime, getsize, isdir, join
from Queue import Empty, Queue
from random import shuffle
from sys import argv, exc_info, exit, stderr, stdout
from threading import current_thread, enumerate as threading_enumerate, Thread
from time import sleep, time
@@ -429,9 +430,10 @@ def st_download(parser, args, print_queue, error_queue):
conn.get_container(container, marker=marker)[1]]
if not objects:
break
marker = objects[-1]
shuffle(objects)
for obj in objects:
object_queue.put((container, obj))
marker = objects[-1]
except ClientException, err:
if err.http_status != 404:
raise
@@ -456,9 +458,10 @@ def st_download(parser, args, print_queue, error_queue):
for c in conn.get_account(marker=marker)[1]]
if not containers:
break
marker = containers[-1]
shuffle(containers)
for container in containers:
container_queue.put(container)
marker = containers[-1]
except ClientException, err:
if err.http_status != 404:
raise