Merge "Shuffle download order (of containers and objects)"

This commit is contained in:
Jenkins
2012-08-17 22:41:58 +00:00
committed by Gerrit Code Review

View File

@@ -22,6 +22,7 @@ from optparse import OptionParser
from os import environ, listdir, makedirs, utime, _exit as os_exit from os import environ, listdir, makedirs, utime, _exit as os_exit
from os.path import basename, dirname, getmtime, getsize, isdir, join from os.path import basename, dirname, getmtime, getsize, isdir, join
from Queue import Empty, Queue from Queue import Empty, Queue
from random import shuffle
from sys import argv, exc_info, exit, stderr, stdout from sys import argv, exc_info, exit, stderr, stdout
from threading import current_thread, enumerate as threading_enumerate, Thread from threading import current_thread, enumerate as threading_enumerate, Thread
from time import sleep, time from time import sleep, time
@@ -429,9 +430,10 @@ def st_download(parser, args, print_queue, error_queue):
conn.get_container(container, marker=marker)[1]] conn.get_container(container, marker=marker)[1]]
if not objects: if not objects:
break break
marker = objects[-1]
shuffle(objects)
for obj in objects: for obj in objects:
object_queue.put((container, obj)) object_queue.put((container, obj))
marker = objects[-1]
except ClientException, err: except ClientException, err:
if err.http_status != 404: if err.http_status != 404:
raise raise
@@ -456,9 +458,10 @@ def st_download(parser, args, print_queue, error_queue):
for c in conn.get_account(marker=marker)[1]] for c in conn.get_account(marker=marker)[1]]
if not containers: if not containers:
break break
marker = containers[-1]
shuffle(containers)
for container in containers: for container in containers:
container_queue.put(container) container_queue.put(container)
marker = containers[-1]
except ClientException, err: except ClientException, err:
if err.http_status != 404: if err.http_status != 404:
raise raise