From 9083beb41fbc0ab8bcbc5a57ded5dc25dfd96b46 Mon Sep 17 00:00:00 2001
From: Darrell Bishop <darrell@swiftstack.com>
Date: Thu, 16 Aug 2012 21:39:00 -0700
Subject: [PATCH] Shuffle download order (of containers and objects)
When downloading the same containers or objects with multiple
invocations of the swift command-line client, you'll get better
throughput and avoid "hot spots" if each client randomizes its download
order.
Note that the marker must be picked *before* shuffling the containers or
objects.
Change-Id: I7240eda57a80e1708c2483827c6329fd57d5fc51
---
bin/swift | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/bin/swift b/bin/swift
index 11f9553a..b1985940 100755
--- a/bin/swift
+++ b/bin/swift
@@ -22,6 +22,7 @@ from optparse import OptionParser
from os import environ, listdir, makedirs, utime, _exit as os_exit
from os.path import basename, dirname, getmtime, getsize, isdir, join
from Queue import Empty, Queue
+from random import shuffle
from sys import argv, exc_info, exit, stderr, stdout
from threading import current_thread, enumerate as threading_enumerate, Thread
from time import sleep
@@ -423,9 +424,10 @@ def st_download(parser, args, print_queue, error_queue):
conn.get_container(container, marker=marker)[1]]
if not objects:
break
+ marker = objects[-1]
+ shuffle(objects)
for obj in objects:
object_queue.put((container, obj))
- marker = objects[-1]
except ClientException, err:
if err.http_status != 404:
raise
@@ -450,9 +452,10 @@ def st_download(parser, args, print_queue, error_queue):
for c in conn.get_account(marker=marker)[1]]
if not containers:
break
+ marker = containers[-1]
+ shuffle(containers)
for container in containers:
container_queue.put(container)
- marker = containers[-1]
except ClientException, err:
if err.http_status != 404:
raise