Browse Source

Merge branch 'master' into feature/crypto

Change-Id: Ie2ad347d20717862f18ef4855190058a85c9a75c
changes/75/249975/1
Alistair Coles 3 years ago
parent
commit
458d5ebb9a
61 changed files with 1452 additions and 303 deletions
  1. 7
    3
      .mailmap
  2. 3
    3
      bin/swift-account-audit
  3. 1
    4
      bin/swift-dispersion-report
  4. 10
    0
      bin/swift-init
  5. 2
    0
      doc/manpages/swift-init.1
  6. 1
    3
      doc/source/api/form_post_middleware.rst
  7. 18
    1
      doc/source/api/object_api_v1_overview.rst
  8. 1
    3
      doc/source/api/temporary_url_middleware.rst
  9. 1
    1
      doc/source/associated_projects.rst
  10. 3
    3
      doc/source/development_saio.rst
  11. 7
    8
      doc/source/howto_installmultinode.rst
  12. 4
    0
      etc/drive-audit.conf-sample
  13. 0
    2
      requirements.txt
  14. 31
    19
      swift/account/backend.py
  15. 2
    1
      swift/account/server.py
  16. 4
    3
      swift/account/utils.py
  17. 2
    1
      swift/common/db.py
  18. 1
    5
      swift/common/direct_client.py
  19. 17
    7
      swift/common/manager.py
  20. 1
    1
      swift/common/memcached.py
  21. 3
    1
      swift/common/middleware/acl.py
  22. 3
    2
      swift/common/middleware/bulk.py
  23. 2
    1
      swift/common/middleware/dlo.py
  24. 6
    22
      swift/common/middleware/keystoneauth.py
  25. 2
    1
      swift/common/middleware/list_endpoints.py
  26. 2
    1
      swift/common/middleware/recon.py
  27. 3
    1
      swift/common/middleware/slo.py
  28. 2
    1
      swift/common/middleware/versioned_writes.py
  29. 2
    1
      swift/common/ring/ring.py
  30. 10
    57
      swift/common/utils.py
  31. 2
    1
      swift/common/wsgi.py
  32. 35
    21
      swift/container/backend.py
  33. 2
    1
      swift/container/replicator.py
  34. 4
    2
      swift/container/server.py
  35. 2
    1
      swift/obj/auditor.py
  36. 0
    5
      swift/proxy/controllers/base.py
  37. 2
    1
      swift/proxy/controllers/info.py
  38. 8
    3
      swift/proxy/controllers/obj.py
  39. 1
    1
      test/functional/swift_test_client.py
  40. 180
    0
      test/functional/tests.py
  41. 210
    0
      test/unit/account/test_backend.py
  42. 103
    13
      test/unit/account/test_server.py
  43. 78
    8
      test/unit/cli/test_ringbuilder.py
  44. 8
    27
      test/unit/common/middleware/test_keystoneauth.py
  45. 8
    7
      test/unit/common/middleware/test_ratelimit.py
  46. 27
    6
      test/unit/common/middleware/test_slo.py
  47. 0
    5
      test/unit/common/test_bufferedhttp.py
  48. 3
    3
      test/unit/common/test_db.py
  49. 5
    5
      test/unit/common/test_db_replicator.py
  50. 108
    2
      test/unit/common/test_direct_client.py
  51. 123
    7
      test/unit/common/test_manager.py
  52. 2
    1
      test/unit/common/test_utils.py
  53. 6
    3
      test/unit/common/test_wsgi.py
  54. 24
    0
      test/unit/container/test_auditor.py
  55. 224
    1
      test/unit/container/test_backend.py
  56. 102
    11
      test/unit/container/test_server.py
  57. 0
    1
      test/unit/proxy/controllers/test_base.py
  58. 24
    8
      test/unit/proxy/controllers/test_obj.py
  59. 2
    1
      test/unit/proxy/test_server.py
  60. 3
    1
      test/unit/test_locale/test_locale.py
  61. 5
    1
      tox.ini

+ 7
- 3
.mailmap View File

@@ -58,7 +58,7 @@ Madhuri Kumari <madhuri.rai07@gmail.com> madhuri <madhuri@madhuri-VirtualBox.(no
58 58
 Morgan Fainberg <morgan.fainberg@gmail.com> <m@metacloud.com>
59 59
 Hua Zhang <zhuadl@cn.ibm.com> <zhuadl@cn.ibm.com>
60 60
 Yummy Bian <yummy.bian@gmail.com> <yummy.bian@gmail.com>
61
-Alistair Coles <alistair.coles@hp.com> <alistair.coles@hp.com>
61
+Alistair Coles <alistair.coles@hpe.com> <alistair.coles@hp.com>
62 62
 Tong Li <litong01@us.ibm.com> <litong01@us.ibm.com>
63 63
 Paul Luse <paul.e.luse@intel.com> <paul.e.luse@intel.com>
64 64
 Yuan Zhou <yuan.zhou@intel.com> <yuan.zhou@intel.com>
@@ -66,9 +66,9 @@ Jola Mirecka <jola.mirecka@hp.com> <jola.mirecka@hp.com>
66 66
 Ning Zhang <ning@zmanda.com> <ning@zmanda.com>
67 67
 Mauro Stettler <mauro.stettler@gmail.com> <mauro.stettler@gmail.com>
68 68
 Pawel Palucki <pawel.palucki@gmail.com> <pawel.palucki@gmail.com>
69
-Guang Yee <guang.yee@hp.com> <guang.yee@hp.com>
69
+Guang Yee <guang.yee@hpe.com> <guang.yee@hp.com>
70 70
 Jing Liuqing <jing.liuqing@99cloud.net> <jing.liuqing@99cloud.net>
71
-Lorcan Browne <lorcan.browne@hp.com> <lorcan.browne@hp.com>
71
+Lorcan Browne <lorcan.browne@hpe.com> <lorcan.browne@hp.com>
72 72
 Eohyung Lee <liquidnuker@gmail.com> <liquid@kt.com>
73 73
 Harshit Chitalia <harshit@acelio.com> <harshit@acelio.com>
74 74
 Richard Hawkins <richard.hawkins@rackspace.com>
@@ -83,3 +83,7 @@ Atsushi Sakai <sakaia@jp.fujitsu.com>
83 83
 Takashi Natsume <natsume.takashi@lab.ntt.co.jp>
84 84
 Nakagawa Masaaki <nakagawamsa@nttdata.co.jp> nakagawamsa
85 85
 Romain Le Disez <romain.ledisez@ovh.net> Romain LE DISEZ
86
+Donagh McCabe <donagh.mccabe@hpe.com> <donagh.mccabe@hp.com>
87
+Eamonn O'Toole <eamonn.otoole@hpe.com> <eamonn.otoole@hp.com>
88
+Gerry Drudy <gerry.drudy@hpe.com> <gerry.drudy@hp.com>
89
+Mark Seger <mark.seger@hpe.com> <mark.seger@hp.com>

+ 3
- 3
bin/swift-account-audit View File

@@ -20,7 +20,7 @@ from hashlib import md5
20 20
 import getopt
21 21
 from itertools import chain
22 22
 
23
-import simplejson
23
+import json
24 24
 from eventlet.greenpool import GreenPool
25 25
 from eventlet.event import Event
26 26
 from six.moves.urllib.parse import quote
@@ -176,7 +176,7 @@ class Auditor(object):
176 176
                         break
177 177
                     if node['id'] not in responses:
178 178
                         responses[node['id']] = dict(resp.getheaders())
179
-                    results = simplejson.loads(resp.read())
179
+                    results = json.loads(resp.read())
180 180
                 except Exception:
181 181
                     self.container_exceptions += 1
182 182
                     consistent = False
@@ -249,7 +249,7 @@ class Auditor(object):
249 249
                               " from %ss:%ss" %
250 250
                               (account, node['ip'], node['device']))
251 251
                         break
252
-                    results = simplejson.loads(resp.read())
252
+                    results = json.loads(resp.read())
253 253
                 except Exception:
254 254
                     self.account_exceptions += 1
255 255
                     consistent = False

+ 1
- 4
bin/swift-dispersion-report View File

@@ -14,15 +14,12 @@
14 14
 # See the License for the specific language governing permissions and
15 15
 # limitations under the License.
16 16
 
17
+import json
17 18
 from collections import defaultdict
18 19
 from six.moves.configparser import ConfigParser
19 20
 from optparse import OptionParser
20 21
 from sys import exit, stdout, stderr
21 22
 from time import time
22
-try:
23
-    import simplejson as json
24
-except ImportError:
25
-    import json
26 23
 
27 24
 from eventlet import GreenPool, hubs, patcher, Timeout
28 25
 from eventlet.pools import Pool

+ 10
- 0
bin/swift-init View File

@@ -50,6 +50,16 @@ def main():
50 50
                       dest="run_dir", default=RUN_DIR,
51 51
                       help="alternative directory to store running pid files "
52 52
                       "default: %s" % RUN_DIR)
53
+    # Changing behaviour if missing config
54
+    parser.add_option('--strict', dest='strict', action='store_true',
55
+                      help="Return non-zero status code if some config is "
56
+                           "missing. Default mode if server is explicitly "
57
+                           "named.")
58
+    # a negative option for strict
59
+    parser.add_option('--non-strict', dest='strict', action='store_false',
60
+                      help="Return zero status code even if some config is "
61
+                           "missing. Default mode if server is one of aliases "
62
+                           "`all`, `main` or `rest`.")
53 63
 
54 64
     options, args = parser.parse_args()
55 65
 

+ 2
- 0
doc/manpages/swift-init.1 View File

@@ -109,6 +109,8 @@ allows one to use the keywords such as "all", "main" and "rest" for the <server>
109 109
 .IP "-c N, --config-num=N \t send command to the Nth server only
110 110
 .IP "-k N, --kill-wait=N \t wait N seconds for processes to die (default 15)
111 111
 .IP "-r RUN_DIR, --run-dir=RUN_DIR directory where the pids will be stored (default /var/run/swift)
112
+.IP "--strict return non-zero status code if some config is missing. Default mode if server is explicitly named."
113
+.IP "--non-strict return zero status code even if some config is missing. Default mode if server is one of aliases `all`, `main` or `rest`."
112 114
 .PD 
113 115
 .RE
114 116
 

+ 1
- 3
doc/source/api/form_post_middleware.rst View File

@@ -17,9 +17,7 @@ URL middleware uses. For information about how to set these keys, see
17 17
 :ref:`secret_keys`.
18 18
 
19 19
 For information about the form **POST** middleware configuration
20
-options, see `Form
21
-post <http://docs.openstack.org/havana/config-reference/content/object-storage-form-post.html>`__
22
-in the *OpenStack Configuration Reference*.
20
+options, see :ref:`formpost` in the *Source Documentation*.
23 21
 
24 22
 Form POST format
25 23
 ~~~~~~~~~~~~~~~~

+ 18
- 1
doc/source/api/object_api_v1_overview.rst View File

@@ -128,7 +128,24 @@ If you have a large number of containers or objects, you can use query
128 128
 parameters to page through large lists of containers or objects. Use the
129 129
 *``marker``*, *``limit``*, and *``end_marker``* query parameters to
130 130
 control how many items are returned in a list and where the list starts
131
-or ends.
131
+or ends. If you want to page through in reverse order, you can use the query
132
+parameter *``reverse``*, noting that your marker and end_markers will be
133
+applied to a reverse listing should be switched. I.e, for a list of objects
134
+``[a, b, c, d, e]`` the non-reversed could be:
135
+
136
+.. code::
137
+
138
+  /v1/{account}/{container}/?marker=a&end_marker=d
139
+  b
140
+  c
141
+
142
+However, when reversed marker and end_marker are applied to a reversed list:
143
+
144
+.. code::
145
+
146
+  /v1/{account}/{container}/?marker=d&end_marker=a&reverse=on
147
+  c
148
+  b
132 149
 
133 150
 Object Storage HTTP requests have the following default constraints.
134 151
 Your service provider might use different default values.

+ 1
- 3
doc/source/api/temporary_url_middleware.rst View File

@@ -15,9 +15,7 @@ downloads the object directly from Object Storage, eliminating the need
15 15
 for the website to act as a proxy for the request.
16 16
 
17 17
 Ask your cloud administrator to enable the temporary URL feature. For
18
-information, see `Temporary
19
-URL <http://docs.openstack.org/havana/config-reference/content/object-storage-tempurl.html>`__
20
-in the *OpenStack Configuration Reference*.
18
+information, see :ref:`tempurl` in the *Source Documentation*.
21 19
 
22 20
 Note
23 21
 ~~~~

+ 1
- 1
doc/source/associated_projects.rst View File

@@ -28,7 +28,7 @@ Authentication
28 28
 --------------
29 29
 
30 30
 * `Keystone <https://github.com/openstack/keystone>`_ - Official Identity Service for OpenStack.
31
-* `Swauth <https://github.com/gholt/swauth>`_ - Older Swift authentication service that only requires Swift itself.
31
+* `Swauth <https://github.com/openstack/swauth>`_ - An alternative Swift authentication service that only requires Swift itself.
32 32
 * `Basicauth <https://github.com/CloudVPS/swift-basicauth>`_ - HTTP Basic authentication support (keystone backed).
33 33
 
34 34
 

+ 3
- 3
doc/source/development_saio.rst View File

@@ -39,7 +39,7 @@ Installing dependencies
39 39
         sudo apt-get install curl gcc memcached rsync sqlite3 xfsprogs \
40 40
                              git-core libffi-dev python-setuptools
41 41
         sudo apt-get install python-coverage python-dev python-nose \
42
-                             python-simplejson python-xattr python-eventlet \
42
+                             python-xattr python-eventlet \
43 43
                              python-greenlet python-pastedeploy \
44 44
                              python-netifaces python-pip python-dnspython \
45 45
                              python-mock
@@ -50,14 +50,14 @@ Installing dependencies
50 50
         sudo yum install curl gcc memcached rsync sqlite xfsprogs git-core \
51 51
                          libffi-devel xinetd python-setuptools \
52 52
                          python-coverage python-devel python-nose \
53
-                         python-simplejson pyxattr python-eventlet \
53
+                         pyxattr python-eventlet \
54 54
                          python-greenlet python-paste-deploy \
55 55
                          python-netifaces python-pip python-dns \
56 56
                          python-mock
57 57
 
58 58
   Note: This installs necessary system dependencies and *most* of the python
59 59
   dependencies. Later in the process setuptools/distribute or pip will install
60
-  and/or upgrade packages. 
60
+  and/or upgrade packages.
61 61
 
62 62
 Next, choose either :ref:`partition-section` or :ref:`loopback-section`.
63 63
 

+ 7
- 8
doc/source/howto_installmultinode.rst View File

@@ -6,6 +6,13 @@ Please refer to the latest official
6 6
 `Openstack Installation Guides <http://docs.openstack.org/#install-guides>`_
7 7
 for the most up-to-date documentation.
8 8
 
9
+Object Storage installation guide for Openstack Liberty
10
+----------------------------------------------------
11
+
12
+ * `openSUSE 13.2 and SUSE Linux Enterprise Server 12 <http://docs.openstack.org/liberty/install-guide-obs/swift.html>`_
13
+ * `RHEL 7, CentOS 7 <http://docs.openstack.org/liberty/install-guide-rdo/swift.html>`_
14
+ * `Ubuntu 14.04 <http://docs.openstack.org/liberty/install-guide-ubuntu/swift.html>`_
15
+
9 16
 Object Storage installation guide for Openstack Kilo
10 17
 ----------------------------------------------------
11 18
 
@@ -26,11 +33,3 @@ Object Storage installation guide for Openstack Icehouse
26 33
  * `openSUSE and SUSE Linux Enterprise Server <http://docs.openstack.org/icehouse/install-guide/install/zypper/content/ch_swift.html>`_
27 34
  * `Red Hat Enterprise Linux, CentOS, and Fedora <http://docs.openstack.org/icehouse/install-guide/install/yum/content/ch_swift.html>`_
28 35
  * `Ubuntu 12.04/14.04 (LTS) <http://docs.openstack.org/icehouse/install-guide/install/apt/content/ch_swift.html>`_
29
-
30
-Object Storage installation guide for Openstack Havana
31
-------------------------------------------------------
32
-
33
- * `Debian 7.0 <http://docs.openstack.org/havana/install-guide/install/apt-debian/content/ch_swift.html>`_
34
- * `openSUSE and SUSE Linux Enterprise Server <http://docs.openstack.org/havana/install-guide/install/zypper/content/ch_swift.html>`_
35
- * `Red Hat Enterprise Linux, CentOS, and Fedora <http://docs.openstack.org/havana/install-guide/install/yum/content/ch_swift.html>`_
36
- * `Ubuntu 12.04 (LTS) <http://docs.openstack.org/havana/install-guide/install/apt/content/ch_swift.html>`_

+ 4
- 0
etc/drive-audit.conf-sample View File

@@ -1,11 +1,15 @@
1 1
 [drive-audit]
2 2
 # device_dir = /srv/node
3
+#
4
+# You can specify default log routing here if you want:
5
+# log_name = drive-audit
3 6
 # log_facility = LOG_LOCAL0
4 7
 # log_level = INFO
5 8
 # log_address = /dev/log
6 9
 # The following caps the length of log lines to the value given; no limit if
7 10
 # set to 0, the default.
8 11
 # log_max_line_length = 0
12
+#
9 13
 # minutes = 60
10 14
 # error_limit = 1
11 15
 # recon_cache_path = /var/cache/swift

+ 0
- 2
requirements.txt View File

@@ -2,14 +2,12 @@
2 2
 # of appearance. Changing the order has an impact on the overall integration
3 3
 # process, which may cause wedges in the gate later.
4 4
 
5
-pbr>=1.6
6 5
 dnspython>=1.12.0;python_version<'3.0'
7 6
 dnspython3>=1.12.0;python_version>='3.0'
8 7
 eventlet>=0.16.1,!=0.17.0
9 8
 greenlet>=0.3.1
10 9
 netifaces>=0.5,!=0.10.0,!=0.10.1
11 10
 pastedeploy>=1.3.3
12
-simplejson>=2.0.9
13 11
 six>=1.9.0
14 12
 xattr>=0.4
15 13
 PyECLib==1.0.7                          # BSD

+ 31
- 19
swift/account/backend.py View File

@@ -366,7 +366,7 @@ class AccountBroker(DatabaseBroker):
366 366
             ''').fetchone())
367 367
 
368 368
     def list_containers_iter(self, limit, marker, end_marker, prefix,
369
-                             delimiter):
369
+                             delimiter, reverse=False):
370 370
         """
371 371
         Get a list of containers sorted by name starting at marker onward, up
372 372
         to limit entries. Entries will begin with the prefix and will not have
@@ -377,15 +377,21 @@ class AccountBroker(DatabaseBroker):
377 377
         :param end_marker: end marker query
378 378
         :param prefix: prefix query
379 379
         :param delimiter: delimiter for query
380
+        :param reverse: reverse the result order.
380 381
 
381 382
         :returns: list of tuples of (name, object_count, bytes_used, 0)
382 383
         """
383 384
         delim_force_gte = False
384 385
         (marker, end_marker, prefix, delimiter) = utf8encode(
385 386
             marker, end_marker, prefix, delimiter)
387
+        if reverse:
388
+            # Reverse the markers if we are reversing the listing.
389
+            marker, end_marker = end_marker, marker
386 390
         self._commit_puts_stale_ok()
387 391
         if delimiter and not prefix:
388 392
             prefix = ''
393
+        if prefix:
394
+            end_prefix = prefix[:-1] + chr(ord(prefix[-1]) + 1)
389 395
         orig_marker = marker
390 396
         with self.get() as conn:
391 397
             results = []
@@ -395,9 +401,13 @@ class AccountBroker(DatabaseBroker):
395 401
                     FROM container
396 402
                     WHERE """
397 403
                 query_args = []
398
-                if end_marker:
404
+                if end_marker and (not prefix or end_marker < end_prefix):
399 405
                     query += ' name < ? AND'
400 406
                     query_args.append(end_marker)
407
+                elif prefix:
408
+                    query += ' name < ? AND'
409
+                    query_args.append(end_prefix)
410
+
401 411
                 if delim_force_gte:
402 412
                     query += ' name >= ? AND'
403 413
                     query_args.append(marker)
@@ -413,38 +423,40 @@ class AccountBroker(DatabaseBroker):
413 423
                     query += ' +deleted = 0'
414 424
                 else:
415 425
                     query += ' deleted = 0'
416
-                query += ' ORDER BY name LIMIT ?'
426
+                query += ' ORDER BY name %s LIMIT ?' % \
427
+                         ('DESC' if reverse else '')
417 428
                 query_args.append(limit - len(results))
418 429
                 curs = conn.execute(query, query_args)
419 430
                 curs.row_factory = None
420 431
 
421
-                if prefix is None:
422
-                    # A delimiter without a specified prefix is ignored
432
+                # Delimiters without a prefix is ignored, further if there
433
+                # is no delimiter then we can simply return the result as
434
+                # prefixes are now handled in the SQL statement.
435
+                if prefix is None or not delimiter:
423 436
                     return [r for r in curs]
424
-                if not delimiter:
425
-                    if not prefix:
426
-                        # It is possible to have a delimiter but no prefix
427
-                        # specified. As above, the prefix will be set to the
428
-                        # empty string, so avoid performing the extra work to
429
-                        # check against an empty prefix.
430
-                        return [r for r in curs]
431
-                    else:
432
-                        return [r for r in curs if r[0].startswith(prefix)]
433 437
 
434 438
                 # We have a delimiter and a prefix (possibly empty string) to
435 439
                 # handle
436 440
                 rowcount = 0
437 441
                 for row in curs:
438 442
                     rowcount += 1
439
-                    marker = name = row[0]
440
-                    if len(results) >= limit or not name.startswith(prefix):
443
+                    name = row[0]
444
+                    if reverse:
445
+                        end_marker = name
446
+                    else:
447
+                        marker = name
448
+
449
+                    if len(results) >= limit:
441 450
                         curs.close()
442 451
                         return results
443 452
                     end = name.find(delimiter, len(prefix))
444 453
                     if end > 0:
445
-                        marker = name[:end] + chr(ord(delimiter) + 1)
446
-                        # we want result to be inclusive of delim+1
447
-                        delim_force_gte = True
454
+                        if reverse:
455
+                            end_marker = name[:end + 1]
456
+                        else:
457
+                            marker = name[:end] + chr(ord(delimiter) + 1)
458
+                            # we want result to be inclusive of delim+1
459
+                            delim_force_gte = True
448 460
                         dir_name = name[:end + 1]
449 461
                         if dir_name != orig_marker:
450 462
                             results.append([dir_name, 0, 0, 1])

+ 2
- 1
swift/account/server.py View File

@@ -191,6 +191,7 @@ class AccountController(BaseStorageServer):
191 191
             return HTTPPreconditionFailed(body='Bad delimiter')
192 192
         limit = constraints.ACCOUNT_LISTING_LIMIT
193 193
         given_limit = get_param(req, 'limit')
194
+        reverse = config_true_value(get_param(req, 'reverse'))
194 195
         if given_limit and given_limit.isdigit():
195 196
             limit = int(given_limit)
196 197
             if limit > constraints.ACCOUNT_LISTING_LIMIT:
@@ -211,7 +212,7 @@ class AccountController(BaseStorageServer):
211 212
             return self._deleted_response(broker, req, HTTPNotFound)
212 213
         return account_listing_response(account, req, out_content_type, broker,
213 214
                                         limit, marker, end_marker, prefix,
214
-                                        delimiter)
215
+                                        delimiter, reverse)
215 216
 
216 217
     @public
217 218
     @replication

+ 4
- 3
swift/account/utils.py View File

@@ -13,11 +13,12 @@
13 13
 # See the License for the specific language governing permissions and
14 14
 # limitations under the License.
15 15
 
16
+import json
16 17
 import time
17 18
 from xml.sax import saxutils
18 19
 
19 20
 from swift.common.swob import HTTPOk, HTTPNoContent
20
-from swift.common.utils import json, Timestamp
21
+from swift.common.utils import Timestamp
21 22
 from swift.common.storage_policy import POLICIES
22 23
 
23 24
 
@@ -70,14 +71,14 @@ def get_response_headers(broker):
70 71
 
71 72
 def account_listing_response(account, req, response_content_type, broker=None,
72 73
                              limit='', marker='', end_marker='', prefix='',
73
-                             delimiter=''):
74
+                             delimiter='', reverse=False):
74 75
     if broker is None:
75 76
         broker = FakeAccountBroker()
76 77
 
77 78
     resp_headers = get_response_headers(broker)
78 79
 
79 80
     account_list = broker.list_containers_iter(limit, marker, end_marker,
80
-                                               prefix, delimiter)
81
+                                               prefix, delimiter, reverse)
81 82
     if response_content_type == 'application/json':
82 83
         data = []
83 84
         for (name, object_count, bytes_used, is_subdir) in account_list:

+ 2
- 1
swift/common/db.py View File

@@ -17,6 +17,7 @@
17 17
 
18 18
 from contextlib import contextmanager, closing
19 19
 import hashlib
20
+import json
20 21
 import logging
21 22
 import os
22 23
 from uuid import uuid4
@@ -32,7 +33,7 @@ from eventlet import sleep, Timeout
32 33
 import sqlite3
33 34
 
34 35
 from swift.common.constraints import MAX_META_COUNT, MAX_META_OVERALL_SIZE
35
-from swift.common.utils import json, Timestamp, renamer, \
36
+from swift.common.utils import Timestamp, renamer, \
36 37
     mkdirs, lock_parent_directory, fallocate
37 38
 from swift.common.exceptions import LockTimeout
38 39
 from swift.common.swob import HTTPBadRequest

+ 1
- 5
swift/common/direct_client.py View File

@@ -18,6 +18,7 @@ Internal client library for making calls directly to the servers rather than
18 18
 through the proxy.
19 19
 """
20 20
 
21
+import json
21 22
 import os
22 23
 import socket
23 24
 from time import time
@@ -34,11 +35,6 @@ from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \
34 35
 from swift.common.swob import HeaderKeyDict
35 36
 from swift.common.utils import quote
36 37
 
37
-try:
38
-    import simplejson as json
39
-except ImportError:
40
-    import json
41
-
42 38
 
43 39
 class DirectClientException(ClientException):
44 40
 

+ 17
- 7
swift/common/manager.py View File

@@ -42,6 +42,8 @@ ALL_SERVERS = ['account-auditor', 'account-server', 'container-auditor',
42 42
 MAIN_SERVERS = ['proxy-server', 'account-server', 'container-server',
43 43
                 'object-server']
44 44
 REST_SERVERS = [s for s in ALL_SERVERS if s not in MAIN_SERVERS]
45
+# aliases mapping
46
+ALIASES = {'all': ALL_SERVERS, 'main': MAIN_SERVERS, 'rest': REST_SERVERS}
45 47
 GRACEFUL_SHUTDOWN_SERVERS = MAIN_SERVERS + ['auth-server']
46 48
 START_ONCE_SERVERS = REST_SERVERS
47 49
 # These are servers that match a type (account-*, container-*, object-*) but
@@ -173,18 +175,17 @@ class Manager(object):
173 175
 
174 176
     def __init__(self, servers, run_dir=RUN_DIR):
175 177
         self.server_names = set()
178
+        self._default_strict = True
176 179
         for server in servers:
177
-            if server == 'all':
178
-                self.server_names.update(ALL_SERVERS)
179
-            elif server == 'main':
180
-                self.server_names.update(MAIN_SERVERS)
181
-            elif server == 'rest':
182
-                self.server_names.update(REST_SERVERS)
180
+            if server in ALIASES:
181
+                self.server_names.update(ALIASES[server])
182
+                self._default_strict = False
183 183
             elif '*' in server:
184 184
                 # convert glob to regex
185 185
                 self.server_names.update([
186 186
                     s for s in ALL_SERVERS if
187 187
                     re.match(server.replace('*', '.*'), s)])
188
+                self._default_strict = False
188 189
             else:
189 190
                 self.server_names.add(server)
190 191
 
@@ -211,8 +212,17 @@ class Manager(object):
211 212
         setup_env()
212 213
         status = 0
213 214
 
215
+        strict = kwargs.get('strict')
216
+        # if strict not set explicitly
217
+        if strict is None:
218
+            strict = self._default_strict
219
+
214 220
         for server in self.servers:
215
-            server.launch(**kwargs)
221
+            status += 0 if server.launch(**kwargs) else 1
222
+
223
+        if not strict:
224
+            status = 0
225
+
216 226
         if not kwargs.get('daemon', True):
217 227
             for server in self.servers:
218 228
                 try:

+ 1
- 1
swift/common/memcached.py View File

@@ -45,6 +45,7 @@ http://github.com/memcached/memcached/blob/1.4.2/doc/protocol.txt
45 45
 """
46 46
 
47 47
 import six.moves.cPickle as pickle
48
+import json
48 49
 import logging
49 50
 import time
50 51
 from bisect import bisect
@@ -56,7 +57,6 @@ from eventlet.pools import Pool
56 57
 from eventlet import Timeout
57 58
 from six.moves import range
58 59
 
59
-from swift.common.utils import json
60 60
 
61 61
 DEFAULT_MEMCACHED_PORT = 11211
62 62
 

+ 3
- 1
swift/common/middleware/acl.py View File

@@ -13,7 +13,9 @@
13 13
 # See the License for the specific language governing permissions and
14 14
 # limitations under the License.
15 15
 
16
-from swift.common.utils import urlparse, json
16
+import json
17
+
18
+from swift.common.utils import urlparse
17 19
 
18 20
 
19 21
 def clean_acl(name, value):

+ 3
- 2
swift/common/middleware/bulk.py View File

@@ -13,6 +13,7 @@
13 13
 # See the License for the specific language governing permissions and
14 14
 # limitations under the License.
15 15
 
16
+import json
16 17
 from six.moves.urllib.parse import quote, unquote
17 18
 import tarfile
18 19
 from xml.sax import saxutils
@@ -23,7 +24,7 @@ from swift.common.swob import Request, HTTPBadGateway, \
23 24
     HTTPCreated, HTTPBadRequest, HTTPNotFound, HTTPUnauthorized, HTTPOk, \
24 25
     HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPNotAcceptable, \
25 26
     HTTPLengthRequired, HTTPException, HTTPServerError, wsgify
26
-from swift.common.utils import json, get_logger, register_swift_info
27
+from swift.common.utils import get_logger, register_swift_info
27 28
 from swift.common import constraints
28 29
 from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND, HTTP_CONFLICT
29 30
 
@@ -32,7 +33,7 @@ class CreateContainerError(Exception):
32 33
     def __init__(self, msg, status_int, status):
33 34
         self.status_int = status_int
34 35
         self.status = status
35
-        Exception.__init__(self, msg)
36
+        super(CreateContainerError, self).__init__(msg)
36 37
 
37 38
 
38 39
 ACCEPTABLE_FORMATS = ['text/plain', 'application/json', 'application/xml',

+ 2
- 1
swift/common/middleware/dlo.py View File

@@ -114,6 +114,7 @@ Here's an example using ``curl`` with tiny 1-byte segments::
114 114
         http://<storage_url>/container/myobject
115 115
 """
116 116
 
117
+import json
117 118
 import os
118 119
 
119 120
 import six
@@ -126,7 +127,7 @@ from swift.common.exceptions import ListingIterError, SegmentError
126 127
 from swift.common.http import is_success
127 128
 from swift.common.swob import Request, Response, \
128 129
     HTTPRequestedRangeNotSatisfiable, HTTPBadRequest, HTTPConflict
129
-from swift.common.utils import get_logger, json, \
130
+from swift.common.utils import get_logger, \
130 131
     RateLimitedIterator, read_conf_dir, quote, close_if_possible, \
131 132
     closing_if_possible
132 133
 from swift.common.request_helpers import SegmentedIterable

+ 6
- 22
swift/common/middleware/keystoneauth.py View File

@@ -196,7 +196,7 @@ class KeystoneAuth(object):
196 196
             conf.get('allow_names_in_acls', 'true'))
197 197
 
198 198
     def __call__(self, environ, start_response):
199
-        identity = self._keystone_identity(environ)
199
+        env_identity = self._keystone_identity(environ)
200 200
 
201 201
         # Check if one of the middleware like tempurl or formpost have
202 202
         # set the swift.authorize_override environ and want to control the
@@ -207,14 +207,13 @@ class KeystoneAuth(object):
207 207
             self.logger.debug(msg)
208 208
             return self.app(environ, start_response)
209 209
 
210
-        if identity:
211
-            self.logger.debug('Using identity: %r', identity)
212
-            environ['keystone.identity'] = identity
213
-            environ['REMOTE_USER'] = identity.get('tenant')
214
-            env_identity = self._integral_keystone_identity(environ)
210
+        if env_identity:
211
+            self.logger.debug('Using identity: %r', env_identity)
212
+            environ['REMOTE_USER'] = env_identity.get('tenant')
213
+            environ['keystone.identity'] = env_identity
215 214
             environ['swift.authorize'] = functools.partial(
216 215
                 self.authorize, env_identity)
217
-            user_roles = (r.lower() for r in identity.get('roles', []))
216
+            user_roles = (r.lower() for r in env_identity.get('roles', []))
218 217
             if self.reseller_admin_role in user_roles:
219 218
                 environ['reseller_request'] = True
220 219
         else:
@@ -238,26 +237,11 @@ class KeystoneAuth(object):
238 237
 
239 238
     def _keystone_identity(self, environ):
240 239
         """Extract the identity from the Keystone auth component."""
241
-        # In next release, we would add user id in env['keystone.identity'] by
242
-        # using _integral_keystone_identity to replace current
243
-        # _keystone_identity. The purpose of keeping it in this release it for
244
-        # back compatibility.
245 240
         if (environ.get('HTTP_X_IDENTITY_STATUS') != 'Confirmed'
246 241
             or environ.get(
247 242
                 'HTTP_X_SERVICE_IDENTITY_STATUS') not in (None, 'Confirmed')):
248 243
             return
249 244
         roles = list_from_csv(environ.get('HTTP_X_ROLES', ''))
250
-        identity = {'user': environ.get('HTTP_X_USER_NAME'),
251
-                    'tenant': (environ.get('HTTP_X_TENANT_ID'),
252
-                               environ.get('HTTP_X_TENANT_NAME')),
253
-                    'roles': roles}
254
-        return identity
255
-
256
-    def _integral_keystone_identity(self, environ):
257
-        """Extract the identity from the Keystone auth component."""
258
-        if environ.get('HTTP_X_IDENTITY_STATUS') != 'Confirmed':
259
-            return
260
-        roles = list_from_csv(environ.get('HTTP_X_ROLES', ''))
261 245
         service_roles = list_from_csv(environ.get('HTTP_X_SERVICE_ROLES', ''))
262 246
         identity = {'user': (environ.get('HTTP_X_USER_ID'),
263 247
                              environ.get('HTTP_X_USER_NAME')),

+ 2
- 1
swift/common/middleware/list_endpoints.py View File

@@ -78,11 +78,12 @@ with this middleware enabled should not be open to an untrusted
78 78
 environment (everyone can query the locality data using this middleware).
79 79
 """
80 80
 
81
+import json
81 82
 
82 83
 from six.moves.urllib.parse import quote, unquote
83 84
 
84 85
 from swift.common.ring import Ring
85
-from swift.common.utils import json, get_logger, split_path
86
+from swift.common.utils import get_logger, split_path
86 87
 from swift.common.swob import Request, Response
87 88
 from swift.common.swob import HTTPBadRequest, HTTPMethodNotAllowed
88 89
 from swift.common.storage_policy import POLICIES

+ 2
- 1
swift/common/middleware/recon.py View File

@@ -14,6 +14,7 @@
14 14
 # limitations under the License.
15 15
 
16 16
 import errno
17
+import json
17 18
 import os
18 19
 import time
19 20
 from swift import gettext_ as _
@@ -21,7 +22,7 @@ from swift import gettext_ as _
21 22
 from swift import __version__ as swiftver
22 23
 from swift.common.storage_policy import POLICIES
23 24
 from swift.common.swob import Request, Response
24
-from swift.common.utils import get_logger, config_true_value, json, \
25
+from swift.common.utils import get_logger, config_true_value, \
25 26
     SWIFT_CONF_FILE
26 27
 from swift.common.constraints import check_mount
27 28
 from resource import getpagesize

+ 3
- 1
swift/common/middleware/slo.py View File

@@ -197,6 +197,7 @@ metadata which can be used for stats purposes.
197 197
 from six.moves import range
198 198
 
199 199
 from datetime import datetime
200
+import json
200 201
 import mimetypes
201 202
 import re
202 203
 import six
@@ -208,7 +209,7 @@ from swift.common.swob import Request, HTTPBadRequest, HTTPServerError, \
208 209
     HTTPOk, HTTPPreconditionFailed, HTTPException, HTTPNotFound, \
209 210
     HTTPUnauthorized, HTTPConflict, HTTPRequestedRangeNotSatisfiable,\
210 211
     Response, Range
211
-from swift.common.utils import json, get_logger, config_true_value, \
212
+from swift.common.utils import get_logger, config_true_value, \
212 213
     get_valid_utf8_str, override_bytes_from_content_type, split_path, \
213 214
     register_swift_info, RateLimitedIterator, quote, close_if_possible, \
214 215
     closing_if_possible
@@ -989,6 +990,7 @@ class StaticLargeObject(object):
989 990
         :params req: a swob.Request with an obj in path
990 991
         :returns: swob.Response whose app_iter set to Bulk.handle_delete_iter
991 992
         """
993
+        req.headers['Content-Type'] = None  # Ignore content-type from client
992 994
         resp = HTTPOk(request=req)
993 995
         out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS)
994 996
         if out_content_type:

+ 2
- 1
swift/common/middleware/versioned_writes.py View File

@@ -113,10 +113,11 @@ Disable versioning from a container (x is any value except empty)::
113 113
 -H "X-Remove-Versions-Location: x" http://<storage_url>/container
114 114
 """
115 115
 
116
+import json
116 117
 import six
117 118
 from six.moves.urllib.parse import quote, unquote
118 119
 import time
119
-from swift.common.utils import get_logger, Timestamp, json, \
120
+from swift.common.utils import get_logger, Timestamp, \
120 121
     register_swift_info, config_true_value
121 122
 from swift.common.request_helpers import get_sys_meta_prefix
122 123
 from swift.common.wsgi import WSGIContext, make_pre_authed_request

+ 2
- 1
swift/common/ring/ring.py View File

@@ -16,6 +16,7 @@
16 16
 import array
17 17
 import six.moves.cPickle as pickle
18 18
 import inspect
19
+import json
19 20
 from collections import defaultdict
20 21
 from gzip import GzipFile
21 22
 from os.path import getmtime
@@ -29,7 +30,7 @@ from tempfile import NamedTemporaryFile
29 30
 
30 31
 from six.moves import range
31 32
 
32
-from swift.common.utils import hash_path, validate_configuration, json
33
+from swift.common.utils import hash_path, validate_configuration
33 34
 from swift.common.ring.utils import tiers_for_dev
34 35
 
35 36
 

+ 10
- 57
swift/common/utils.py View File

@@ -21,16 +21,15 @@ import errno
21 21
 import fcntl
22 22
 import grp
23 23
 import hmac
24
+import json
24 25
 import operator
25 26
 import os
26 27
 import pwd
27 28
 import re
28 29
 import sys
29
-import threading as stdlib_threading
30 30
 import time
31 31
 import uuid
32 32
 import functools
33
-import weakref
34 33
 import email.parser
35 34
 from hashlib import md5, sha1
36 35
 from random import random, shuffle
@@ -40,10 +39,6 @@ import ctypes.util
40 39
 from optparse import OptionParser
41 40
 
42 41
 from tempfile import mkstemp, NamedTemporaryFile
43
-try:
44
-    import simplejson as json
45
-except ImportError:
46
-    import json
47 42
 import glob
48 43
 import itertools
49 44
 import stat
@@ -63,7 +58,6 @@ import six
63 58
 from six.moves import cPickle as pickle
64 59
 from six.moves.configparser import (ConfigParser, NoSectionError,
65 60
                                     NoOptionError, RawConfigParser)
66
-from six.moves.queue import Queue, Empty
67 61
 from six.moves import range
68 62
 from six.moves.urllib.parse import ParseResult
69 63
 from six.moves.urllib.parse import quote as _quote
@@ -74,6 +68,11 @@ import swift.common.exceptions
74 68
 from swift.common.http import is_success, is_redirection, HTTP_NOT_FOUND, \
75 69
     HTTP_PRECONDITION_FAILED, HTTP_REQUESTED_RANGE_NOT_SATISFIABLE
76 70
 
71
+if six.PY3:
72
+    stdlib_queue = eventlet.patcher.original('queue')
73
+else:
74
+    stdlib_queue = eventlet.patcher.original('Queue')
75
+stdlib_threading = eventlet.patcher.original('threading')
77 76
 
78 77
 # logging doesn't import patched as cleanly as one would like
79 78
 from logging.handlers import SysLogHandler
@@ -1245,27 +1244,6 @@ def timing_stats(**dec_kwargs):
1245 1244
     return decorating_func
1246 1245
 
1247 1246
 
1248
-class LoggingHandlerWeakRef(weakref.ref):
1249
-    """
1250
-    Like a weak reference, but passes through a couple methods that logging
1251
-    handlers need.
1252
-    """
1253
-
1254
-    def close(self):
1255
-        referent = self()
1256
-        try:
1257
-            if referent:
1258
-                referent.close()
1259
-        except KeyError:
1260
-            # This is to catch an issue with old py2.6 versions
1261
-            pass
1262
-
1263
-    def flush(self):
1264
-        referent = self()
1265
-        if referent:
1266
-            referent.flush()
1267
-
1268
-
1269 1247
 # double inheritance to support property with setter
1270 1248
 class LogAdapter(logging.LoggerAdapter, object):
1271 1249
     """
@@ -1565,31 +1543,6 @@ def get_logger(conf, name=None, log_to_console=False, log_route=None,
1565 1543
                 print('Invalid custom handler format [%s]' % hook,
1566 1544
                       file=sys.stderr)
1567 1545
 
1568
-    # Python 2.6 has the undesirable property of keeping references to all log
1569
-    # handlers around forever in logging._handlers and logging._handlerList.
1570
-    # Combine that with handlers that keep file descriptors, and you get an fd
1571
-    # leak.
1572
-    #
1573
-    # And no, we can't share handlers; a SyslogHandler has a socket, and if
1574
-    # two greenthreads end up logging at the same time, you could get message
1575
-    # overlap that garbles the logs and makes eventlet complain.
1576
-    #
1577
-    # Python 2.7 uses weakrefs to avoid the leak, so let's do that too.
1578
-    if sys.version_info[0] == 2 and sys.version_info[1] <= 6:
1579
-        try:
1580
-            logging._acquireLock()  # some thread-safety thing
1581
-            for handler in adapted_logger.logger.handlers:
1582
-                if handler in logging._handlers:
1583
-                    wr = LoggingHandlerWeakRef(handler)
1584
-                    del logging._handlers[handler]
1585
-                    logging._handlers[wr] = 1
1586
-                for i, handler_ref in enumerate(logging._handlerList):
1587
-                    if handler_ref is handler:
1588
-                        logging._handlerList[i] = LoggingHandlerWeakRef(
1589
-                            handler)
1590
-        finally:
1591
-            logging._releaseLock()
1592
-
1593 1546
     return adapted_logger
1594 1547
 
1595 1548
 
@@ -2333,7 +2286,7 @@ class GreenAsyncPile(object):
2333 2286
     def next(self):
2334 2287
         try:
2335 2288
             rv = self._responses.get_nowait()
2336
-        except Empty:
2289
+        except eventlet.queue.Empty:
2337 2290
             if self._inflight == 0:
2338 2291
                 raise StopIteration()
2339 2292
             rv = self._responses.get()
@@ -2984,8 +2937,8 @@ class ThreadPool(object):
2984 2937
 
2985 2938
     def __init__(self, nthreads=2):
2986 2939
         self.nthreads = nthreads
2987
-        self._run_queue = Queue()
2988
-        self._result_queue = Queue()
2940
+        self._run_queue = stdlib_queue.Queue()
2941
+        self._result_queue = stdlib_queue.Queue()
2989 2942
         self._threads = []
2990 2943
         self._alive = True
2991 2944
 
@@ -3065,7 +3018,7 @@ class ThreadPool(object):
3065 3018
             while True:
3066 3019
                 try:
3067 3020
                     ev, success, result = queue.get(block=False)
3068
-                except Empty:
3021
+                except stdlib_queue.Empty:
3069 3022
                     break
3070 3023
 
3071 3024
                 try:

+ 2
- 1
swift/common/wsgi.py View File

@@ -407,7 +407,8 @@ def run_server(conf, logger, sock, global_conf=None):
407 407
     wsgi.WRITE_TIMEOUT = int(conf.get('client_timeout') or 60)
408 408
 
409 409
     eventlet.hubs.use_hub(get_hub())
410
-    eventlet.patcher.monkey_patch(all=False, socket=True)
410
+    # NOTE(sileht): monkey-patching thread is required by python-keystoneclient
411
+    eventlet.patcher.monkey_patch(all=False, socket=True, thread=True)
411 412
     eventlet_debug = config_true_value(conf.get('eventlet_debug', 'no'))
412 413
     eventlet.debug.hub_exceptions(eventlet_debug)
413 414
     wsgi_logger = NullLogger()

+ 35
- 21
swift/container/backend.py View File

@@ -557,7 +557,7 @@ class ContainerBroker(DatabaseBroker):
557 557
             conn.commit()
558 558
 
559 559
     def list_objects_iter(self, limit, marker, end_marker, prefix, delimiter,
560
-                          path=None, storage_policy_index=0):
560
+                          path=None, storage_policy_index=0, reverse=False):
561 561
         """
562 562
         Get a list of objects sorted by name starting at marker onward, up
563 563
         to limit entries.  Entries will begin with the prefix and will not
@@ -570,6 +570,7 @@ class ContainerBroker(DatabaseBroker):
570 570
         :param delimiter: delimiter for query
571 571
         :param path: if defined, will set the prefix and delimiter based on
572 572
                      the path
573
+        :param reverse: reverse the result order.
573 574
 
574 575
         :returns: list of tuples of (name, created_at, size, content_type,
575 576
                   etag)
@@ -578,6 +579,9 @@ class ContainerBroker(DatabaseBroker):
578 579
         (marker, end_marker, prefix, delimiter, path) = utf8encode(
579 580
             marker, end_marker, prefix, delimiter, path)
580 581
         self._commit_puts_stale_ok()
582
+        if reverse:
583
+            # Reverse the markers if we are reversing the listing.
584
+            marker, end_marker = end_marker, marker
581 585
         if path is not None:
582 586
             prefix = path
583 587
             if path:
@@ -585,6 +589,8 @@ class ContainerBroker(DatabaseBroker):
585 589
             delimiter = '/'
586 590
         elif delimiter and not prefix:
587 591
             prefix = ''
592
+        if prefix:
593
+            end_prefix = prefix[:-1] + chr(ord(prefix[-1]) + 1)
588 594
         orig_marker = marker
589 595
         with self.get() as conn:
590 596
             results = []
@@ -592,9 +598,13 @@ class ContainerBroker(DatabaseBroker):
592 598
                 query = '''SELECT name, created_at, size, content_type, etag
593 599
                            FROM object WHERE'''
594 600
                 query_args = []
595
-                if end_marker:
601
+                if end_marker and (not prefix or end_marker < end_prefix):
596 602
                     query += ' name < ? AND'
597 603
                     query_args.append(end_marker)
604
+                elif prefix:
605
+                    query += ' name < ? AND'
606
+                    query_args.append(end_prefix)
607
+
598 608
                 if delim_force_gte:
599 609
                     query += ' name >= ? AND'
600 610
                     query_args.append(marker)
@@ -611,8 +621,8 @@ class ContainerBroker(DatabaseBroker):
611 621
                 else:
612 622
                     query += ' deleted = 0'
613 623
                 orig_tail_query = '''
614
-                    ORDER BY name LIMIT ?
615
-                '''
624
+                    ORDER BY name %s LIMIT ?
625
+                ''' % ('DESC' if reverse else '')
616 626
                 orig_tail_args = [limit - len(results)]
617 627
                 # storage policy filter
618 628
                 policy_tail_query = '''
@@ -633,26 +643,24 @@ class ContainerBroker(DatabaseBroker):
633 643
                                         tuple(query_args + tail_args))
634 644
                 curs.row_factory = None
635 645
 
636
-                if prefix is None:
637
-                    # A delimiter without a specified prefix is ignored
646
+                # Delimiters without a prefix is ignored, further if there
647
+                # is no delimiter then we can simply return the result as
648
+                # prefixes are now handled in the SQL statement.
649
+                if prefix is None or not delimiter:
638 650
                     return [r for r in curs]
639
-                if not delimiter:
640
-                    if not prefix:
641
-                        # It is possible to have a delimiter but no prefix
642
-                        # specified. As above, the prefix will be set to the
643
-                        # empty string, so avoid performing the extra work to
644
-                        # check against an empty prefix.
645
-                        return [r for r in curs]
646
-                    else:
647
-                        return [r for r in curs if r[0].startswith(prefix)]
648 651
 
649 652
                 # We have a delimiter and a prefix (possibly empty string) to
650 653
                 # handle
651 654
                 rowcount = 0
652 655
                 for row in curs:
653 656
                     rowcount += 1
654
-                    marker = name = row[0]
655
-                    if len(results) >= limit or not name.startswith(prefix):
657
+                    name = row[0]
658
+                    if reverse:
659
+                        end_marker = name
660
+                    else:
661
+                        marker = name
662
+
663
+                    if len(results) >= limit:
656 664
                         curs.close()
657 665
                         return results
658 666
                     end = name.find(delimiter, len(prefix))
@@ -660,13 +668,19 @@ class ContainerBroker(DatabaseBroker):
660 668
                         if name == path:
661 669
                             continue
662 670
                         if end >= 0 and len(name) > end + len(delimiter):
663
-                            marker = name[:end] + chr(ord(delimiter) + 1)
671
+                            if reverse:
672
+                                end_marker = name[:end + 1]
673
+                            else:
674
+                                marker = name[:end] + chr(ord(delimiter) + 1)
664 675
                             curs.close()
665 676
                             break
666 677
                     elif end > 0:
667
-                        marker = name[:end] + chr(ord(delimiter) + 1)
668
-                        # we want result to be inclusive of delim+1
669
-                        delim_force_gte = True
678
+                        if reverse:
679
+                            end_marker = name[:end + 1]
680
+                        else:
681
+                            marker = name[:end] + chr(ord(delimiter) + 1)
682
+                            # we want result to be inclusive of delim+1
683
+                            delim_force_gte = True
670 684
                         dir_name = name[:end + 1]
671 685
                         if dir_name != orig_marker:
672 686
                             results.append([dir_name, '0', 0, None, ''])

+ 2
- 1
swift/container/replicator.py View File

@@ -15,6 +15,7 @@
15 15
 
16 16
 import os
17 17
 import itertools
18
+import json
18 19
 import time
19 20
 from collections import defaultdict
20 21
 from eventlet import Timeout
@@ -28,7 +29,7 @@ from swift.common.storage_policy import POLICIES
28 29
 from swift.common.exceptions import DeviceUnavailable
29 30
 from swift.common.http import is_success
30 31
 from swift.common.db import DatabaseAlreadyExists
31
-from swift.common.utils import (json, Timestamp, hash_path,
32
+from swift.common.utils import (Timestamp, hash_path,
32 33
                                 storage_directory, quorum_size)
33 34
 
34 35
 

+ 4
- 2
swift/container/server.py View File

@@ -13,6 +13,7 @@
13 13
 # See the License for the specific language governing permissions and
14 14
 # limitations under the License.
15 15
 
16
+import json
16 17
 import os
17 18
 import time
18 19
 import traceback
@@ -30,7 +31,7 @@ from swift.common.request_helpers import get_param, get_listing_content_type, \
30 31
     split_and_validate_path, is_sys_or_user_meta
31 32
 from swift.common.utils import get_logger, hash_path, public, \
32 33
     Timestamp, storage_directory, validate_sync_to, \
33
-    config_true_value, json, timing_stats, replication, \
34
+    config_true_value, timing_stats, replication, \
34 35
     override_bytes_from_content_type, get_log_line
35 36
 from swift.common.constraints import check_mount, valid_timestamp, check_utf8
36 37
 from swift.common import constraints
@@ -452,6 +453,7 @@ class ContainerController(BaseStorageServer):
452 453
         end_marker = get_param(req, 'end_marker')
453 454
         limit = constraints.CONTAINER_LISTING_LIMIT
454 455
         given_limit = get_param(req, 'limit')
456
+        reverse = config_true_value(get_param(req, 'reverse'))
455 457
         if given_limit and given_limit.isdigit():
456 458
             limit = int(given_limit)
457 459
             if limit > constraints.CONTAINER_LISTING_LIMIT:
@@ -471,7 +473,7 @@ class ContainerController(BaseStorageServer):
471 473
             return HTTPNotFound(request=req, headers=resp_headers)
472 474
         container_list = broker.list_objects_iter(
473 475
             limit, marker, end_marker, prefix, delimiter, path,
474
-            storage_policy_index=info['storage_policy_index'])
476
+            storage_policy_index=info['storage_policy_index'], reverse=reverse)
475 477
         return self.create_listing(req, out_content_type, info, resp_headers,
476 478
                                    broker.metadata, container_list, container)
477 479
 

+ 2
- 1
swift/obj/auditor.py View File

@@ -13,6 +13,7 @@
13 13
 # See the License for the specific language governing permissions and
14 14
 # limitations under the License.
15 15
 
16
+import json
16 17
 import os
17 18
 import sys
18 19
 import time
@@ -24,7 +25,7 @@ from eventlet import Timeout
24 25
 
25 26
 from swift.obj import diskfile
26 27
 from swift.common.utils import get_logger, ratelimit_sleep, dump_recon_cache, \
27
-    list_from_csv, json, listdir
28
+    list_from_csv, listdir
28 29
 from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist
29 30
 from swift.common.daemon import Daemon
30 31
 

+ 0
- 5
swift/proxy/controllers/base.py View File

@@ -920,7 +920,6 @@ class ResumingGetter(object):
920 920
                         if nchunks % 5 == 0:
921 921
                             sleep()
922 922
 
923
-            part_iter = None
924 923
             try:
925 924
                 while True:
926 925
                     start_byte, end_byte, length, headers, part = \
@@ -932,10 +931,6 @@ class ResumingGetter(object):
932 931
                            'entity_length': length, 'headers': headers,
933 932
                            'part_iter': part_iter}
934 933
                     self.pop_range()
935
-            except GeneratorExit:
936
-                if part_iter:
937
-                    part_iter.close()
938
-                raise
939 934
             except StopIteration:
940 935
                 req.environ['swift.non_client_disconnect'] = True
941 936
 

+ 2
- 1
swift/proxy/controllers/info.py View File

@@ -13,9 +13,10 @@
13 13
 # See the License for the specific language governing permissions and
14 14
 # limitations under the License.
15 15
 
16
+import json
16 17
 from time import time
17 18
 
18
-from swift.common.utils import public, get_hmac, get_swift_info, json, \
19
+from swift.common.utils import public, get_hmac, get_swift_info, \
19 20
     streq_const_time
20 21
 from swift.proxy.controllers.base import Controller, delay_denial
21 22
 from swift.common.swob import HTTPOk, HTTPForbidden, HTTPUnauthorized

+ 8
- 3
swift/proxy/controllers/obj.py View File

@@ -29,6 +29,7 @@ from six.moves.urllib.parse import unquote, quote
29 29
 
30 30
 import collections
31 31
 import itertools
32
+import json
32 33
 import time
33 34
 import math
34 35
 import random
@@ -42,7 +43,7 @@ from eventlet.timeout import Timeout
42 43
 
43 44
 from swift.common.utils import (
44 45
     clean_content_type, config_true_value, ContextPool, csv_append,
45
-    GreenAsyncPile, GreenthreadSafeIterator, json, Timestamp,
46
+    GreenAsyncPile, GreenthreadSafeIterator, Timestamp,
46 47
     normalize_delete_at_timestamp, public, get_expirer_container,
47 48
     document_iters_to_http_response_body, parse_content_range,
48 49
     quorum_size, reiterate, close_if_possible)
@@ -59,7 +60,8 @@ from swift.common.http import (
59 60
     is_informational, is_success, is_client_error, is_server_error,
60 61
     HTTP_CONTINUE, HTTP_CREATED, HTTP_MULTIPLE_CHOICES,
61 62
     HTTP_INTERNAL_SERVER_ERROR, HTTP_SERVICE_UNAVAILABLE,
62
-    HTTP_INSUFFICIENT_STORAGE, HTTP_PRECONDITION_FAILED, HTTP_CONFLICT)
63
+    HTTP_INSUFFICIENT_STORAGE, HTTP_PRECONDITION_FAILED, HTTP_CONFLICT,
64
+    HTTP_UNPROCESSABLE_ENTITY)
63 65
 from swift.common.storage_policy import (POLICIES, REPL_POLICY, EC_POLICY,
64 66
                                          ECDriverError, PolicyError)
65 67
 from swift.proxy.controllers.base import Controller, delay_denial, \
@@ -880,7 +882,9 @@ class ReplicatedObjectController(BaseObjectController):
880 882
                     conn.resp = None
881 883
                     conn.node = node
882 884
                     return conn
883
-                elif is_success(resp.status) or resp.status == HTTP_CONFLICT:
885
+                elif (is_success(resp.status)
886
+                      or resp.status in (HTTP_CONFLICT,
887
+                                         HTTP_UNPROCESSABLE_ENTITY)):
884 888
                     conn.resp = resp
885 889
                     conn.node = node
886 890
                     return conn
@@ -1415,6 +1419,7 @@ class ECAppIter(object):
1415 1419
             finally:
1416 1420
                 queue.resize(2)  # ensure there's room
1417 1421
                 queue.put(None)
1422
+                frag_iter.close()
1418 1423
 
1419 1424
         with ContextPool(len(fragment_iters)) as pool:
1420 1425
             for frag_iter, queue in zip(fragment_iters, queues):

+ 1
- 1
test/functional/swift_test_client.py View File

@@ -14,12 +14,12 @@
14 14
 # limitations under the License.
15 15
 
16 16
 import hashlib
17
+import json
17 18
 import os
18 19
 import random
19 20
 import socket
20 21
 import time
21 22
 
22
-import simplejson as json
23 23
 from nose import SkipTest
24 24
 from xml.dom import minidom
25 25
 

+ 180
- 0
test/functional/tests.py View File

@@ -327,6 +327,77 @@ class TestAccountNoContainersUTF8(Base2, TestAccountNoContainers):
327 327
     set_up = False
328 328
 
329 329
 
330
+class TestAccountSortingEnv(object):
331
+    @classmethod
332
+    def setUp(cls):
333
+        cls.conn = Connection(tf.config)
334
+        cls.conn.authenticate()
335
+        cls.account = Account(cls.conn, tf.config.get('account',
336
+                                                      tf.config['username']))
337
+        cls.account.delete_containers()
338
+
339
+        postfix = Utils.create_name()
340
+        cls.cont_items = ('a1', 'a2', 'A3', 'b1', 'B2', 'a10', 'b10', 'zz')
341
+        cls.cont_items = ['%s%s' % (x, postfix) for x in cls.cont_items]
342
+
343
+        for container in cls.cont_items:
344
+            c = cls.account.container(container)
345
+            if not c.create():
346
+                raise ResponseError(cls.conn.response)
347
+
348
+
349
+class TestAccountSorting(Base):
350
+    env = TestAccountSortingEnv
351
+    set_up = False
352
+
353
+    def testAccountContainerListSorting(self):
354
+        # name (byte order) sorting.
355
+        cont_list = sorted(self.env.cont_items)
356
+        cont_list.reverse()
357
+        cont_listing = self.env.account.containers(parms={'reverse': 'on'})
358
+        self.assert_status(200)
359
+        self.assertEqual(cont_list, cont_listing)
360
+
361
+    def testAccountContainerListSortingByPrefix(self):
362
+        cont_list = sorted(c for c in self.env.cont_items if c.startswith('a'))
363
+        cont_list.reverse()
364
+        cont_listing = self.env.account.containers(parms={
365
+            'reverse': 'on', 'prefix': 'a'})
366
+        self.assert_status(200)
367
+        self.assertEqual(cont_list, cont_listing)
368
+
369
+    def testAccountContainerListSortingByMarkersExclusive(self):
370
+        first_item = self.env.cont_items[3]  # 'b1' + postfix
371
+        last_item = self.env.cont_items[4]  # 'B2' + postfix
372
+
373
+        cont_list = sorted(c for c in self.env.cont_items
374
+                           if last_item < c < first_item)
375
+        cont_list.reverse()
376
+        cont_listing = self.env.account.containers(parms={
377
+            'reverse': 'on', 'marker': first_item, 'end_marker': last_item})
378
+        self.assert_status(200)
379
+        self.assertEqual(cont_list, cont_listing)
380
+
381
+    def testAccountContainerListSortingByMarkersInclusive(self):
382
+        first_item = self.env.cont_items[3]  # 'b1' + postfix
383
+        last_item = self.env.cont_items[4]  # 'B2' + postfix
384
+
385
+        cont_list = sorted(c for c in self.env.cont_items
386
+                           if last_item <= c <= first_item)
387
+        cont_list.reverse()
388
+        cont_listing = self.env.account.containers(parms={
389
+            'reverse': 'on', 'marker': first_item + '\x00',
390
+            'end_marker': last_item[:-1] + chr(ord(last_item[-1]) - 1)})
391
+        self.assert_status(200)
392
+        self.assertEqual(cont_list, cont_listing)
393
+
394
+    def testAccountContainerListSortingByReversedMarkers(self):
395
+        cont_listing = self.env.account.containers(parms={
396
+            'reverse': 'on', 'marker': 'B', 'end_marker': 'b1'})
397
+        self.assert_status(204)
398
+        self.assertEqual([], cont_listing)
399
+
400
+
330 401
 class TestContainerEnv(object):
331 402
     @classmethod
332 403
     def setUp(cls):
@@ -647,6 +718,115 @@ class TestContainerUTF8(Base2, TestContainer):
647 718
     set_up = False
648 719
 
649 720
 
721
+class TestContainerSortingEnv(object):
722
+    @classmethod
723
+    def setUp(cls):
724
+        cls.conn = Connection(tf.config)
725
+        cls.conn.authenticate()
726
+        cls.account = Account(cls.conn, tf.config.get('account',
727
+                                                      tf.config['username']))
728
+        cls.account.delete_containers()
729
+
730
+        cls.container = cls.account.container(Utils.create_name())
731
+        if not cls.container.create():
732
+            raise ResponseError(cls.conn.response)
733
+
734
+        cls.file_items = ('a1', 'a2', 'A3', 'b1', 'B2', 'a10', 'b10', 'zz')
735
+        cls.files = list()
736
+        cls.file_size = 128
737
+        for name in cls.file_items:
738
+            file_item = cls.container.file(name)
739
+            file_item.write_random(cls.file_size)
740
+            cls.files.append(file_item.name)
741
+
742
+
743
+class TestContainerSorting(Base):
744
+    env = TestContainerSortingEnv
745
+    set_up = False
746
+
747
+    def testContainerFileListSortingReversed(self):
748
+        file_list = list(sorted(self.env.file_items))
749
+        file_list.reverse()
750
+        cont_files = self.env.container.files(parms={'reverse': 'on'})
751
+        self.assert_status(200)
752
+        self.assertEqual(file_list, cont_files)
753
+
754
+    def testContainerFileSortingByPrefixReversed(self):
755
+        cont_list = sorted(c for c in self.env.file_items if c.startswith('a'))
756
+        cont_list.reverse()
757
+        cont_listing = self.env.container.files(parms={
758
+            'reverse': 'on', 'prefix': 'a'})
759
+        self.assert_status(200)
760
+        self.assertEqual(cont_list, cont_listing)
761
+
762
+    def testContainerFileSortingByMarkersExclusiveReversed(self):
763
+        first_item = self.env.file_items[3]  # 'b1' + postfix
764
+        last_item = self.env.file_items[4]  # 'B2' + postfix
765
+
766
+        cont_list = sorted(c for c in self.env.file_items
767
+                           if last_item < c < first_item)
768
+        cont_list.reverse()
769
+        cont_listing = self.env.container.files(parms={
770
+            'reverse': 'on', 'marker': first_item, 'end_marker': last_item})
771
+        self.assert_status(200)
772
+        self.assertEqual(cont_list, cont_listing)
773
+
774
+    def testContainerFileSortingByMarkersInclusiveReversed(self):
775
+        first_item = self.env.file_items[3]  # 'b1' + postfix
776
+        last_item = self.env.file_items[4]  # 'B2' + postfix
777
+
778
+        cont_list = sorted(c for c in self.env.file_items
779
+                           if last_item <= c <= first_item)
780
+        cont_list.reverse()
781
+        cont_listing = self.env.container.files(parms={
782
+            'reverse': 'on', 'marker': first_item + '\x00',
783
+            'end_marker': last_item[:-1] + chr(ord(last_item[-1]) - 1)})
784
+        self.assert_status(200)
785
+        self.assertEqual(cont_list, cont_listing)
786
+
787
+    def testContainerFileSortingByReversedMarkersReversed(self):
788
+        cont_listing = self.env.container.files(parms={
789
+            'reverse': 'on', 'marker': 'B', 'end_marker': 'b1'})
790
+        self.assert_status(204)
791
+        self.assertEqual([], cont_listing)
792
+
793
+    def testContainerFileListSorting(self):
794
+        file_list = list(sorted(self.env.file_items))
795
+        cont_files = self.env.container.files()
796
+        self.assert_status(200)
797
+        self.assertEqual(file_list, cont_files)
798
+
799
+        # Lets try again but with reverse is specifically turned off
800
+        cont_files = self.env.container.files(parms={'reverse': 'off'})
801
+        self.assert_status(200)
802
+        self.assertEqual(file_list, cont_files)
803
+
804
+        cont_files = self.env.container.files(parms={'reverse': 'false'})
805
+        self.assert_status(200)
806
+        self.assertEqual(file_list, cont_files)
807
+
808
+        cont_files = self.env.container.files(parms={'reverse': 'no'})
809
+        self.assert_status(200)
810
+        self.assertEqual(file_list, cont_files)
811
+
812
+        cont_files = self.env.container.files(parms={'reverse': ''})
813
+        self.assert_status(200)
814
+        self.assertEqual(file_list, cont_files)
815
+
816
+        # Lets try again but with a incorrect reverse values
817
+        cont_files = self.env.container.files(parms={'reverse': 'foo'})
818
+        self.assert_status(200)
819
+        self.assertEqual(file_list, cont_files)
820
+
821
+        cont_files = self.env.container.files(parms={'reverse': 'hai'})
822
+        self.assert_status(200)
823
+        self.assertEqual(file_list, cont_files)
824
+
825
+        cont_files = self.env.container.files(parms={'reverse': 'o=[]::::>'})
826
+        self.assert_status(200)
827
+        self.assertEqual(file_list, cont_files)
828
+
829
+
650 830
 class TestContainerPathsEnv(object):
651 831
     @classmethod
652 832
     def setUp(cls):

+ 210
- 0
test/unit/account/test_backend.py View File

@@ -29,6 +29,7 @@ import sqlite3
29 29
 import itertools
30 30
 from contextlib import contextmanager
31 31
 import random
32
+import mock
32 33
 
33 34
 from swift.account.backend import AccountBroker
34 35
 from swift.common.utils import Timestamp
@@ -65,6 +66,13 @@ class TestAccountBroker(unittest.TestCase):
65 66
             curs.execute('SELECT 1')
66 67
             self.assertEqual(curs.fetchall()[0][0], 1)
67 68
 
69
+    def test_initialize_fail(self):
70
+        broker = AccountBroker(':memory:')
71
+        with self.assertRaises(ValueError) as cm:
72
+            broker.initialize(Timestamp('1').internal)
73
+        self.assertEqual(str(cm.exception), 'Attempting to create a new'
74
+                         ' database with no account set')
75
+
68 76
     def test_exception(self):
69 77
         # Test AccountBroker throwing a conn away after exception
70 78
         first_conn = None
@@ -416,20 +424,48 @@ class TestAccountBroker(unittest.TestCase):
416 424
         self.assertEqual(listing[0][0], '0-0100')
417 425
         self.assertEqual(listing[-1][0], '0-0109')
418 426
 
427
+        listing = broker.list_containers_iter(10, '', None, '0-00', '-',
428
+                                              reverse=True)
429
+        self.assertEqual(len(listing), 10)
430
+        self.assertEqual(listing[0][0], '0-0099')
431
+        self.assertEqual(listing[-1][0], '0-0090')
432
+
419 433
         listing = broker.list_containers_iter(10, '', None, '0-', '-')
420 434
         self.assertEqual(len(listing), 10)
421 435
         self.assertEqual(listing[0][0], '0-0000')
422 436
         self.assertEqual(listing[-1][0], '0-0009')
423 437
 
438
+        listing = broker.list_containers_iter(10, '', None, '0-', '-',
439
+                                              reverse=True)
440
+        self.assertEqual(len(listing), 10)
441
+        self.assertEqual(listing[0][0], '0-0124')
442
+        self.assertEqual(listing[-1][0], '0-0115')
443
+
424 444
         listing = broker.list_containers_iter(10, '', None, '', '-')
425 445
         self.assertEqual(len(listing), 4)
426 446
         self.assertEqual([row[0] for row in listing],
427 447
                          ['0-', '1-', '2-', '3-'])
428 448
 
449
+        listing = broker.list_containers_iter(10, '', None, '', '-',
450
+                                              reverse=True)
451
+        self.assertEqual(len(listing), 4)
452
+        self.assertEqual([row[0] for row in listing],
453
+                         ['3-', '2-', '1-', '0-'])
454
+
429 455
         listing = broker.list_containers_iter(10, '2-', None, None, '-')
430 456
         self.assertEqual(len(listing), 1)
431 457
         self.assertEqual([row[0] for row in listing], ['3-'])
432 458
 
459
+        listing = broker.list_containers_iter(10, '2-', None, None, '-',
460
+                                              reverse=True)
461
+        self.assertEqual(len(listing), 2)
462
+        self.assertEqual([row[0] for row in listing], ['1-', '0-'])
463
+
464
+        listing = broker.list_containers_iter(10, '2.', None, None, '-',
465
+                                              reverse=True)
466
+        self.assertEqual(len(listing), 3)
467
+        self.assertEqual([row[0] for row in listing], ['2-', '1-', '0-'])
468
+
433 469
         listing = broker.list_containers_iter(10, '', None, '2', '-')
434 470
         self.assertEqual(len(listing), 1)
435 471
         self.assertEqual([row[0] for row in listing], ['2-'])
@@ -469,6 +505,168 @@ class TestAccountBroker(unittest.TestCase):
469 505
         self.assertEqual([row[0] for row in listing],
470 506
                          ['3-0049-', '3-0049-0049'])
471 507
 
508
+    def test_list_objects_iter_order_and_reverse(self):
509
+        # Test ContainerBroker.list_objects_iter
510
+        broker = AccountBroker(':memory:', account='a')
511
+        broker.initialize(Timestamp('1').internal, 0)
512
+
513
+        broker.put_container(
514
+            'c1', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
515
+        broker.put_container(
516
+            'c10', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
517
+        broker.put_container(
518
+            'C1', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
519
+        broker.put_container(
520
+            'c2', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
521
+        broker.put_container(
522
+            'c3', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
523
+        broker.put_container(
524
+            'C4', Timestamp(0).internal, 0, 0, 0, POLICIES.default.idx)
525
+
526
+        listing = broker.list_containers_iter(100, None, None, '', '',
527
+                                              reverse=False)
528
+        self.assertEqual([row[0] for row in listing],
529
+                         ['C1', 'C4', 'c1', 'c10', 'c2', 'c3'])
530
+        listing = broker.list_containers_iter(100, None, None, '', '',
531
+                                              reverse=True)
532
+        self.assertEqual([row[0] for row in listing],
533
+                         ['c3', 'c2', 'c10', 'c1', 'C4', 'C1'])
534
+        listing = broker.list_containers_iter(2, None, None, '', '',
535
+                                              reverse=True)
536
+        self.assertEqual([row[0] for row in listing],
537
+                         ['c3', 'c2'])
538
+        listing = broker.list_containers_iter(100, 'c2', 'C4', '', '',
539
+                                              reverse=True)
540
+        self.assertEqual([row[0] for row in listing],
541
+                         ['c10', 'c1'])
542
+
543
+    def test_reverse_prefix_delim(self):
544
+        expectations = [
545
+            {
546
+                'containers': [
547
+                    'topdir1-subdir1,0-c1',
548
+                    'topdir1-subdir1,1-c1',
549
+                    'topdir1-subdir1-c1',
550
+                ],
551
+                'params': {
552
+                    'prefix': 'topdir1-',
553
+                    'delimiter': '-',
554
+                },
555
+                'expected': [
556
+                    'topdir1-subdir1,0-',
557
+                    'topdir1-subdir1,1-',
558
+                    'topdir1-subdir1-',
559
+                ],
560
+            },
561
+            {
562
+                'containers': [
563
+                    'topdir1-subdir1,0-c1',
564
+                    'topdir1-subdir1,1-c1',
565
+                    'topdir1-subdir1-c1',
566
+                    'topdir1-subdir1.',
567
+                    'topdir1-subdir1.-c1',
568
+                ],
569
+                'params': {
570
+                    'prefix': 'topdir1-',
571
+                    'delimiter': '-',
572
+                },
573
+                'expected': [
574
+                    'topdir1-subdir1,0-',
575
+                    'topdir1-subdir1,1-',
576
+                    'topdir1-subdir1-',
577
+                    'topdir1-subdir1.',
578
+                    'topdir1-subdir1.-',
579
+                ],
580
+            },
581
+            {
582
+                'containers': [
583
+                    'topdir1-subdir1-c1',
584
+                    'topdir1-subdir1,0-c1',
585
+                    'topdir1-subdir1,1-c1',
586
+                ],
587
+                'params': {
588
+                    'prefix': 'topdir1-',
589
+                    'delimiter': '-',
590
+                    'reverse': True,
591
+                },
592
+                'expected': [
593
+                    'topdir1-subdir1-',
594
+                    'topdir1-subdir1,1-',
595
+                    'topdir1-subdir1,0-',
596
+                ],
597
+            },
598
+            {
599
+                'containers': [
600
+                    'topdir1-subdir1.-c1',
601
+                    'topdir1-subdir1.',
602
+                    'topdir1-subdir1-c1',
603
+                    'topdir1-subdir1-',
604
+                    'topdir1-subdir1,',
605
+                    'topdir1-subdir1,0-c1',
606
+                    'topdir1-subdir1,1-c1',
607
+                ],
608
+                'params': {
609
+                    'prefix': 'topdir1-',
610
+                    'delimiter': '-',
611
+                    'reverse': True,
612
+                },
613
+                'expected': [
614
+                    'topdir1-subdir1.-',
615
+                    'topdir1-subdir1.',
616
+                    'topdir1-subdir1-',
617
+                    'topdir1-subdir1,1-',
618
+                    'topdir1-subdir1,0-',
619
+                    'topdir1-subdir1,',
620
+                ],
621
+            },
622
+            {
623
+                'containers': [
624
+                    '1',
625
+                    '2',
626
+                    '3:1',
627
+                    '3:2:1',
628
+                    '3:2:2',
629
+                    '3:3',
630
+                    '4',
631
+                ],
632
+                'params': {
633
+                    'prefix': '3:',
634
+                    'delimiter': ':',
635
+                    'reverse': True,
636
+                },
637
+                'expected': [
638
+                    '3:3',
639
+                    '3:2:',
640
+                    '3:1',
641
+                ],
642
+            },
643
+        ]
644
+        ts = make_timestamp_iter()
645
+        default_listing_params = {
646
+            'limit': 10000,
647
+            'marker': '',
648
+            'end_marker': None,
649
+            'prefix': None,
650
+            'delimiter': None,
651
+        }
652
+        failures = []
653
+        for expected in expectations:
654
+            broker = AccountBroker(':memory:', account='a')
655
+            broker.initialize(next(ts).internal, 0)
656
+            for name in expected['containers']:
657
+                broker.put_container(name, next(ts).internal, 0, 0, 0,
658
+                                     POLICIES.default.idx)
659
+            params = default_listing_params.copy()
660
+            params.update(expected['params'])
661
+            listing = list(c[0] for c in broker.list_containers_iter(**params))
662
+            if listing != expected['expected']:
663
+                expected['listing'] = listing
664
+                failures.append(
665
+                    "With containers %(containers)r, the params %(params)r "
666
+                    "produced %(listing)r instead of %(expected)r" % expected)
667
+        self.assertFalse(failures, "Found the following failures:\n%s" %
668
+                         '\n'.join(failures))
669
+
472 670
     def test_double_check_trailing_delimiter(self):
473 671
         # Test AccountBroker.list_containers_iter for an
474 672
         # account that has an odd container with a trailing delimiter
@@ -1487,3 +1685,15 @@ class TestAccountBrokerBeforePerPolicyContainerTrack(
1487 1685
         self.assertEqual(len(policy_info), 2)
1488 1686
         for policy_stat in policy_info.values():
1489 1687
             self.assertEqual(policy_stat['container_count'], 1)
1688
+
1689
+    def test_migrate_add_storage_policy_index_fail(self):
1690
+        broker = AccountBroker(':memory:', account='a')
1691
+        broker.initialize(Timestamp('1').internal)
1692
+        with mock.patch.object(
1693
+                broker, 'create_policy_stat_table',
1694
+                side_effect=sqlite3.OperationalError('foobar')):
1695
+            with broker.get() as conn:
1696
+                self.assertRaisesRegexp(
1697
+                    sqlite3.OperationalError, '.*foobar.*',
1698
+                    broker._migrate_add_storage_policy_index,
1699
+                    conn=conn)

+ 103
- 13
test/unit/account/test_server.py View File

@@ -24,18 +24,19 @@ from test.unit import FakeLogger
24 24
 import itertools
25 25
 import random
26 26
 
27
-import simplejson
27
+import json
28 28
 from six import BytesIO
29 29
 from six import StringIO
30 30
 import xml.dom.minidom
31 31
 
32 32
 from swift import __version__ as swift_version
33
-from swift.common.swob import Request
33
+from swift.common.swob import (Request, WsgiBytesIO, HTTPNoContent)
34 34
 from swift.common import constraints
35 35
 from swift.account.server import AccountController
36
-from swift.common.utils import normalize_timestamp, replication, public
36
+from swift.common.utils import (normalize_timestamp, replication, public,
37
+                                mkdirs, storage_directory)
37 38
 from swift.common.request_helpers import get_sys_meta_prefix
38
-from test.unit import patch_policies
39
+from test.unit import patch_policies, debug_logger
39 40
 from swift.common.storage_policy import StoragePolicy, POLICIES
40 41
 
41 42
 
@@ -155,6 +156,49 @@ class TestAccountController(unittest.TestCase):
155 156
         resp = req.get_response(self.controller)
156 157
         self.assertEqual(resp.status_int, 507)
157 158
 
159
+    def test_REPLICATE_insufficient_storage(self):
160
+        conf = {'devices': self.testdir, 'mount_check': 'true'}
161
+        self.account_controller = AccountController(conf)
162
+
163
+        def fake_check_mount(*args, **kwargs):
164
+            return False
165
+
166
+        with mock.patch("swift.common.constraints.check_mount",
167
+                        fake_check_mount):
168
+            req = Request.blank('/sda1/p/suff',
169
+                                environ={'REQUEST_METHOD': 'REPLICATE'},
170
+                                headers={})
171
+            resp = req.get_response(self.account_controller)
172
+        self.assertEqual(resp.status_int, 507)
173
+
174
+    def test_REPLICATE_works(self):
175
+        mkdirs(os.path.join(self.testdir, 'sda1', 'account', 'p', 'a', 'a'))
176
+        db_file = os.path.join(self.testdir, 'sda1',
177
+                               storage_directory('account', 'p', 'a'),
178
+                               'a' + '.db')
179
+        open(db_file, 'w')
180
+
181
+        def fake_rsync_then_merge(self, drive, db_file, args):
182
+            return HTTPNoContent()
183
+
184
+        with mock.patch("swift.common.db_replicator.ReplicatorRpc."
185
+                        "rsync_then_merge", fake_rsync_then_merge):
186
+            req = Request.blank('/sda1/p/a/',
187
+                                environ={'REQUEST_METHOD': 'REPLICATE'},
188
+                                headers={})
189
+            json_string = '["rsync_then_merge", "a.db"]'
190
+            inbuf = WsgiBytesIO(json_string)
191
+            req.environ['wsgi.input'] = inbuf
192
+            resp = req.get_response(self.controller)
193
+        self.assertEqual(resp.status_int, 204)
194
+
195
+        # check valuerror
196
+        wsgi_input_valuerror = '["sync" : sync, "-1"]'
197
+        inbuf1 = WsgiBytesIO(wsgi_input_valuerror)
198
+        req.environ['wsgi.input'] = inbuf1
199
+        resp = req.get_response(self.controller)
200
+        self.assertEqual(resp.status_int, 400)
201
+
158 202
     def test_HEAD_not_found(self):
159 203
         # Test the case in which account does not exist (can be recreated)
160 204
         req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'HEAD'})
@@ -762,7 +806,7 @@ class TestAccountController(unittest.TestCase):
762 806
                             environ={'REQUEST_METHOD': 'GET'})
763 807
         resp = req.get_response(self.controller)
764 808
         self.assertEqual(resp.status_int, 200)
765
-        self.assertEqual(simplejson.loads(resp.body),
809
+        self.assertEqual(json.loads(resp.body),
766 810
                          [{'count': 0, 'bytes': 0, 'name': 'c1'},
767 811
                           {'count': 0, 'bytes': 0, 'name': 'c2'}])
768 812
         req = Request.blank('/sda1/p/a/c1', environ={'REQUEST_METHOD': 'PUT'},
@@ -783,7 +827,7 @@ class TestAccountController(unittest.TestCase):
783 827
                             environ={'REQUEST_METHOD': 'GET'})
784 828
         resp = req.get_response(self.controller)
785 829
         self.assertEqual(resp.status_int, 200)
786
-        self.assertEqual(simplejson.loads(resp.body),
830
+        self.assertEqual(json.loads(resp.body),
787 831
                          [{'count': 1, 'bytes': 2, 'name': 'c1'},
788 832
                           {'count': 3, 'bytes': 4, 'name': 'c2'}])
789 833
         self.assertEqual(resp.content_type, 'application/json')
@@ -987,7 +1031,7 @@ class TestAccountController(unittest.TestCase):
987 1031
                             environ={'REQUEST_METHOD': 'GET'})
988 1032
         resp = req.get_response(self.controller)
989 1033
         self.assertEqual(resp.status_int, 200)
990
-        self.assertEqual(simplejson.loads(resp.body),
1034
+        self.assertEqual(json.loads(resp.body),
991 1035
                          [{'count': 2, 'bytes': 3, 'name': 'c0'},
992 1036
                           {'count': 2, 'bytes': 3, 'name': 'c1'},
993 1037
                           {'count': 2, 'bytes': 3, 'name': 'c2'}])
@@ -995,7 +1039,7 @@ class TestAccountController(unittest.TestCase):
995 1039
                             environ={'REQUEST_METHOD': 'GET'})
996 1040
         resp = req.get_response(self.controller)
997 1041
         self.assertEqual(resp.status_int, 200)
998
-        self.assertEqual(simplejson.loads(resp.body),
1042
+        self.assertEqual(json.loads(resp.body),
999 1043
                          [{'count': 2, 'bytes': 3, 'name': 'c3'},
1000 1044
                           {'count': 2, 'bytes': 3, 'name': 'c4'}])
1001 1045
 
@@ -1108,7 +1152,7 @@ class TestAccountController(unittest.TestCase):
1108 1152
         req.accept = 'application/*'
1109 1153
         resp = req.get_response(self.controller)
1110 1154
         self.assertEqual(resp.status_int, 200)
1111
-        self.assertEqual(len(simplejson.loads(resp.body)), 1)
1155
+        self.assertEqual(len(json.loads(resp.body)), 1)
1112 1156
 
1113 1157
     def test_GET_accept_json(self):
1114 1158
         req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT',
@@ -1125,7 +1169,7 @@ class TestAccountController(unittest.TestCase):
1125 1169
         req.accept = 'application/json'
1126 1170
         resp = req.get_response(self.controller)
1127 1171
         self.assertEqual(resp.status_int, 200)
1128
-        self.assertEqual(len(simplejson.loads(resp.body)), 1)
1172
+        self.assertEqual(len(json.loads(resp.body)), 1)
1129 1173
 
1130 1174
     def test_GET_accept_xml(self):
1131 1175
         req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT',
@@ -1261,14 +1305,14 @@ class TestAccountController(unittest.TestCase):
1261 1305
         resp = req.get_response(self.controller)
1262 1306
         self.assertEqual(resp.status_int, 200)
1263 1307
         self.assertEqual([n.get('name', 's:' + n.get('subdir', 'error'))
1264
-                          for n in simplejson.loads(resp.body)], ['s:sub.'])
1308
+                          for n in json.loads(resp.body)], ['s:sub.'])
1265 1309
         req = Request.blank('/sda1/p/a?prefix=sub.&delimiter=.&format=json',
1266 1310
                             environ={'REQUEST_METHOD': 'GET'})
1267 1311
         resp = req.get_response(self.controller)
1268 1312
         self.assertEqual(resp.status_int, 200)
1269 1313
         self.assertEqual(
1270 1314
             [n.get('name', 's:' + n.get('subdir', 'error'))
1271
-             for n in simplejson.loads(resp.body)],
1315
+             for n in json.loads(resp.body)],
1272 1316
             ['sub.0', 's:sub.0.', 'sub.1', 's:sub.1.', 'sub.2', 's:sub.2.'])
1273 1317
         req = Request.blank('/sda1/p/a?prefix=sub.1.&delimiter=.&format=json',
1274 1318
                             environ={'REQUEST_METHOD': 'GET'})
@@ -1276,7 +1320,7 @@ class TestAccountController(unittest.TestCase):
1276 1320
         self.assertEqual(resp.status_int, 200)
1277 1321
         self.assertEqual(
1278 1322
             [n.get('name', 's:' + n.get('subdir', 'error'))
1279
-             for n in simplejson.loads(resp.body)],
1323
+             for n in json.loads(resp.body)],
1280 1324
             ['sub.1.0', 'sub.1.1', 'sub.1.2'])
1281 1325
 
1282 1326
     def test_GET_prefix_delimiter_xml(self):
@@ -1690,6 +1734,52 @@ class TestAccountController(unittest.TestCase):
1690 1734
             self.assertEqual(errbuf.getvalue(), '')
1691 1735
             self.assertEqual(outbuf.getvalue()[:4], '405 ')
1692 1736
 
1737
+    def test__call__raise_timeout(self):
1738
+        inbuf = WsgiBytesIO()
1739
+        errbuf = StringIO()
1740
+        outbuf = StringIO()
1741
+        self.logger = debug_logger('test')
1742
+        self.account_controller = AccountController(
1743
+            {'devices': self.testdir, 'mount_check': 'false',
1744
+             'replication_server': 'false', 'log_requests': 'false'},
1745
+            logger=self.logger)
1746
+
1747
+        def start_response(*args):
1748
+            # Sends args to outbuf
1749
+            outbuf.writelines(args)
1750
+
1751
+        method = 'PUT'
1752
+
1753
+        env = {'REQUEST_METHOD': method,
1754
+               'SCRIPT_NAME': '',
1755
+               'PATH_INFO': '/sda1/p/a/c',
1756
+               'SERVER_NAME': '127.0.0.1',
1757
+               'SERVER_PORT': '8080',
1758
+               'SERVER_PROTOCOL': 'HTTP/1.0',
1759
+               'CONTENT_LENGTH': '0',
1760
+               'wsgi.version': (1, 0),
1761
+               'wsgi.url_scheme': 'http',
1762
+               'wsgi.input': inbuf,
1763
+               'wsgi.errors': errbuf,
1764
+               'wsgi.multithread': False,
1765
+               'wsgi.multiprocess': False,
1766
+               'wsgi.run_once': False}
1767
+
1768
+        @public
1769
+        def mock_put_method(*args, **kwargs):
1770
+            raise Exception()
1771
+
1772
+        with mock.patch.object(self.account_controller, method,
1773
+                               new=mock_put_method):
1774
+            response = self.account_controller.__call__(env, start_response)
1775
+            self.assertTrue(response[0].startswith(
1776
+                'Traceback (most recent call last):'))
1777
+            self.assertEqual(self.logger.get_lines_for_level('error'), [
1778
+                'ERROR __call__ error with %(method)s %(path)s : ' % {
1779
+                    'method': 'PUT', 'path': '/sda1/p/a/c'},
1780
+            ])
1781
+            self.assertEqual(self.logger.get_lines_for_level('info'), [])
1782
+
1693 1783
     def test_GET_log_requests_true(self):
1694 1784
         self.controller.logger = FakeLogger()
1695 1785
         self.controller.log_requests = True

+ 78
- 8
test/unit/cli/test_ringbuilder.py View File

@@ -29,13 +29,19 @@ from swift.common.ring import RingBuilder
29 29
 
30 30
 class RunSwiftRingBuilderMixin(object):
31 31
 
32
-    def run_srb(self, *argv):
32
+    def run_srb(self, *argv, **kwargs):
33 33
         if len(argv) == 1 and isinstance(argv[0], six.string_types):
34 34
             # convert a single string to a list
35 35
             argv = shlex.split(argv[0])
36 36
         mock_stdout = six.StringIO()
37 37
         mock_stderr = six.StringIO()
38 38
 
39
+        if 'exp_results' in kwargs:
40
+            exp_results = kwargs['exp_results']
41
+            argv = argv[:-1]
42
+        else:
43
+            exp_results = None
44
+
39 45
         srb_args = ["", self.tempfile] + [str(s) for s in argv]
40 46
 
41 47
         try:
@@ -43,7 +49,13 @@ class RunSwiftRingBuilderMixin(object):
43 49
                 with mock.patch("sys.stderr", mock_stderr):
44 50
                     ringbuilder.main(srb_args)
45 51
         except SystemExit as err:
46
-            if err.code not in (0, 1):  # (success, warning)
52
+            valid_exit_codes = None
53
+            if exp_results is not None and 'valid_exit_codes' in exp_results:
54
+                valid_exit_codes = exp_results['valid_exit_codes']
55
+            else:
56
+                valid_exit_codes = (0, 1)  # (success, warning)
57
+
58
+            if err.code not in valid_exit_codes:
47 59
                 msg = 'Unexpected exit status %s\n' % err.code
48 60
                 msg += 'STDOUT:\n%s\nSTDERR:\n%s\n' % (
49 61
                     mock_stdout.getvalue(), mock_stderr.getvalue())
@@ -282,6 +294,11 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin):
282 294
         self.assertEqual(ring.replicas, 3.14159265359)
283 295
         self.assertEqual(ring.min_part_hours, 1)
284 296
 
297
+    def test_create_ring_number_of_arguments(self):
298
+        # Test missing arguments
299
+        argv = ["", self.tmpfile, "create"]
300
+        self.assertRaises(SystemExit, ringbuilder.main, argv)
301
+
285 302
     def test_add_device_ipv4_old_format(self):
286 303
         self.create_sample_ring()
287 304
         # Test ipv4(old format)
@@ -302,6 +319,14 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin):
302 319
         self.assertEqual(dev['replication_port'], 6000)
303 320
         self.assertEqual(dev['meta'], 'some meta data')
304 321
 
322
+    def test_add_duplicate_devices(self):
323
+        self.create_sample_ring()
324
+        # Test adding duplicate devices
325
+        argv = ["", self.tmpfile, "add",
326
+                "r1z1-127.0.0.1:6000/sda9", "3.14159265359",
327
+                "r1z1-127.0.0.1:6000/sda9", "2"]
328
+        self.assertRaises(SystemExit, ringbuilder.main, argv)
329
+
305 330
     def test_add_device_ipv6_old_format(self):
306 331
         self.create_sample_ring()
307 332
         # Test ipv6(old format)
@@ -439,6 +464,18 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin):
439 464
             err = e
440 465
         self.assertEqual(err.code, 2)
441 466
 
467
+    def test_add_device_old_missing_region(self):
468
+        self.create_sample_ring()
469
+        # Test add device without specifying a region
470
+        argv = ["", self.tmpfile, "add",
471
+                "z3-127.0.0.1:6000/sde3_some meta data", "3.14159265359"]
472
+        exp_results = {'valid_exit_codes': [2]}
473
+        self.run_srb(*argv, exp_results=exp_results)
474
+        # Check that ring was created with sane value for region
475
+        ring = RingBuilder.load(self.tmpfile)
476
+        dev = ring.devs[-1]
477
+        self.assertTrue(dev['region'] > 0)
478
+
442 479
     def test_remove_device(self):
443 480
         for search_value in self.search_values:
444 481
             self.create_sample_ring()
@@ -700,6 +737,7 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin):
700 737
         self.assertTrue(ring.validate())
701 738
 
702 739
     def test_remove_device_number_of_arguments(self):
740
+        self.create_sample_ring()
703 741
         # Test Number of arguments abnormal
704 742
         argv = ["", self.tmpfile, "remove"]
705 743
         err = None
@@ -911,6 +949,7 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin):
911 949
         self.assertTrue(ring.validate())
912 950
 
913 951
     def test_set_weight_number_of_arguments(self):
952
+        self.create_sample_ring()
914 953
         # Test Number of arguments abnormal
915 954
         argv = ["", self.tmpfile, "set_weight"]
916 955
         err = None
@@ -1201,6 +1240,7 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin):
1201 1240
         self.assertTrue(ring.validate())
1202 1241
 
1203 1242
     def test_set_info_number_of_arguments(self):
1243
+        self.create_sample_ring()
1204 1244
         # Test Number of arguments abnormal
1205 1245
         argv = ["", self.tmpfile, "set_info"]
1206 1246
         err = None
@@ -1253,6 +1293,7 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin):
1253 1293
         self.assertEqual(ring.min_part_hours, 24)
1254 1294
 
1255 1295
     def test_set_min_part_hours_number_of_arguments(self):
1296
+        self.create_sample_ring()
1256 1297
         # Test Number of arguments abnormal
1257 1298
         argv = ["", self.tmpfile, "set_min_part_hours"]
1258 1299
         err = None
@@ -1326,7 +1367,14 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin):
1326 1367
         self.assertTrue('1000.00%' in out)
1327 1368
         self.assertTrue('10.000000' in out)
1328 1369
 
1370
+    def test_set_overload_number_of_arguments(self):
1371
+        self.create_sample_ring()
1372
+        # Test missing arguments
1373
+        argv = ["", self.tmpfile, "set_overload"]
1374
+        self.assertRaises(SystemExit, ringbuilder.main, argv)
1375
+
1329 1376
     def test_set_replicas_number_of_arguments(self):
1377
+        self.create_sample_ring()
1330 1378
         # Test Number of arguments abnormal
1331 1379
         argv = ["", self.tmpfile, "set_replicas"]
1332 1380
         err = None
@@ -1337,6 +1385,7 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin):
1337 1385
         self.assertEqual(err.code, 2)
1338 1386
 
1339 1387
     def test_set_replicas_invalid_value(self):
1388
+        self.create_sample_ring()
1340 1389
         # Test not a valid number
1341 1390
         argv = ["", self.tmpfile, "set_replicas", "test"]
1342 1391
         err = None
@@ -1516,6 +1565,7 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin):
1516 1565
         self.assertRaises(SystemExit, ringbuilder.main, argv)
1517 1566
 
1518 1567
     def test_search_device_number_of_arguments(self):
1568
+        self.create_sample_ring()
1519 1569
         # Test Number of arguments abnormal
1520 1570
         argv = ["", self.tmpfile, "search"]
1521 1571
         err = None
@@ -1628,6 +1678,7 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin):
1628 1678
         self.assertRaises(SystemExit, ringbuilder.main, argv)
1629 1679
 
1630 1680
     def test_list_parts_number_of_arguments(self):
1681
+        self.create_sample_ring()
1631 1682
         # Test Number of arguments abnormal
1632 1683
         argv = ["", self.tmpfile, "list_parts"]
1633 1684
         err = None
@@ -1650,6 +1701,7 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin):
1650 1701
         self.assertEqual(err.code, 2)
1651 1702
 
1652 1703
     def test_unknown(self):
1704
+        self.create_sample_ring()
1653 1705
         argv = ["", self.tmpfile, "unknown"]
1654 1706
         err = None
1655 1707
         try:
@@ -1754,6 +1806,12 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin):
1754 1806
         self.assertTrue(ring.validate())
1755 1807
         self.assertEqual(ring.devs[3], None)
1756 1808
 
1809
+    def test_rebalance_with_seed(self):
1810
+        self.create_sample_ring()
1811
+        # Test rebalance using explicit seed parameter
1812
+        argv = ["", self.tmpfile, "rebalance", "--seed", "2"]
1813
+        self.assertRaises(SystemExit, ringbuilder.main, argv)
1814
+
1757 1815
     def test_write_ring(self):
1758 1816
         self.create_sample_ring()
1759 1817
         argv = ["", self.tmpfile, "rebalance"]
@@ -1768,12 +1826,8 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin):
1768 1826
         argv = ["", self.tmpfile, "rebalance"]
1769 1827
         self.assertRaises(SystemExit, ringbuilder.main, argv)
1770 1828
         argv = ["", self.tmpfile, "write_builder"]
1771
-        err = None
1772
-        try:
1773
-            ringbuilder.main(argv)
1774
-        except SystemExit as e:
1775
-            err = e
1776
-        self.assertEqual(err.code, 2)
1829
+        exp_results = {'valid_exit_codes': [2]}
1830
+        self.run_srb(*argv, exp_results=exp_results)
1777 1831
 
1778 1832
     def test_write_builder_after_device_removal(self):
1779 1833
         # Test regenerating builder file after having removed a device
@@ -1896,6 +1950,22 @@ class TestCommands(unittest.TestCase, RunSwiftRingBuilderMixin):
1896 1950
             "Ring Builder file does not exist: object.builder\n"
1897 1951
         self.assertEqual(expected, mock_stdout.getvalue())
1898 1952
 
1953
+    def test_main_no_arguments(self):
1954
+        # Test calling main with no arguments
1955
+        argv = []
1956
+        self.assertRaises(SystemExit, ringbuilder.main, argv)
1957
+
1958
+    def test_main_single_argument(self):
1959
+        # Test calling main with single argument
1960
+        argv = [""]
1961
+        self.assertRaises(SystemExit, ringbuilder.main, argv)
1962
+
1963
+    def test_main_with_safe(self):
1964
+        # Test calling main with '-safe' argument
1965
+        self.create_sample_ring()
1966
+        argv = ["-safe", self.tmpfile]
1967
+        self.assertRaises(SystemExit, ringbuilder.main, argv)
1968
+
1899 1969
 
1900 1970
 class TestRebalanceCommand(unittest.TestCase, RunSwiftRingBuilderMixin):
1901 1971
 

+ 8
- 27
test/unit/common/middleware/test_keystoneauth.py View File

@@ -582,7 +582,7 @@ class BaseTestAuthorize(unittest.TestCase):
582 582
                                  user_domain_id)
583 583
         token_info = _fake_token_info(version=auth_version)
584 584
         env.update({'keystone.token_info': token_info})
585
-        return self.test_auth._integral_keystone_identity(env)
585
+        return self.test_auth._keystone_identity(env)
586 586
 
587 587
 
588 588
 class TestAuthorize(BaseTestAuthorize):
@@ -604,7 +604,7 @@ class TestAuthorize(BaseTestAuthorize):
604 604
         req = self._make_request(path, headers=headers, environ=default_env)
605 605
         req.acl = acl
606 606
 
607
-        env_identity = self.test_auth._integral_keystone_identity(req.environ)
607
+        env_identity = self.test_auth._keystone_identity(req.environ)
608 608
         result = self.test_auth.authorize(env_identity, req)
609 609
 
610 610
         # if we have requested an exception but nothing came back then
@@ -912,25 +912,6 @@ class TestAuthorize(BaseTestAuthorize):
912 912
         self._check_authenticate(acl=acl, identity=id, env=env)
913 913
 
914 914
     def test_keystone_identity(self):
915
-        user_name = 'U_NAME'
916
-        project = ('P_ID', 'P_NAME')
917
-        roles = ('ROLE1', 'ROLE2')
918
-
919
-        req = Request.blank('/v/a/c/o')
920
-        req.headers.update({'X-Identity-Status': 'Confirmed',
921
-                            'X-Roles': ' %s , %s ' % roles,
922
-                            'X-User-Name': user_name,
923
-                            'X-Tenant-Id': project[0],
924
-                            'X-Tenant-Name': project[1]})
925
-
926
-        expected = {'user': user_name,
927
-                    'tenant': project,
928
-                    'roles': list(roles)}
929
-        data = self.test_auth._keystone_identity(req.environ)
930
-
931
-        self.assertEqual(expected, data)
932
-
933
-    def test_integral_keystone_identity(self):
934 915
         user = ('U_ID', 'U_NAME')
935 916
         roles = ('ROLE1', 'ROLE2')
936 917
         service_roles = ('ROLE3', 'ROLE4')
@@ -940,7 +921,7 @@ class TestAuthorize(BaseTestAuthorize):
940 921
 
941 922
         # no valid identity info in headers
942 923
         req = Request.blank('/v/a/c/o')
943
-        data = self.test_auth._integral_keystone_identity(req.environ)
924
+        data = self.test_auth._keystone_identity(req.environ)
944 925
         self.assertEqual(None, data)
945 926
 
946 927
         # valid identity info in headers, but status unconfirmed
@@ -954,7 +935,7 @@ class TestAuthorize(BaseTestAuthorize):
954 935
                             'X-User-Domain-Name': user_domain[1],
955 936
                             'X-Project-Domain-Id': project_domain[0],
956 937
                             'X-Project-Domain-Name': project_domain[1]})
957
-        data = self.test_auth._integral_keystone_identity(req.environ)
938
+        data = self.test_auth._keystone_identity(req.environ)
958 939
         self.assertEqual(None, data)
959 940
 
960 941
         # valid identity info in headers, no token info in environ
@@ -966,7 +947,7 @@ class TestAuthorize(BaseTestAuthorize):
966 947
                     'user_domain': (None, None),
967 948
                     'project_domain': (None, None),
968 949
                     'auth_version': 0}
969
-        data = self.test_auth._integral_keystone_identity(req.environ)
950
+        data = self.test_auth._keystone_identity(req.environ)
970 951
         self.assertEqual(expected, data)
971 952
 
972 953
         # v2 token info in environ
@@ -978,7 +959,7 @@ class TestAuthorize(BaseTestAuthorize):
978 959
                     'user_domain': (None, None),
979 960
                     'project_domain': (None, None),
980 961
                     'auth_version': 2}
981
-        data = self.test_auth._integral_keystone_identity(req.environ)
962
+        data = self.test_auth._keystone_identity(req.environ)
982 963
         self.assertEqual(expected, data)
983 964
 
984 965
         # v3 token info in environ
@@ -990,7 +971,7 @@ class TestAuthorize(BaseTestAuthorize):
990 971
                     'user_domain': user_domain,
991 972
                     'project_domain': project_domain,
992 973
                     'auth_version': 3}
993
-        data = self.test_auth._integral_keystone_identity(req.environ)
974
+        data = self.test_auth._keystone_identity(req.environ)
994 975
         self.assertEqual(expected, data)
995 976
 
996 977
         # service token in environ
@@ -1002,7 +983,7 @@ class TestAuthorize(BaseTestAuthorize):
1002 983
                     'user_domain': user_domain,
1003 984
                     'project_domain': project_domain,
1004 985
                     'auth_version': 3}
1005
-        data = self.test_auth._integral_keystone_identity(req.environ)
986
+        data = self.test_auth._keystone_identity(req.environ)
1006 987
         self.assertEqual(expected, data)
1007 988
 
1008 989
     def test_get_project_domain_id(self):

+ 8
- 7
test/unit/common/middleware/test_ratelimit.py View File

@@ -18,7 +18,6 @@ import time
18 18
 import eventlet
19 19
 import mock
20 20
 from contextlib import contextmanager
21
-from threading import Thread
22 21
 
23 22
 from test.unit import FakeLogger
24 23
 from swift.common.middleware import ratelimit
@@ -28,6 +27,8 @@ from swift.common.memcached import MemcacheConnectionError
28 27
 from swift.common.swob import Request
29 28
 from swift.common import utils
30 29
 
30
+threading = eventlet.patcher.original('threading')
31
+
31 32
 
32 33
 class FakeMemcache(object):
33 34
 
@@ -313,10 +314,10 @@ class TestRateLimit(unittest.TestCase):
313 314
         req = Request.blank('/v/a/c')
314 315
         req.environ['swift.cache'] = FakeMemcache()
315 316
 
316
-        class rate_caller(Thread):
317
+        class rate_caller(threading.Thread):
317 318
 
318 319
             def __init__(self, parent):
319
-                Thread.__init__(self)
320
+                threading.Thread.__init__(self)
320 321
                 self.parent = parent
321 322
 
322 323
             def run(self):
@@ -356,10 +357,10 @@ class TestRateLimit(unittest.TestCase):
356 357
         req = Request.blank('/v/b/c')
357 358
         req.environ['swift.cache'] = FakeMemcache()
358 359
 
359
-        class rate_caller(Thread):
360
+        class rate_caller(threading.Thread):
360 361
 
361 362
             def __init__(self, parent):
362
-                Thread.__init__(self)
363
+                threading.Thread.__init__(self)
363 364
                 self.parent = parent
364 365
 
365 366
             def run(self):
@@ -505,11 +506,11 @@ class TestRateLimit(unittest.TestCase):
505 506
         req.method = 'PUT'
506 507
         req.environ = {}
507 508
 
508
-        class rate_caller(Thread):
509
+        class rate_caller(threading.Thread):
509 510
 
510 511
             def __init__(self, name):
511 512
                 self.myname = name
512
-                Thread.__init__(self)
513
+                threading.Thread.__init__(self)
513 514
 
514 515
             def run(self):
515 516
                 for j in range(num_calls):

+ 27
- 6
test/unit/common/middleware/test_slo.py View File

@@ -17,6 +17,7 @@
17 17
 from six.moves import range
18 18
 
19 19
 import hashlib
20
+import json
20 21
 import time
21 22
 import unittest
22 23
 from mock import patch
@@ -25,17 +26,17 @@ from swift.common import swob, utils
25 26
 from swift.common.exceptions import ListingIterError, SegmentError
26 27
 from swift.common.middleware import slo
27 28
 from swift.common.swob import Request, Response, HTTPException
28
-from swift.common.utils import quote, json, closing_if_possible
29
+from swift.common.utils import quote, closing_if_possible
29 30
 from test.unit.common.middleware.helpers import FakeSwift
30 31
 
31 32
 
32 33
 test_xml_data = '''<?xml version="1.0" encoding="UTF-8"?>
33 34
 <static_large_object>
34
-<object_segment>
35
-<path>/cont/object</path>
36
-<etag>etagoftheobjectsegment</etag>
37
-<size_bytes>100</size_bytes>
38
-</object_segment>
35
+  <object_segment>
36
+    <path>/cont/object</path>
37
+    <etag>etagoftheobjectsegment</etag>
38
+    <size_bytes>100</size_bytes>
39
+  </object_segment>
39 40
 </static_large_object>
40 41
 '''
41 42
 test_json_data = json.dumps([{'path': '/cont/object',
@@ -1034,6 +1035,26 @@ class TestSloDeleteManifest(SloTestCase):
1034 1035
         self.assertEqual(resp_data['Errors'],
1035 1036
                          [['/deltest-unauth/q_17', '401 Unauthorized']])
1036 1037
 
1038
+    def test_handle_multipart_delete_client_content_type(self):
1039
+        req = Request.blank(
1040
+            '/v1/AUTH_test/deltest/man-all-there?multipart-manifest=delete',
1041
+            environ={'REQUEST_METHOD': 'DELETE', 'CONTENT_TYPE': 'foo/bar'},
1042
+            headers={'Accept': 'application/json'})
1043
+        status, headers, body = self.call_slo(req)
1044
+
1045
+        self.assertEqual(status, '200 OK')
1046
+        resp_data = json.loads(body)
1047
+        self.assertEqual(resp_data["Number Deleted"], 3)
1048
+
1049
+        self.assertEqual(
1050
+            self.app.calls,
1051
+            [('GET',
1052
+              '/v1/AUTH_test/deltest/man-all-there?multipart-manifest=get'),
1053
+             ('DELETE', '/v1/AUTH_test/deltest/b_2?multipart-manifest=delete'),
1054
+             ('DELETE', '/v1/AUTH_test/deltest/c_3?multipart-manifest=delete'),
1055
+             ('DELETE', ('/v1/AUTH_test/deltest/' +
1056
+                         'man-all-there?multipart-manifest=delete'))])
1057
+
1037 1058
 
1038 1059
 class TestSloHeadManifest(SloTestCase):
1039 1060
     def setUp(self):

+ 0
- 5
test/unit/common/test_bufferedhttp.py View File

@@ -111,11 +111,6 @@ class TestBufferedHTTP(unittest.TestCase):
111 111
             bufferedhttp.HTTPSConnection = origHTTPSConnection
112 112
 
113 113
     def test_unicode_values(self):
114
-        # simplejson may decode the ring devices as str or unicode
115
-        # depending on whether speedups is installed and/or the values are
116
-        # non-ascii. Verify all types are tolerated in combination with
117
-        # whatever type path might be and possible encoded non-ascii in
118
-        # a header value.
119 114
         with mock.patch('swift.common.bufferedhttp.HTTPSConnection',
120 115
                         MockHTTPSConnection):
121 116
             for dev in ('sda', u'sda', u'sdá', u'sdá'.encode('utf-8')):

+ 3
- 3
test/unit/common/test_db.py View File

@@ -23,7 +23,7 @@ from shutil import rmtree, copy
23 23
 from uuid import uuid4
24 24
 import six.moves.cPickle as pickle
25 25
 
26
-import simplejson
26
+import json
27 27
 import sqlite3
28 28
 import itertools
29 29
 import time
@@ -39,7 +39,7 @@ from swift.common.constraints import \
39 39
 from swift.common.db import chexor, dict_factory, get_db_connection, \
40 40
     DatabaseBroker, DatabaseConnectionError, DatabaseAlreadyExists, \
41 41
     GreenDBConnection, PICKLE_PROTOCOL
42
-from swift.common.utils import normalize_timestamp, mkdirs, json, Timestamp
42
+from swift.common.utils import normalize_timestamp, mkdirs, Timestamp
43 43
 from swift.common.exceptions import LockTimeout
44 44
 from swift.common.swob import HTTPException
45 45
 
@@ -963,7 +963,7 @@ class TestDatabaseBroker(unittest.TestCase):
963 963
         broker.db_contains_type = 'test'
964 964
         broker_creation = normalize_timestamp(1)
965 965
         broker_uuid = str(uuid4())
966
-        broker_metadata = metadata and simplejson.dumps(
966
+        broker_metadata = metadata and json.dumps(
967 967
             {'Test': ('Value', normalize_timestamp(1))}) or ''
968 968
 
969 969
         def _initialize(conn, put_timestamp, **kwargs):

+ 5
- 5
test/unit/common/test_db_replicator.py View File

@@ -25,7 +25,7 @@ from mock import patch, call
25 25
 from shutil import rmtree, copy
26 26
 from tempfile import mkdtemp, NamedTemporaryFile
27 27
 import mock
28
-import simplejson
28
+import json
29 29
 
30 30
 from swift.container.backend import DATADIR
31 31
 from swift.common import db_replicator
@@ -1207,7 +1207,7 @@ class TestReplToNode(unittest.TestCase):
1207 1207
 
1208 1208
     def test_repl_to_node_usync_success(self):
1209 1209
         rinfo = {"id": 3, "point": -1, "max_row": 10, "hash": "c"}
1210
-        self.http = ReplHttp(simplejson.dumps(rinfo))
1210
+        self.http = ReplHttp(json.dumps(rinfo))
1211 1211
         local_sync = self.broker.get_sync()
1212 1212
         self.assertEqual(self.replicator._repl_to_node(
1213 1213
             self.fake_node, self.broker, '0', self.fake_info), True)
@@ -1218,7 +1218,7 @@ class TestReplToNode(unittest.TestCase):
1218 1218
 
1219 1219
     def test_repl_to_node_rsync_success(self):
1220 1220
         rinfo = {"id": 3, "point": -1, "max_row": 9, "hash": "c"}
1221
-        self.http = ReplHttp(simplejson.dumps(rinfo))
1221
+        self.http = ReplHttp(json.dumps(rinfo))
1222 1222
         self.broker.get_sync()
1223 1223
         self.assertEqual(self.replicator._repl_to_node(
1224 1224
             self.fake_node, self.broker, '0', self.fake_info), True)
@@ -1235,7 +1235,7 @@ class TestReplToNode(unittest.TestCase):
1235 1235
 
1236 1236
     def test_repl_to_node_already_in_sync(self):
1237 1237
         rinfo = {"id": 3, "point": -1, "max_row": 20, "hash": "b"}
1238
-        self.http = ReplHttp(simplejson.dumps(rinfo))
1238
+        self.http = ReplHttp(json.dumps(rinfo))
1239 1239
         self.broker.get_sync()
1240 1240
         self.assertEqual(self.replicator._repl_to_node(
1241 1241
             self.fake_node, self.broker, '0', self.fake_info), True)
@@ -1282,7 +1282,7 @@ class TestReplToNode(unittest.TestCase):
1282 1282
             rinfo['max_row'] = r
1283 1283
             self.fake_info['max_row'] = l
1284 1284
             self.replicator._usync_db = mock.Mock(return_value=True)
1285
-            self.http = ReplHttp(simplejson.dumps(rinfo))
1285
+            self.http = ReplHttp(json.dumps(rinfo))
1286 1286
             local_sync = self.broker.get_sync()
1287 1287
             self.assertEqual(self.replicator._repl_to_node(
1288 1288
                 self.fake_node, self.broker, '0', self.fake_info), True)

+ 108
- 2
test/unit/common/test_direct_client.py View File

@@ -29,6 +29,7 @@ from swift.common.exceptions import ClientException
29 29
 from swift.common.utils import Timestamp
30 30
 from swift.common.swob import HeaderKeyDict, RESPONSE_REASONS
31 31
 from swift.common.storage_policy import POLICIES
32
+from six.moves.http_client import HTTPException
32 33
 
33 34
 from test.unit import patch_policies
34 35
 
@@ -68,8 +69,13 @@ class FakeConn(object):
68 69
     def getheaders(self):
69 70
         return self.resp_headers.items()
70 71
 
71
-    def read(self):
72
-        return self.body
72
+    def read(self, amt=None):
73
+        if amt is None:
74
+            return self.body
75
+        elif isinstance(self.body, six.StringIO):
76
+            return self.body.read(amt)
77
+        else:
78
+            return Exception('Not a StringIO entry')
73 79
 
74 80
     def send(self, data):