Browse Source

Merge remote-tracking branch 'remotes/origin/master' into merge-master

Include a trivial whitespace change in .zuul.yaml to make sure Zuul
realizes it changed on master.

Change-Id: I24069547265db9ebd39e565f966f4d0b82c0aea2
changes/45/648245/2
Romain LE DISEZ 6 months ago
parent
commit
6afc1130fd
53 changed files with 1525 additions and 860 deletions
  1. 1
    1
      .mailmap
  2. 30
    1
      .zuul.yaml
  3. 5
    1
      AUTHORS
  4. 58
    0
      CHANGELOG
  5. 1
    1
      doc/saio/swift/object-expirer.conf
  6. 322
    289
      doc/source/development_saio.rst
  7. 2
    1
      etc/proxy-server.conf-sample
  8. 69
    0
      releasenotes/notes/2_21_0_release-d8ae33ef18b7be3a.yaml
  9. 2
    0
      releasenotes/source/index.rst
  10. 6
    0
      releasenotes/source/stein.rst
  11. 40
    50
      swift/account/reaper.py
  12. 6
    1
      swift/cli/manage_shard_ranges.py
  13. 8
    0
      swift/common/bufferedhttp.py
  14. 14
    10
      swift/common/middleware/bulk.py
  15. 9
    10
      swift/common/middleware/copy.py
  16. 32
    6
      swift/common/middleware/crypto/keymaster.py
  17. 2
    1
      swift/common/middleware/dlo.py
  18. 17
    4
      swift/common/middleware/formpost.py
  19. 1
    1
      swift/common/middleware/s3api/controllers/s3_acl.py
  20. 53
    7
      swift/common/middleware/s3api/s3request.py
  21. 19
    0
      swift/common/middleware/s3api/s3token.py
  22. 5
    5
      swift/common/middleware/slo.py
  23. 4
    1
      swift/common/middleware/tempauth.py
  24. 7
    9
      swift/common/request_helpers.py
  25. 44
    0
      swift/common/swob.py
  26. 1
    2
      swift/common/utils.py
  27. 14
    31
      swift/common/wsgi.py
  28. 1
    1
      swift/container/backend.py
  29. 7
    48
      swift/obj/diskfile.py
  30. 3
    0
      swift/obj/mem_diskfile.py
  31. 16
    41
      swift/proxy/controllers/obj.py
  32. 13
    4
      test/functional/__init__.py
  33. 3
    2
      test/functional/test_dlo.py
  34. 33
    1
      test/functional/test_object.py
  35. 13
    0
      test/functional/test_slo.py
  36. 10
    5
      test/functional/tests.py
  37. 2
    2
      test/probe/test_replication_servers_working.py
  38. 12
    9
      test/unit/account/test_reaper.py
  39. 72
    0
      test/unit/common/middleware/crypto/test_keymaster.py
  40. 3
    3
      test/unit/common/middleware/s3api/test_multi_upload.py
  41. 54
    0
      test/unit/common/middleware/s3api/test_obj.py
  42. 75
    1
      test/unit/common/middleware/s3api/test_s3request.py
  43. 23
    0
      test/unit/common/middleware/test_copy.py
  44. 220
    207
      test/unit/common/middleware/test_formpost.py
  45. 30
    0
      test/unit/common/middleware/test_tempauth.py
  46. 0
    51
      test/unit/common/test_wsgi.py
  47. 23
    11
      test/unit/helpers.py
  48. 29
    4
      test/unit/obj/test_diskfile.py
  49. 21
    2
      test/unit/obj/test_reconstructor.py
  50. 58
    4
      test/unit/proxy/controllers/test_obj.py
  51. 27
    30
      test/unit/proxy/test_server.py
  52. 1
    2
      test/unit/proxy/test_sysmeta.py
  53. 4
    0
      tox.ini

+ 1
- 1
.mailmap View File

@@ -79,7 +79,7 @@ Minwoo Bae <minwoob@us.ibm.com> Minwoo B
79 79
 Jaivish Kothari <jaivish.kothari@nectechnologies.in> <janonymous.codevulture@gmail.com>
80 80
 Michael Matur <michael.matur@gmail.com>
81 81
 Kazuhiro Miyahara <miyahara.kazuhiro@lab.ntt.co.jp>
82
-Alexandra Settle <alexandra.settle@rackspace.com>
82
+Alexandra Settle <asettle@suse.com> <alexandra.settle@rackspace.com>
83 83
 Kenichiro Matsuda <matsuda_kenichi@jp.fujitsu.com>
84 84
 Atsushi Sakai <sakaia@jp.fujitsu.com>
85 85
 Takashi Natsume <natsume.takashi@lab.ntt.co.jp>

+ 30
- 1
.zuul.yaml View File

@@ -35,6 +35,7 @@
35 35
 - job:
36 36
     name: swift-tox-py35
37 37
     parent: swift-tox-base
38
+    nodeset: ubuntu-xenial
38 39
     description: |
39 40
       Run unit-tests for swift under cPython version 3.5.
40 41
 
@@ -67,6 +68,25 @@
67 68
         NOSE_COVER_HTML_DIR: '{toxinidir}/cover'
68 69
     post-run: tools/playbooks/common/cover-post.yaml
69 70
 
71
+- job:
72
+    name: swift-tox-py37
73
+    parent: swift-tox-base
74
+    nodeset: ubuntu-bionic
75
+    description: |
76
+      Run unit-tests for swift under cPython version 3.7.
77
+
78
+      Uses tox with the ``py37`` environment.
79
+      It sets TMPDIR to an XFS mount point created via
80
+      tools/test-setup.sh.
81
+    vars:
82
+      tox_envlist: py37
83
+      bindep_profile: test py37
84
+      python_version: 3.7
85
+      tox_environment:
86
+        NOSE_COVER_HTML: 1
87
+        NOSE_COVER_HTML_DIR: '{toxinidir}/cover'
88
+    post-run: tools/playbooks/common/cover-post.yaml
89
+
70 90
 - job:
71 91
     name: swift-tox-func
72 92
     parent: swift-tox-base
@@ -316,6 +336,11 @@
316 336
               - ^(api-ref|doc|releasenotes)/.*$
317 337
               - ^test/(functional|probe)/.*$
318 338
             voting: false
339
+        - swift-tox-py37:
340
+            irrelevant-files:
341
+              - ^(api-ref|doc|releasenotes)/.*$
342
+              - ^test/(functional|probe)/.*$
343
+            voting: false
319 344
         - swift-tox-func:
320 345
             irrelevant-files:
321 346
               - ^(api-ref|doc|releasenotes)/.*$
@@ -344,7 +369,10 @@
344 369
         - swift-tox-func-s3api-ceph-s3tests-tempauth:
345 370
             irrelevant-files:
346 371
               - ^(api-ref|releasenotes)/.*$
347
-              - ^test/probe/.*$
372
+              # Keep doc/saio -- we use those sample configs in the saio playbooks
373
+              # Also keep doc/s3api -- it holds known failures for these tests
374
+              - ^doc/(requirements.txt|(manpages|source)/.*)$
375
+              - ^test/(unit|probe)/.*$
348 376
               - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
349 377
         - swift-probetests-centos-7:
350 378
             irrelevant-files:
@@ -428,3 +456,4 @@
428 456
     post:
429 457
       jobs:
430 458
         - publish-openstack-python-branch-tarball
459
+

+ 5
- 1
AUTHORS View File

@@ -45,7 +45,7 @@ Alex Holden (alex@alexjonasholden.com)
45 45
 Alex Pecoraro (alex.pecoraro@emc.com)
46 46
 Alex Szarka (szarka@inf.u-szeged.hu)
47 47
 Alex Yang (alex890714@gmail.com)
48
-Alexandra Settle (alexandra.settle@rackspace.com)
48
+Alexandra Settle (asettle@suse.com)
49 49
 Alexandre Lécuyer (alexandre.lecuyer@corp.ovh.com)
50 50
 Alfredo Moralejo (amoralej@redhat.com)
51 51
 Alistair Coles (alistairncoles@gmail.com)
@@ -166,6 +166,7 @@ Fujita Tomonori (fujita.tomonori@lab.ntt.co.jp)
166 166
 Félix Cantournet (felix.cantournet@cloudwatt.com)
167 167
 Gage Hugo (gh159m@att.com)
168 168
 Ganesh Maharaj Mahalingam (ganesh.mahalingam@intel.com)
169
+gaobin (gaobin@inspur.com)
169 170
 gaofei (gao.fei@inspur.com)
170 171
 Gaurav B. Gangalwar (gaurav@gluster.com)
171 172
 gecong1973 (ge.cong@zte.com.cn)
@@ -173,6 +174,7 @@ gengchc2 (geng.changcai2@zte.com.cn)
173 174
 Gerard Gine (ggine@swiftstack.com)
174 175
 Gerry Drudy (gerry.drudy@hpe.com)
175 176
 Gil Vernik (gilv@il.ibm.com)
177
+Gleb Samsonov (sams-gleb@yandex.ru)
176 178
 Gonéri Le Bouder (goneri.lebouder@enovance.com)
177 179
 Graham Hayes (graham.hayes@hpe.com)
178 180
 Gregory Haynes (greg@greghaynes.net)
@@ -276,6 +278,7 @@ Mehdi Abaakouk (sileht@redhat.com)
276 278
 melissaml (ma.lei@99cloud.net)
277 279
 Michael Matur (michael.matur@gmail.com)
278 280
 Michael Shuler (mshuler@gmail.com)
281
+Michele Valsecchi (mvalsecc@redhat.com)
279 282
 Mike Fedosin (mfedosin@mirantis.com)
280 283
 Mingyu Li (li.mingyu@99cloud.net)
281 284
 Minwoo Bae (minwoob@us.ibm.com)
@@ -286,6 +289,7 @@ Monty Taylor (mordred@inaugust.com)
286 289
 Morgan Fainberg (morgan.fainberg@gmail.com)
287 290
 Morita Kazutaka (morita.kazutaka@gmail.com)
288 291
 Motonobu Ichimura (motonobu@gmail.com)
292
+Nadeem Syed (snadeem.hameed@gmail.com)
289 293
 Nakagawa Masaaki (nakagawamsa@nttdata.co.jp)
290 294
 Nakul Dahiwade (nakul.dahiwade@intel.com)
291 295
 Nam Nguyen Hoai (namnh@vn.fujitsu.com)

+ 58
- 0
CHANGELOG View File

@@ -1,3 +1,61 @@
1
+swift (2.21.0, OpenStack Stein release)
2
+
3
+    * Change the behavior of the EC reconstructor to perform a
4
+      fragment rebuild to a handoff node when a primary peer responds
5
+      with 507 to the REPLICATE request. This changes EC to match the
6
+      existing behavior of replication when drives fail. After a
7
+      rebalance of EC rings (potentially removing unmounted/failed
8
+      devices), it's most IO efficient to run in handoffs_only mode to
9
+      avoid unnecessary rebuilds.
10
+
11
+    * O_TMPFILE support is now detected by attempting to use it
12
+      instead of looking at the kernel version. This allows older
13
+      kernels with backported patches to take advantage of the
14
+      O_TMPFILE functionality.
15
+
16
+    * Add slo_manifest_hook callback to allow other middlewares to
17
+      impose additional constraints on or make edits to SLO manifests
18
+      before being written. For example, a middleware could enforce
19
+      minimum segment size or insert data segments.
20
+
21
+    * Fixed an issue with multi-region EC policies that caused the EC
22
+      reconstructor to constantly attempt cross-region rebuild
23
+      traffic.
24
+
25
+    * Fixed an issue where S3 API v4 signatures would not be validated
26
+      against the body of the request, allowing a replay attack if
27
+      request headers were captured by a malicious third party.
28
+
29
+    * Display crypto data/metadata details in swift-object-info.
30
+
31
+    * formpost can now accept a content-encoding parameter.
32
+
33
+    * Fixed an issue where multipart uploads with the S3 API would
34
+      sometimes report an error despite all segments being upload
35
+      successfully.
36
+
37
+    * Multipart object segments are now actually deleted when the
38
+      multipart object is deleted via the S3 API.
39
+
40
+    * Swift now returns a 503 (instead of a 500) when an account
41
+      auto-create fails.
42
+
43
+    * Fixed a bug where encryption would store the incorrect key
44
+      metadata if the object name starts with a slash.
45
+
46
+    * Fixed an issue where an object server failure during a client
47
+      download could leave an open socket between the proxy and
48
+      client.
49
+
50
+    * Fixed an issue where deleted EC objects didn't have their
51
+      on-disk directories cleaned up. This would cause extra resource
52
+      usage on the object servers.
53
+
54
+    * Fixed issue where bulk requests using xml and expect
55
+      100-continue would return a malformed HTTP response.
56
+
57
+    * Various other minor bug fixes and improvements.
58
+
1 59
 swift (2.20.0)
2 60
 
3 61
     * S3 API compatibility updates

+ 1
- 1
doc/saio/swift/object-expirer.conf View File

@@ -27,7 +27,7 @@ log_level = INFO
27 27
 interval = 300
28 28
 # auto_create_account_prefix = .
29 29
 # report_interval = 300
30
-# concurrency is the level of concurrency o use to do the work, this value
30
+# concurrency is the level of concurrency to use to do the work, this value
31 31
 # must be set to at least 1
32 32
 # concurrency = 1
33 33
 # processes is how many parts to divide the work into, one part per process

+ 322
- 289
doc/source/development_saio.rst View File

@@ -76,9 +76,10 @@ Installing dependencies
76 76
                             python2-netifaces python2-pip python2-dnspython \
77 77
                             python2-mock
78 78
 
79
-  Note: This installs necessary system dependencies and *most* of the python
80
-  dependencies. Later in the process setuptools/distribute or pip will install
81
-  and/or upgrade packages.
79
+.. note::
80
+   This installs necessary system dependencies and *most* of the python
81
+   dependencies. Later in the process setuptools/distribute or pip will install
82
+   and/or upgrade packages.
82 83
 
83 84
 Next, choose either :ref:`partition-section` or :ref:`loopback-section`.
84 85
 
@@ -90,50 +91,52 @@ Using a partition for storage
90 91
 If you are going to use a separate partition for Swift data, be sure to add
91 92
 another device when creating the VM, and follow these instructions:
92 93
 
93
-  #. Set up a single partition::
94
+#. Set up a single partition::
94 95
 
95
-        sudo fdisk /dev/sdb
96
-        sudo mkfs.xfs /dev/sdb1
96
+      sudo fdisk /dev/sdb
97
+      sudo mkfs.xfs /dev/sdb1
97 98
 
98
-  #. Edit ``/etc/fstab`` and add::
99
+#. Edit ``/etc/fstab`` and add::
99 100
 
100
-        /dev/sdb1 /mnt/sdb1 xfs noatime,nodiratime,nobarrier,logbufs=8 0 0
101
+      /dev/sdb1 /mnt/sdb1 xfs noatime,nodiratime,nobarrier,logbufs=8 0 0
101 102
 
102
-  #. Create the mount point and the individualized links::
103
+#. Create the mount point and the individualized links::
103 104
 
104
-        sudo mkdir /mnt/sdb1
105
-        sudo mount /mnt/sdb1
106
-        sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4
107
-        sudo chown ${USER}:${USER} /mnt/sdb1/*
108
-        sudo mkdir /srv
109
-        for x in {1..4}; do sudo ln -s /mnt/sdb1/$x /srv/$x; done
110
-        sudo mkdir -p /srv/1/node/sdb1 /srv/1/node/sdb5 \
111
-                      /srv/2/node/sdb2 /srv/2/node/sdb6 \
112
-                      /srv/3/node/sdb3 /srv/3/node/sdb7 \
113
-                      /srv/4/node/sdb4 /srv/4/node/sdb8 \
114
-                      /var/run/swift
115
-        sudo chown -R ${USER}:${USER} /var/run/swift
116
-        # **Make sure to include the trailing slash after /srv/$x/**
117
-        for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done
105
+      sudo mkdir /mnt/sdb1
106
+      sudo mount /mnt/sdb1
107
+      sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4
108
+      sudo chown ${USER}:${USER} /mnt/sdb1/*
109
+      sudo mkdir /srv
110
+      for x in {1..4}; do sudo ln -s /mnt/sdb1/$x /srv/$x; done
111
+      sudo mkdir -p /srv/1/node/sdb1 /srv/1/node/sdb5 \
112
+                    /srv/2/node/sdb2 /srv/2/node/sdb6 \
113
+                    /srv/3/node/sdb3 /srv/3/node/sdb7 \
114
+                    /srv/4/node/sdb4 /srv/4/node/sdb8 \
115
+                    /var/run/swift
116
+      sudo chown -R ${USER}:${USER} /var/run/swift
117
+      # **Make sure to include the trailing slash after /srv/$x/**
118
+      for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done
118 119
 
119
-     Note: For OpenSuse users, a user's primary group is `users`, so you have 2 options:
120
+   .. note::
121
+      For OpenSuse users, a user's primary group is ``users``, so you have 2 options:
120 122
 
121
-     * Change `${USER}:${USER}` to `${USER}:users` in all references of this guide; or
122
-     * Create a group for your username and add yourself to it::
123
+      * Change ``${USER}:${USER}`` to ``${USER}:users`` in all references of this guide; or
124
+      * Create a group for your username and add yourself to it::
123 125
 
124
-        sudo groupadd ${USER} && sudo gpasswd -a ${USER} ${USER}
126
+         sudo groupadd ${USER} && sudo gpasswd -a ${USER} ${USER}
125 127
 
126
-     Note: We create the mount points and mount the storage disk under
127
-     /mnt/sdb1. This disk will contain one directory per simulated swift node,
128
-     each owned by the current swift user.
128
+   .. note::
129
+      We create the mount points and mount the storage disk under
130
+      /mnt/sdb1. This disk will contain one directory per simulated swift node,
131
+      each owned by the current swift user.
129 132
 
130
-     We then create symlinks to these directories under /srv.
131
-     If the disk sdb is unmounted, files will not be written under
132
-     /srv/\*, because the symbolic link destination /mnt/sdb1/* will not
133
-     exist. This prevents disk sync operations from writing to the root
134
-     partition in the event a drive is unmounted.
133
+   We then create symlinks to these directories under /srv.
134
+   If the disk sdb is unmounted, files will not be written under
135
+   /srv/\*, because the symbolic link destination /mnt/sdb1/* will not
136
+   exist. This prevents disk sync operations from writing to the root
137
+   partition in the event a drive is unmounted.
135 138
 
136
-  #. Next, skip to :ref:`common-dev-section`.
139
+#. Next, skip to :ref:`common-dev-section`.
137 140
 
138 141
 
139 142
 .. _loopback-section:
@@ -144,51 +147,53 @@ Using a loopback device for storage
144 147
 If you want to use a loopback device instead of another partition, follow
145 148
 these instructions:
146 149
 
147
-  #. Create the file for the loopback device::
150
+#. Create the file for the loopback device::
148 151
 
149
-        sudo mkdir /srv
150
-        sudo truncate -s 1GB /srv/swift-disk
151
-        sudo mkfs.xfs /srv/swift-disk
152
+      sudo mkdir /srv
153
+      sudo truncate -s 1GB /srv/swift-disk
154
+      sudo mkfs.xfs /srv/swift-disk
152 155
 
153
-     Modify size specified in the ``truncate`` command to make a larger or
154
-     smaller partition as needed.
156
+   Modify size specified in the ``truncate`` command to make a larger or
157
+   smaller partition as needed.
155 158
 
156
-  #. Edit `/etc/fstab` and add::
159
+#. Edit `/etc/fstab` and add::
157 160
 
158
-        /srv/swift-disk /mnt/sdb1 xfs loop,noatime,nodiratime,nobarrier,logbufs=8 0 0
161
+      /srv/swift-disk /mnt/sdb1 xfs loop,noatime,nodiratime,nobarrier,logbufs=8 0 0
159 162
 
160
-  #. Create the mount point and the individualized links::
163
+#. Create the mount point and the individualized links::
161 164
 
162
-        sudo mkdir /mnt/sdb1
163
-        sudo mount /mnt/sdb1
164
-        sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4
165
-        sudo chown ${USER}:${USER} /mnt/sdb1/*
166
-        for x in {1..4}; do sudo ln -s /mnt/sdb1/$x /srv/$x; done
167
-        sudo mkdir -p /srv/1/node/sdb1 /srv/1/node/sdb5 \
168
-                      /srv/2/node/sdb2 /srv/2/node/sdb6 \
169
-                      /srv/3/node/sdb3 /srv/3/node/sdb7 \
170
-                      /srv/4/node/sdb4 /srv/4/node/sdb8 \
171
-                      /var/run/swift
172
-        sudo chown -R ${USER}:${USER} /var/run/swift
173
-        # **Make sure to include the trailing slash after /srv/$x/**
174
-        for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done
165
+      sudo mkdir /mnt/sdb1
166
+      sudo mount /mnt/sdb1
167
+      sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4
168
+      sudo chown ${USER}:${USER} /mnt/sdb1/*
169
+      for x in {1..4}; do sudo ln -s /mnt/sdb1/$x /srv/$x; done
170
+      sudo mkdir -p /srv/1/node/sdb1 /srv/1/node/sdb5 \
171
+                    /srv/2/node/sdb2 /srv/2/node/sdb6 \
172
+                    /srv/3/node/sdb3 /srv/3/node/sdb7 \
173
+                    /srv/4/node/sdb4 /srv/4/node/sdb8 \
174
+                    /var/run/swift
175
+      sudo chown -R ${USER}:${USER} /var/run/swift
176
+      # **Make sure to include the trailing slash after /srv/$x/**
177
+      for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done
175 178
 
176
-     Note: For OpenSuse users, a user's primary group is `users`, so you have 2 options:
179
+   .. note::
180
+      For OpenSuse users, a user's primary group is ``users``, so you have 2 options:
177 181
 
178
-     * Change `${USER}:${USER}` to `${USER}:users` in all references of this guide; or
179
-     * Create a group for your username and add yourself to it::
182
+      * Change ``${USER}:${USER}`` to ``${USER}:users`` in all references of this guide; or
183
+      * Create a group for your username and add yourself to it::
180 184
 
181
-        sudo groupadd ${USER} && sudo gpasswd -a ${USER} ${USER}
185
+         sudo groupadd ${USER} && sudo gpasswd -a ${USER} ${USER}
182 186
 
183
-     Note: We create the mount points and mount the loopback file under
184
-     /mnt/sdb1. This file will contain one directory per simulated swift node,
185
-     each owned by the current swift user.
187
+   .. note::
188
+      We create the mount points and mount the loopback file under
189
+      /mnt/sdb1. This file will contain one directory per simulated swift node,
190
+      each owned by the current swift user.
186 191
 
187
-     We then create symlinks to these directories under /srv.
188
-     If the loopback file is unmounted, files will not be written under
189
-     /srv/\*, because the symbolic link destination /mnt/sdb1/* will not
190
-     exist. This prevents disk sync operations from writing to the root
191
-     partition in the event a drive is unmounted.
192
+   We then create symlinks to these directories under /srv.
193
+   If the loopback file is unmounted, files will not be written under
194
+   /srv/\*, because the symbolic link destination /mnt/sdb1/* will not
195
+   exist. This prevents disk sync operations from writing to the root
196
+   partition in the event a drive is unmounted.
192 197
 
193 198
 .. _common-dev-section:
194 199
 
@@ -229,117 +234,120 @@ To persist this, edit and add the following to ``/etc/fstab``::
229 234
 Getting the code
230 235
 ----------------
231 236
 
232
-  #. Check out the python-swiftclient repo::
237
+#. Check out the python-swiftclient repo::
233 238
 
234
-        cd $HOME; git clone https://github.com/openstack/python-swiftclient.git
239
+      cd $HOME; git clone https://github.com/openstack/python-swiftclient.git
235 240
 
236
-  #. Build a development installation of python-swiftclient::
241
+#. Build a development installation of python-swiftclient::
237 242
 
238
-        cd $HOME/python-swiftclient; sudo python setup.py develop; cd -
243
+      cd $HOME/python-swiftclient; sudo python setup.py develop; cd -
239 244
 
240
-     Ubuntu 12.04 users need to install python-swiftclient's dependencies before the installation of
241
-     python-swiftclient. This is due to a bug in an older version of setup tools::
245
+   Ubuntu 12.04 users need to install python-swiftclient's dependencies before the installation of
246
+   python-swiftclient. This is due to a bug in an older version of setup tools::
242 247
 
243
-        cd $HOME/python-swiftclient; sudo pip install -r requirements.txt; sudo python setup.py develop; cd -
248
+      cd $HOME/python-swiftclient; sudo pip install -r requirements.txt; sudo python setup.py develop; cd -
244 249
 
245
-  #. Check out the swift repo::
250
+#. Check out the swift repo::
246 251
 
247
-        git clone https://github.com/openstack/swift.git
252
+      git clone https://github.com/openstack/swift.git
248 253
 
249
-  #. Build a development installation of swift::
254
+#. Build a development installation of swift::
250 255
 
251
-        cd $HOME/swift; sudo pip install --no-binary cryptography -r requirements.txt; sudo python setup.py develop; cd -
256
+      cd $HOME/swift; sudo pip install --no-binary cryptography -r requirements.txt; sudo python setup.py develop; cd -
252 257
 
253
-     Note: Due to a difference in libssl.so naming in OpenSuse to other Linux distros the wheel/binary wont work so the
254
-     cryptography must be built, thus the ``--no-binary cryptography``.
258
+   .. note::
259
+      Due to a difference in how ``libssl.so`` is named in OpenSuse vs. other Linux distros the
260
+      wheel/binary won't work; thus we use ``--no-binary cryptography`` to build ``cryptography``
261
+      locally.
255 262
 
256
-     Fedora 19 or later users might have to perform the following if development
257
-     installation of swift fails::
263
+   Fedora 19 or later users might have to perform the following if development
264
+   installation of swift fails::
258 265
 
259
-        sudo pip install -U xattr
266
+      sudo pip install -U xattr
260 267
 
261
-  #. Install swift's test dependencies::
268
+#. Install swift's test dependencies::
262 269
 
263
-        cd $HOME/swift; sudo pip install -r test-requirements.txt
270
+      cd $HOME/swift; sudo pip install -r test-requirements.txt
264 271
 
265 272
 ----------------
266 273
 Setting up rsync
267 274
 ----------------
268 275
 
269
-  #. Create ``/etc/rsyncd.conf``::
276
+#. Create ``/etc/rsyncd.conf``::
270 277
 
271
-        sudo cp $HOME/swift/doc/saio/rsyncd.conf /etc/
272
-        sudo sed -i "s/<your-user-name>/${USER}/" /etc/rsyncd.conf
278
+      sudo cp $HOME/swift/doc/saio/rsyncd.conf /etc/
279
+      sudo sed -i "s/<your-user-name>/${USER}/" /etc/rsyncd.conf
273 280
 
274
-     Here is the default ``rsyncd.conf`` file contents maintained in the repo
275
-     that is copied and fixed up above:
281
+   Here is the default ``rsyncd.conf`` file contents maintained in the repo
282
+   that is copied and fixed up above:
276 283
 
277
-     .. literalinclude:: /../saio/rsyncd.conf
284
+   .. literalinclude:: /../saio/rsyncd.conf
285
+      :language: ini
278 286
 
279
-  #. On Ubuntu, edit the following line in ``/etc/default/rsync``::
287
+#. On Ubuntu, edit the following line in ``/etc/default/rsync``::
280 288
 
281
-        RSYNC_ENABLE=true
289
+      RSYNC_ENABLE=true
282 290
 
283
-     On Fedora, edit the following line in ``/etc/xinetd.d/rsync``::
291
+   On Fedora, edit the following line in ``/etc/xinetd.d/rsync``::
284 292
 
285
-        disable = no
293
+      disable = no
286 294
 
287
-     One might have to create the above files to perform the edits.
295
+   One might have to create the above files to perform the edits.
288 296
 
289
-     On OpenSuse, nothing needs to happen here.
297
+   On OpenSuse, nothing needs to happen here.
290 298
 
291
-  #. On platforms with SELinux in ``Enforcing`` mode, either set to ``Permissive``::
299
+#. On platforms with SELinux in ``Enforcing`` mode, either set to ``Permissive``::
292 300
 
293
-        sudo setenforce Permissive
301
+      sudo setenforce Permissive
294 302
 
295
-     Or just allow rsync full access::
303
+   Or just allow rsync full access::
296 304
 
297
-        sudo setsebool -P rsync_full_access 1
305
+      sudo setsebool -P rsync_full_access 1
298 306
 
299
-  #. Start the rsync daemon
307
+#. Start the rsync daemon
300 308
 
301
-     * On Ubuntu 14.04, run::
309
+   * On Ubuntu 14.04, run::
302 310
 
303
-        sudo service rsync restart
311
+      sudo service rsync restart
304 312
 
305
-     * On Ubuntu 16.04, run::
313
+   * On Ubuntu 16.04, run::
306 314
 
307
-        sudo systemctl enable rsync
308
-        sudo systemctl start rsync
315
+      sudo systemctl enable rsync
316
+      sudo systemctl start rsync
309 317
 
310
-     * On Fedora, run::
318
+   * On Fedora, run::
311 319
 
312
-        sudo systemctl restart xinetd.service
313
-        sudo systemctl enable rsyncd.service
314
-        sudo systemctl start rsyncd.service
320
+      sudo systemctl restart xinetd.service
321
+      sudo systemctl enable rsyncd.service
322
+      sudo systemctl start rsyncd.service
315 323
 
316
-     * On OpenSuse, run::
324
+   * On OpenSuse, run::
317 325
 
318
-        sudo systemctl enable rsyncd.service
319
-        sudo systemctl start rsyncd.service
326
+      sudo systemctl enable rsyncd.service
327
+      sudo systemctl start rsyncd.service
320 328
 
321
-     * On other xinetd based systems simply run::
329
+   * On other xinetd based systems simply run::
322 330
 
323
-        sudo service xinetd restart
331
+      sudo service xinetd restart
324 332
 
325
-  #. Verify rsync is accepting connections for all servers::
333
+#. Verify rsync is accepting connections for all servers::
326 334
 
327
-        rsync rsync://pub@localhost/
335
+      rsync rsync://pub@localhost/
328 336
 
329
-     You should see the following output from the above command::
337
+   You should see the following output from the above command::
330 338
 
331
-        account6012
332
-        account6022
333
-        account6032
334
-        account6042
335
-        container6011
336
-        container6021
337
-        container6031
338
-        container6041
339
-        object6010
340
-        object6020
341
-        object6030
342
-        object6040
339
+      account6012
340
+      account6022
341
+      account6032
342
+      account6042
343
+      container6011
344
+      container6021
345
+      container6031
346
+      container6041
347
+      object6010
348
+      object6020
349
+      object6030
350
+      object6040
343 351
 
344 352
 ------------------
345 353
 Starting memcached
@@ -362,50 +370,51 @@ running, tokens cannot be validated, and accessing Swift becomes impossible.
362 370
 Optional: Setting up rsyslog for individual logging
363 371
 ---------------------------------------------------
364 372
 
365
-  #. Install the swift rsyslogd configuration::
373
+#. Install the swift rsyslogd configuration::
366 374
 
367
-        sudo cp $HOME/swift/doc/saio/rsyslog.d/10-swift.conf /etc/rsyslog.d/
375
+      sudo cp $HOME/swift/doc/saio/rsyslog.d/10-swift.conf /etc/rsyslog.d/
368 376
 
369
-     Note: OpenSuse may have the systemd logger installed, so if you want this
370
-     to work, you need to install rsyslog::
377
+   Note: OpenSuse may have the systemd logger installed, so if you want this
378
+   to work, you need to install rsyslog::
371 379
 
372
-        sudo zypper install rsyslog
373
-        sudo systemctl start rsyslog.service
374
-        sudo systemctl enable rsyslog.service
380
+      sudo zypper install rsyslog
381
+      sudo systemctl start rsyslog.service
382
+      sudo systemctl enable rsyslog.service
375 383
 
376
-     Be sure to review that conf file to determine if you want all the logs
377
-     in one file vs. all the logs separated out, and if you want hourly logs
378
-     for stats processing. For convenience, we provide its default contents
379
-     below:
384
+   Be sure to review that conf file to determine if you want all the logs
385
+   in one file vs. all the logs separated out, and if you want hourly logs
386
+   for stats processing. For convenience, we provide its default contents
387
+   below:
380 388
 
381
-     .. literalinclude:: /../saio/rsyslog.d/10-swift.conf
389
+   .. literalinclude:: /../saio/rsyslog.d/10-swift.conf
390
+      :language: ini
382 391
 
383
-  #. Edit ``/etc/rsyslog.conf`` and make the following change (usually in the
384
-     "GLOBAL DIRECTIVES" section)::
392
+#. Edit ``/etc/rsyslog.conf`` and make the following change (usually in the
393
+   "GLOBAL DIRECTIVES" section)::
385 394
 
386
-        $PrivDropToGroup adm
395
+      $PrivDropToGroup adm
387 396
 
388
-  #. If using hourly logs (see above) perform::
397
+#. If using hourly logs (see above) perform::
389 398
 
390
-        sudo mkdir -p /var/log/swift/hourly
399
+      sudo mkdir -p /var/log/swift/hourly
391 400
 
392
-     Otherwise perform::
401
+   Otherwise perform::
393 402
 
394
-        sudo mkdir -p /var/log/swift
403
+      sudo mkdir -p /var/log/swift
395 404
 
396
-  #. Setup the logging directory and start syslog:
405
+#. Setup the logging directory and start syslog:
397 406
 
398
-     * On Ubuntu::
407
+   * On Ubuntu::
399 408
 
400
-        sudo chown -R syslog.adm /var/log/swift
401
-        sudo chmod -R g+w /var/log/swift
402
-        sudo service rsyslog restart
409
+      sudo chown -R syslog.adm /var/log/swift
410
+      sudo chmod -R g+w /var/log/swift
411
+      sudo service rsyslog restart
403 412
 
404
-     * On Fedora and OpenSuse::
413
+   * On Fedora and OpenSuse::
405 414
 
406
-        sudo chown -R root:adm /var/log/swift
407
-        sudo chmod -R g+w /var/log/swift
408
-        sudo systemctl restart rsyslog.service
415
+      sudo chown -R root:adm /var/log/swift
416
+      sudo chmod -R g+w /var/log/swift
417
+      sudo systemctl restart rsyslog.service
409 418
 
410 419
 ---------------------
411 420
 Configuring each node
@@ -415,89 +424,106 @@ After performing the following steps, be sure to verify that Swift has access
415 424
 to resulting configuration files (sample configuration files are provided with
416 425
 all defaults in line-by-line comments).
417 426
 
418
-  #. Optionally remove an existing swift directory::
427
+#. Optionally remove an existing swift directory::
419 428
 
420
-        sudo rm -rf /etc/swift
429
+      sudo rm -rf /etc/swift
421 430
 
422
-  #. Populate the ``/etc/swift`` directory itself::
431
+#. Populate the ``/etc/swift`` directory itself::
423 432
 
424
-        cd $HOME/swift/doc; sudo cp -r saio/swift /etc/swift; cd -
425
-        sudo chown -R ${USER}:${USER} /etc/swift
433
+      cd $HOME/swift/doc; sudo cp -r saio/swift /etc/swift; cd -
434
+      sudo chown -R ${USER}:${USER} /etc/swift
426 435
 
427
-  #. Update ``<your-user-name>`` references in the Swift config files::
436
+#. Update ``<your-user-name>`` references in the Swift config files::
428 437
 
429
-        find /etc/swift/ -name \*.conf | xargs sudo sed -i "s/<your-user-name>/${USER}/"
438
+      find /etc/swift/ -name \*.conf | xargs sudo sed -i "s/<your-user-name>/${USER}/"
430 439
 
431 440
 The contents of the configuration files provided by executing the above
432 441
 commands are as follows:
433 442
 
434
-  #. ``/etc/swift/swift.conf``
443
+#. ``/etc/swift/swift.conf``
435 444
 
436
-     .. literalinclude:: /../saio/swift/swift.conf
445
+   .. literalinclude:: /../saio/swift/swift.conf
446
+      :language: ini
437 447
 
438
-  #. ``/etc/swift/proxy-server.conf``
448
+#. ``/etc/swift/proxy-server.conf``
439 449
 
440
-     .. literalinclude:: /../saio/swift/proxy-server.conf
450
+   .. literalinclude:: /../saio/swift/proxy-server.conf
451
+      :language: ini
441 452
 
442
-  #. ``/etc/swift/object-expirer.conf``
453
+#. ``/etc/swift/object-expirer.conf``
443 454
 
444
-     .. literalinclude:: /../saio/swift/object-expirer.conf
455
+   .. literalinclude:: /../saio/swift/object-expirer.conf
456
+      :language: ini
445 457
 
446
-  #. ``/etc/swift/container-reconciler.conf``
458
+#. ``/etc/swift/container-reconciler.conf``
447 459
 
448
-     .. literalinclude:: /../saio/swift/container-reconciler.conf
460
+   .. literalinclude:: /../saio/swift/container-reconciler.conf
461
+      :language: ini
449 462
 
450
-  #. ``/etc/swift/container-sync-realms.conf``
463
+#. ``/etc/swift/container-sync-realms.conf``
451 464
 
452
-     .. literalinclude:: /../saio/swift/container-sync-realms.conf
465
+   .. literalinclude:: /../saio/swift/container-sync-realms.conf
466
+      :language: ini
453 467
 
454
-  #. ``/etc/swift/account-server/1.conf``
468
+#. ``/etc/swift/account-server/1.conf``
455 469
 
456
-     .. literalinclude:: /../saio/swift/account-server/1.conf
470
+   .. literalinclude:: /../saio/swift/account-server/1.conf
471
+      :language: ini
457 472
 
458
-  #. ``/etc/swift/container-server/1.conf``
473
+#. ``/etc/swift/container-server/1.conf``
459 474
 
460
-     .. literalinclude:: /../saio/swift/container-server/1.conf
475
+   .. literalinclude:: /../saio/swift/container-server/1.conf
476
+      :language: ini
461 477
 
462
-  #. ``/etc/swift/object-server/1.conf``
478
+#. ``/etc/swift/object-server/1.conf``
463 479
 
464
-     .. literalinclude:: /../saio/swift/object-server/1.conf
480
+   .. literalinclude:: /../saio/swift/object-server/1.conf
481
+      :language: ini
465 482
 
466
-  #. ``/etc/swift/account-server/2.conf``
483
+#. ``/etc/swift/account-server/2.conf``
467 484
 
468
-     .. literalinclude:: /../saio/swift/account-server/2.conf
485
+   .. literalinclude:: /../saio/swift/account-server/2.conf
486
+      :language: ini
469 487
 
470
-  #. ``/etc/swift/container-server/2.conf``
488
+#. ``/etc/swift/container-server/2.conf``
471 489
 
472
-     .. literalinclude:: /../saio/swift/container-server/2.conf
490
+   .. literalinclude:: /../saio/swift/container-server/2.conf
491
+      :language: ini
473 492
 
474
-  #. ``/etc/swift/object-server/2.conf``
493
+#. ``/etc/swift/object-server/2.conf``
475 494
 
476
-     .. literalinclude:: /../saio/swift/object-server/2.conf
495
+   .. literalinclude:: /../saio/swift/object-server/2.conf
496
+      :language: ini
477 497
 
478
-  #. ``/etc/swift/account-server/3.conf``
498
+#. ``/etc/swift/account-server/3.conf``
479 499
 
480
-     .. literalinclude:: /../saio/swift/account-server/3.conf
500
+   .. literalinclude:: /../saio/swift/account-server/3.conf
501
+      :language: ini
481 502
 
482
-  #. ``/etc/swift/container-server/3.conf``
503
+#. ``/etc/swift/container-server/3.conf``
483 504
 
484
-     .. literalinclude:: /../saio/swift/container-server/3.conf
505
+   .. literalinclude:: /../saio/swift/container-server/3.conf
506
+      :language: ini
485 507
 
486
-  #. ``/etc/swift/object-server/3.conf``
508
+#. ``/etc/swift/object-server/3.conf``
487 509
 
488
-     .. literalinclude:: /../saio/swift/object-server/3.conf
510
+   .. literalinclude:: /../saio/swift/object-server/3.conf
511
+      :language: ini
489 512
 
490
-  #. ``/etc/swift/account-server/4.conf``
513
+#. ``/etc/swift/account-server/4.conf``
491 514
 
492
-     .. literalinclude:: /../saio/swift/account-server/4.conf
515
+   .. literalinclude:: /../saio/swift/account-server/4.conf
516
+      :language: ini
493 517
 
494
-  #. ``/etc/swift/container-server/4.conf``
518
+#. ``/etc/swift/container-server/4.conf``
495 519
 
496
-     .. literalinclude:: /../saio/swift/container-server/4.conf
520
+   .. literalinclude:: /../saio/swift/container-server/4.conf
521
+      :language: ini
497 522
 
498
-  #. ``/etc/swift/object-server/4.conf``
523
+#. ``/etc/swift/object-server/4.conf``
499 524
 
500
-     .. literalinclude:: /../saio/swift/object-server/4.conf
525
+   .. literalinclude:: /../saio/swift/object-server/4.conf
526
+      :language: ini
501 527
 
502 528
 .. _setup_scripts:
503 529
 
@@ -505,139 +531,146 @@ commands are as follows:
505 531
 Setting up scripts for running Swift
506 532
 ------------------------------------
507 533
 
508
-  #. Copy the SAIO scripts for resetting the environment::
534
+#. Copy the SAIO scripts for resetting the environment::
535
+
536
+      mkdir -p $HOME/bin
537
+      cd $HOME/swift/doc; cp saio/bin/* $HOME/bin; cd -
538
+      chmod +x $HOME/bin/*
539
+
540
+#. Edit the ``$HOME/bin/resetswift`` script
509 541
 
510
-        mkdir -p $HOME/bin
511
-        cd $HOME/swift/doc; cp saio/bin/* $HOME/bin; cd -
512
-        chmod +x $HOME/bin/*
542
+   The template ``resetswift`` script looks like the following:
513 543
 
514
-  #. Edit the ``$HOME/bin/resetswift`` script
544
+   .. literalinclude:: /../saio/bin/resetswift
545
+      :language: bash
515 546
 
516
-     The template ``resetswift`` script looks like the following:
547
+   If you are using a loopback device add an environment var to
548
+   substitute ``/dev/sdb1`` with ``/srv/swift-disk``::
517 549
 
518
-        .. literalinclude:: /../saio/bin/resetswift
550
+      echo "export SAIO_BLOCK_DEVICE=/srv/swift-disk" >> $HOME/.bashrc
519 551
 
520
-     If you are using a loopback device add an environment var to
521
-     substitute ``/dev/sdb1`` with ``/srv/swift-disk``::
552
+   If you did not set up rsyslog for individual logging, remove the ``find
553
+   /var/log/swift...`` line::
522 554
 
523
-        echo "export SAIO_BLOCK_DEVICE=/srv/swift-disk" >> $HOME/.bashrc
555
+      sed -i "/find \/var\/log\/swift/d" $HOME/bin/resetswift
524 556
 
525
-     If you did not set up rsyslog for individual logging, remove the ``find
526
-     /var/log/swift...`` line::
527 557
 
528
-        sed -i "/find \/var\/log\/swift/d" $HOME/bin/resetswift
558
+#. Install the sample configuration file for running tests::
529 559
 
560
+      cp $HOME/swift/test/sample.conf /etc/swift/test.conf
530 561
 
531
-  #. Install the sample configuration file for running tests::
562
+   The template ``test.conf`` looks like the following:
532 563
 
533
-        cp $HOME/swift/test/sample.conf /etc/swift/test.conf
564
+   .. literalinclude:: /../../test/sample.conf
565
+      :language: ini
534 566
 
535
-     The template ``test.conf`` looks like the following:
567
+#. Add an environment variable for running tests below::
536 568
 
537
-        .. literalinclude:: /../../test/sample.conf
569
+      echo "export SWIFT_TEST_CONFIG_FILE=/etc/swift/test.conf" >> $HOME/.bashrc
538 570
 
539
-  #. Add an environment variable for running tests below::
571
+#. Be sure that your ``PATH`` includes the ``bin`` directory::
540 572
 
541
-        echo "export SWIFT_TEST_CONFIG_FILE=/etc/swift/test.conf" >> $HOME/.bashrc
573
+      echo "export PATH=${PATH}:$HOME/bin" >> $HOME/.bashrc
542 574
 
543
-  #. Be sure that your ``PATH`` includes the ``bin`` directory::
575
+#. Source the above environment variables into your current environment::
544 576
 
545
-        echo "export PATH=${PATH}:$HOME/bin" >> $HOME/.bashrc
577
+      . $HOME/.bashrc
546 578
 
547
-  #. Source the above environment variables into your current environment::
579
+#. Construct the initial rings using the provided script::
548 580
 
549
-        . $HOME/.bashrc
581
+      remakerings
550 582
 
551
-  #. Construct the initial rings using the provided script::
583
+   The ``remakerings`` script looks like the following:
552 584
 
553
-        remakerings
585
+   .. literalinclude:: /../saio/bin/remakerings
586
+      :language: bash
554 587
 
555
-     The ``remakerings`` script looks like the following:
588
+   You can expect the output from this command to produce the following.  Note
589
+   that 3 object rings are created in order to test storage policies and EC in
590
+   the SAIO environment.  The EC ring is the only one with all 8 devices.
591
+   There are also two replication rings, one for 3x replication and another
592
+   for 2x replication, but those rings only use 4 devices:
556 593
 
557
-        .. literalinclude:: /../saio/bin/remakerings
558 594
 
559
-     You can expect the output from this command to produce the following.  Note
560
-     that 3 object rings are created in order to test storage policies and EC in
561
-     the SAIO environment.  The EC ring is the only one with all 8 devices.
562
-     There are also two replication rings, one for 3x replication and another
563
-     for 2x replication, but those rings only use 4 devices::
595
+   .. code-block:: console
564 596
 
565
-        Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
566
-        Device d1r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 1
567
-        Device d2r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 2
568
-        Device d3r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 3
569
-        Reassigned 3072 (300.00%) partitions. Balance is now 0.00.  Dispersion is now 0.00
570
-        Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
571
-        Device d1r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 1
572
-        Device d2r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 2
573
-        Device d3r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 3
574
-        Reassigned 2048 (200.00%) partitions. Balance is now 0.00.  Dispersion is now 0.00
575
-        Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
576
-        Device d1r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb5_"" with 1.0 weight got id 1
577
-        Device d2r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 2
578
-        Device d3r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb6_"" with 1.0 weight got id 3
579
-        Device d4r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 4
580
-        Device d5r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb7_"" with 1.0 weight got id 5
581
-        Device d6r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 6
582
-        Device d7r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb8_"" with 1.0 weight got id 7
583
-        Reassigned 6144 (600.00%) partitions. Balance is now 0.00.  Dispersion is now 0.00
584
-        Device d0r1z1-127.0.0.1:6011R127.0.0.1:6011/sdb1_"" with 1.0 weight got id 0
585
-        Device d1r1z2-127.0.0.2:6021R127.0.0.2:6021/sdb2_"" with 1.0 weight got id 1
586
-        Device d2r1z3-127.0.0.3:6031R127.0.0.3:6031/sdb3_"" with 1.0 weight got id 2
587
-        Device d3r1z4-127.0.0.4:6041R127.0.0.4:6041/sdb4_"" with 1.0 weight got id 3
588
-        Reassigned 3072 (300.00%) partitions. Balance is now 0.00.  Dispersion is now 0.00
589
-        Device d0r1z1-127.0.0.1:6012R127.0.0.1:6012/sdb1_"" with 1.0 weight got id 0
590
-        Device d1r1z2-127.0.0.2:6022R127.0.0.2:6022/sdb2_"" with 1.0 weight got id 1
591
-        Device d2r1z3-127.0.0.3:6032R127.0.0.3:6032/sdb3_"" with 1.0 weight got id 2
592
-        Device d3r1z4-127.0.0.4:6042R127.0.0.4:6042/sdb4_"" with 1.0 weight got id 3
593
-        Reassigned 3072 (300.00%) partitions. Balance is now 0.00.  Dispersion is now 0.00
597
+      Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
598
+      Device d1r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 1
599
+      Device d2r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 2
600
+      Device d3r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 3
601
+      Reassigned 3072 (300.00%) partitions. Balance is now 0.00.  Dispersion is now 0.00
602
+      Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
603
+      Device d1r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 1
604
+      Device d2r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 2
605
+      Device d3r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 3
606
+      Reassigned 2048 (200.00%) partitions. Balance is now 0.00.  Dispersion is now 0.00
607
+      Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
608
+      Device d1r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb5_"" with 1.0 weight got id 1
609
+      Device d2r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 2
610
+      Device d3r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb6_"" with 1.0 weight got id 3
611
+      Device d4r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 4
612
+      Device d5r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb7_"" with 1.0 weight got id 5
613
+      Device d6r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 6
614
+      Device d7r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb8_"" with 1.0 weight got id 7
615
+      Reassigned 6144 (600.00%) partitions. Balance is now 0.00.  Dispersion is now 0.00
616
+      Device d0r1z1-127.0.0.1:6011R127.0.0.1:6011/sdb1_"" with 1.0 weight got id 0
617
+      Device d1r1z2-127.0.0.2:6021R127.0.0.2:6021/sdb2_"" with 1.0 weight got id 1
618
+      Device d2r1z3-127.0.0.3:6031R127.0.0.3:6031/sdb3_"" with 1.0 weight got id 2
619
+      Device d3r1z4-127.0.0.4:6041R127.0.0.4:6041/sdb4_"" with 1.0 weight got id 3
620
+      Reassigned 3072 (300.00%) partitions. Balance is now 0.00.  Dispersion is now 0.00
621
+      Device d0r1z1-127.0.0.1:6012R127.0.0.1:6012/sdb1_"" with 1.0 weight got id 0
622
+      Device d1r1z2-127.0.0.2:6022R127.0.0.2:6022/sdb2_"" with 1.0 weight got id 1
623
+      Device d2r1z3-127.0.0.3:6032R127.0.0.3:6032/sdb3_"" with 1.0 weight got id 2
624
+      Device d3r1z4-127.0.0.4:6042R127.0.0.4:6042/sdb4_"" with 1.0 weight got id 3
625
+      Reassigned 3072 (300.00%) partitions. Balance is now 0.00.  Dispersion is now 0.00
594 626
 
595 627
 
596
-  #. Read more about Storage Policies and your SAIO :doc:`policies_saio`
628
+#. Read more about Storage Policies and your SAIO :doc:`policies_saio`
597 629
 
598
-  #. Verify the unit tests run::
630
+#. Verify the unit tests run::
599 631
 
600
-        $HOME/swift/.unittests
632
+      $HOME/swift/.unittests
601 633
 
602
-     Note that the unit tests do not require any swift daemons running.
634
+   Note that the unit tests do not require any swift daemons running.
603 635
 
604
-  #. Start the "main" Swift daemon processes (proxy, account, container, and
605
-     object)::
636
+#. Start the "main" Swift daemon processes (proxy, account, container, and
637
+   object)::
606 638
 
607
-        startmain
639
+      startmain
608 640
 
609
-     (The "``Unable to increase file descriptor limit.  Running as non-root?``"
610
-     warnings are expected and ok.)
641
+   (The "``Unable to increase file descriptor limit.  Running as non-root?``"
642
+   warnings are expected and ok.)
611 643
 
612
-     The ``startmain`` script looks like the following:
644
+   The ``startmain`` script looks like the following:
613 645
 
614
-        .. literalinclude:: /../saio/bin/startmain
646
+   .. literalinclude:: /../saio/bin/startmain
647
+      :language: bash
615 648
 
616
-  #. Get an ``X-Storage-Url`` and ``X-Auth-Token``::
649
+#. Get an ``X-Storage-Url`` and ``X-Auth-Token``::
617 650
 
618
-        curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0
651
+      curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0
619 652
 
620
-  #. Check that you can ``GET`` account::
653
+#. Check that you can ``GET`` account::
621 654
 
622
-        curl -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>
655
+      curl -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>
623 656
 
624
-  #. Check that ``swift`` command provided by the python-swiftclient package works::
657
+#. Check that ``swift`` command provided by the python-swiftclient package works::
625 658
 
626
-        swift -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat
659
+      swift -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat
627 660
 
628
-  #. Verify the functional tests run::
661
+#. Verify the functional tests run::
629 662
 
630
-        $HOME/swift/.functests
663
+      $HOME/swift/.functests
631 664
 
632
-     (Note: functional tests will first delete everything in the configured
633
-     accounts.)
665
+   (Note: functional tests will first delete everything in the configured
666
+   accounts.)
634 667
 
635
-  #. Verify the probe tests run::
668
+#. Verify the probe tests run::
636 669
 
637
-        $HOME/swift/.probetests
670
+      $HOME/swift/.probetests
638 671
 
639
-     (Note: probe tests will reset your environment as they call ``resetswift``
640
-     for each test.)
672
+   (Note: probe tests will reset your environment as they call ``resetswift``
673
+   for each test.)
641 674
 
642 675
 ----------------
643 676
 Debugging Issues

+ 2
- 1
etc/proxy-server.conf-sample View File

@@ -589,7 +589,8 @@ reseller_prefix = AUTH_
589 589
 # useful if there are multiple auth systems in the proxy pipeline.
590 590
 delay_auth_decision = False
591 591
 
592
-# Keystone server details
592
+# Keystone server details. Note that this differs from how swift3 was
593
+# configured: in particular, the Keystone API version must be included.
593 594
 auth_uri = http://keystonehost:35357/v3
594 595
 
595 596
 # Connect/read timeout to use when communicating with Keystone

+ 69
- 0
releasenotes/notes/2_21_0_release-d8ae33ef18b7be3a.yaml View File

@@ -0,0 +1,69 @@
1
+---
2
+features:
3
+  - |
4
+    Change the behavior of the EC reconstructor to perform a
5
+    fragment rebuild to a handoff node when a primary peer responds
6
+    with 507 to the REPLICATE request. This changes EC to match the
7
+    existing behavior of replication when drives fail. After a
8
+    rebalance of EC rings (potentially removing unmounted/failed
9
+    devices), it's most IO efficient to run in handoffs_only mode to
10
+    avoid unnecessary rebuilds.
11
+
12
+  - |
13
+    O_TMPFILE support is now detected by attempting to use it
14
+    instead of looking at the kernel version. This allows older
15
+    kernels with backported patches to take advantage of the
16
+    O_TMPFILE functionality.
17
+
18
+  - |
19
+    Add slo_manifest_hook callback to allow other middlewares to
20
+    impose additional constraints on or make edits to SLO manifests
21
+    before being written. For example, a middleware could enforce
22
+    minimum segment size or insert data segments.
23
+
24
+  - |
25
+    Fixed an issue with multi-region EC policies that caused the EC
26
+    reconstructor to constantly attempt cross-region rebuild
27
+    traffic.
28
+
29
+  - |
30
+    Fixed an issue where S3 API v4 signatures would not be validated
31
+    against the body of the request, allowing a replay attack if
32
+    request headers were captured by a malicious third party.
33
+
34
+  - Display crypto data/metadata details in swift-object-info.
35
+
36
+  - formpost can now accept a content-encoding parameter.
37
+
38
+  - |
39
+    Fixed an issue where multipart uploads with the S3 API would
40
+    sometimes report an error despite all segments being upload
41
+    successfully.
42
+
43
+  - |
44
+    Multipart object segments are now actually deleted when the
45
+    multipart object is deleted via the S3 API.
46
+
47
+  - |
48
+    Swift now returns a 503 (instead of a 500) when an account
49
+    auto-create fails.
50
+
51
+  - |
52
+    Fixed a bug where encryption would store the incorrect key
53
+    metadata if the object name starts with a slash.
54
+
55
+  - |
56
+    Fixed an issue where an object server failure during a client
57
+    download could leave an open socket between the proxy and
58
+    client.
59
+
60
+  - |
61
+    Fixed an issue where deleted EC objects didn't have their
62
+    on-disk directories cleaned up. This would cause extra resource
63
+    usage on the object servers.
64
+
65
+  - |
66
+    Fixed issue where bulk requests using xml and expect
67
+    100-continue would return a malformed HTTP response.
68
+
69
+  - Various other minor bug fixes and improvements.

+ 2
- 0
releasenotes/source/index.rst View File

@@ -7,6 +7,8 @@
7 7
 
8 8
    current
9 9
 
10
+   stein
11
+
10 12
    rocky
11 13
 
12 14
    queens

+ 6
- 0
releasenotes/source/stein.rst View File

@@ -0,0 +1,6 @@
1
+===================================
2
+ Stein Series Release Notes
3
+===================================
4
+
5
+.. release-notes::
6
+   :branch: stable/stein

+ 40
- 50
swift/account/reaper.py View File

@@ -16,7 +16,6 @@
16 16
 import os
17 17
 import random
18 18
 import socket
19
-from swift import gettext_ as _
20 19
 from logging import DEBUG
21 20
 from math import sqrt
22 21
 from time import time
@@ -142,10 +141,10 @@ class AccountReaper(Daemon):
142 141
                     continue
143 142
                 self.reap_device(device)
144 143
         except (Exception, Timeout):
145
-            self.logger.exception(_("Exception in top-level account reaper "
146
-                                    "loop"))
144
+            self.logger.exception("Exception in top-level account reaper "
145
+                                  "loop")
147 146
         elapsed = time() - begin
148
-        self.logger.info(_('Devices pass completed: %.02fs'), elapsed)
147
+        self.logger.info('Devices pass completed: %.02fs', elapsed)
149 148
 
150 149
     def reap_device(self, device):
151 150
         """
@@ -255,19 +254,15 @@ class AccountReaper(Daemon):
255 254
                 self.delay_reaping:
256 255
             return False
257 256
         account = info['account']
258
-        self.logger.info(_('Beginning pass on account %s'), account)
257
+        self.logger.info('Beginning pass on account %s', account)
259 258
         self.reset_stats()
260 259
         container_limit = 1000
261 260
         if container_shard is not None:
262 261
             container_limit *= len(nodes)
263 262
         try:
264
-            marker = ''
265
-            while True:
266
-                containers = \
267
-                    list(broker.list_containers_iter(container_limit, marker,
268
-                                                     None, None, None))
269
-                if not containers:
270
-                    break
263
+            containers = list(broker.list_containers_iter(
264
+                container_limit, '', None, None, None))
265
+            while containers:
271 266
                 try:
272 267
                     for (container, _junk, _junk, _junk, _junk) in containers:
273 268
                         if six.PY3:
@@ -284,43 +279,44 @@ class AccountReaper(Daemon):
284 279
                     self.container_pool.waitall()
285 280
                 except (Exception, Timeout):
286 281
                     self.logger.exception(
287
-                        _('Exception with containers for account %s'), account)
288
-                marker = containers[-1][0]
289
-                if marker == '':
290
-                    break
291
-            log = 'Completed pass on account %s' % account
282
+                        'Exception with containers for account %s', account)
283
+                containers = list(broker.list_containers_iter(
284
+                    container_limit, containers[-1][0], None, None, None))
285
+            log_buf = ['Completed pass on account %s' % account]
292 286
         except (Exception, Timeout):
293
-            self.logger.exception(
294
-                _('Exception with account %s'), account)
295
-            log = _('Incomplete pass on account %s') % account
287
+            self.logger.exception('Exception with account %s', account)
288
+            log_buf = ['Incomplete pass on account %s' % account]
296 289
         if self.stats_containers_deleted:
297
-            log += _(', %s containers deleted') % self.stats_containers_deleted
290
+            log_buf.append(', %s containers deleted' %
291
+                           self.stats_containers_deleted)
298 292
         if self.stats_objects_deleted:
299
-            log += _(', %s objects deleted') % self.stats_objects_deleted
293
+            log_buf.append(', %s objects deleted' % self.stats_objects_deleted)
300 294
         if self.stats_containers_remaining:
301
-            log += _(', %s containers remaining') % \
302
-                self.stats_containers_remaining
295
+            log_buf.append(', %s containers remaining' %
296
+                           self.stats_containers_remaining)
303 297
         if self.stats_objects_remaining:
304
-            log += _(', %s objects remaining') % self.stats_objects_remaining
298
+            log_buf.append(', %s objects remaining' %
299
+                           self.stats_objects_remaining)
305 300
         if self.stats_containers_possibly_remaining:
306
-            log += _(', %s containers possibly remaining') % \
307
-                self.stats_containers_possibly_remaining
301
+            log_buf.append(', %s containers possibly remaining' %
302
+                           self.stats_containers_possibly_remaining)
308 303
         if self.stats_objects_possibly_remaining:
309
-            log += _(', %s objects possibly remaining') % \
310
-                self.stats_objects_possibly_remaining
304
+            log_buf.append(', %s objects possibly remaining' %
305
+                           self.stats_objects_possibly_remaining)
311 306
         if self.stats_return_codes:
312
-            log += _(', return codes: ')
307
+            log_buf.append(', return codes: ')
313 308
             for code in sorted(self.stats_return_codes):
314
-                log += '%s %sxxs, ' % (self.stats_return_codes[code], code)
315
-            log = log[:-2]
316
-        log += _(', elapsed: %.02fs') % (time() - begin)
317
-        self.logger.info(log)
309
+                log_buf.append('%s %sxxs, ' % (self.stats_return_codes[code],
310
+                                               code))
311
+            log_buf[-1] = log_buf[-1][:-2]
312
+        log_buf.append(', elapsed: %.02fs' % (time() - begin))
313
+        self.logger.info(''.join(log_buf))
318 314
         self.logger.timing_since('timing', self.start_time)
319 315
         delete_timestamp = Timestamp(info['delete_timestamp'])
320 316
         if self.stats_containers_remaining and \
321 317
            begin - float(delete_timestamp) >= self.reap_not_done_after:
322 318
             self.logger.warning(
323
-                _('Account %(account)s has not been reaped since %(time)s') %
319
+                'Account %(account)s has not been reaped since %(time)s' %
324 320
                 {'account': account, 'time': delete_timestamp.isoformat})
325 321
         return True
326 322
 
@@ -379,14 +375,14 @@ class AccountReaper(Daemon):
379 375
             except ClientException as err:
380 376
                 if self.logger.getEffectiveLevel() <= DEBUG:
381 377
                     self.logger.exception(
382
-                        _('Exception with %(ip)s:%(port)s/%(device)s'), node)
378
+                        'Exception with %(ip)s:%(port)s/%(device)s', node)
383 379
                 self.stats_return_codes[err.http_status // 100] = \
384 380
                     self.stats_return_codes.get(err.http_status // 100, 0) + 1
385 381
                 self.logger.increment(
386 382
                     'return_codes.%d' % (err.http_status // 100,))
387 383
             except (Timeout, socket.error) as err:
388 384
                 self.logger.error(
389
-                    _('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
385
+                    'Timeout Exception with %(ip)s:%(port)s/%(device)s',
390 386
                     node)
391 387
             if not objects:
392 388
                 break
@@ -397,21 +393,15 @@ class AccountReaper(Daemon):
397 393
                     self.logger.error('ERROR: invalid storage policy index: %r'
398 394
                                       % policy_index)
399 395
                 for obj in objects:
400
-                    obj_name = obj['name']
401
-                    if isinstance(obj_name, six.text_type):
402
-                        obj_name = obj_name.encode('utf8')
403 396
                     pool.spawn(self.reap_object, account, container, part,
404
-                               nodes, obj_name, policy_index)
397
+                               nodes, obj['name'], policy_index)
405 398
                 pool.waitall()
406 399
             except (Exception, Timeout):
407
-                self.logger.exception(_('Exception with objects for container '
408
-                                        '%(container)s for account %(account)s'
409
-                                        ),
400
+                self.logger.exception('Exception with objects for container '
401
+                                      '%(container)s for account %(account)s',
410 402
                                       {'container': container,
411 403
                                        'account': account})
412 404
             marker = objects[-1]['name']
413
-            if marker == '':
414
-                break
415 405
         successes = 0
416 406
         failures = 0
417 407
         timestamp = Timestamp.now()
@@ -434,7 +424,7 @@ class AccountReaper(Daemon):
434 424
             except ClientException as err:
435 425
                 if self.logger.getEffectiveLevel() <= DEBUG:
436 426
                     self.logger.exception(
437
-                        _('Exception with %(ip)s:%(port)s/%(device)s'), node)
427
+                        'Exception with %(ip)s:%(port)s/%(device)s', node)
438 428
                 failures += 1
439 429
                 self.logger.increment('containers_failures')
440 430
                 self.stats_return_codes[err.http_status // 100] = \
@@ -443,7 +433,7 @@ class AccountReaper(Daemon):
443 433
                     'return_codes.%d' % (err.http_status // 100,))
444 434
             except (Timeout, socket.error) as err:
445 435
                 self.logger.error(
446
-                    _('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
436
+                    'Timeout Exception with %(ip)s:%(port)s/%(device)s',
447 437
                     node)
448 438
                 failures += 1
449 439
                 self.logger.increment('containers_failures')
@@ -510,7 +500,7 @@ class AccountReaper(Daemon):
510 500
             except ClientException as err:
511 501
                 if self.logger.getEffectiveLevel() <= DEBUG:
512 502
                     self.logger.exception(
513
-                        _('Exception with %(ip)s:%(port)s/%(device)s'), node)
503
+                        'Exception with %(ip)s:%(port)s/%(device)s', node)
514 504
                 failures += 1
515 505
                 self.logger.increment('objects_failures')
516 506
                 self.stats_return_codes[err.http_status // 100] = \
@@ -521,7 +511,7 @@ class AccountReaper(Daemon):
521 511
                 failures += 1
522 512
                 self.logger.increment('objects_failures')
523 513
                 self.logger.error(
524
-                    _('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
514
+                    'Timeout Exception with %(ip)s:%(port)s/%(device)s',
525 515
                     node)
526 516
             if successes > failures:
527 517
                 self.stats_objects_deleted += 1

+ 6
- 1
swift/cli/manage_shard_ranges.py View File

@@ -515,7 +515,12 @@ def main(args=None):
515 515
     logger = get_logger({}, name='ContainerBroker', log_to_console=True)
516 516
     broker = ContainerBroker(args.container_db, logger=logger,
517 517
                              skip_commits=True)
518
-    broker.get_info()
518
+    try:
519
+        broker.get_info()
520
+    except Exception as exc:
521
+        print('Error opening container DB %s: %s' % (args.container_db, exc),
522
+              file=sys.stderr)
523
+        return 2
519 524
     print('Loaded db broker for %s.' % broker.path, file=sys.stderr)
520 525
     return args.func(broker, args)
521 526
 

+ 8
- 0
swift/common/bufferedhttp.py View File

@@ -178,6 +178,14 @@ class BufferedHTTPConnection(HTTPConnection):
178 178
         return ret
179 179
 
180 180
     def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
181
+        '''Send a request to the server.
182
+
183
+        :param method: specifies an HTTP request method, e.g. 'GET'.
184
+        :param url: specifies the object being requested, e.g. '/index.html'.
185
+        :param skip_host: if True does not add automatically a 'Host:' header
186
+        :param skip_accept_encoding: if True does not add automatically an
187
+           'Accept-Encoding:' header
188
+        '''
181 189
         self._method = method
182 190
         self._path = url
183 191
         return HTTPConnection.putrequest(self, method, url, skip_host,

+ 14
- 10
swift/common/middleware/bulk.py View File

@@ -380,6 +380,10 @@ class Bulk(object):
380 380
             query request.
381 381
         """
382 382
         last_yield = time()
383
+        if out_content_type and out_content_type.endswith('/xml'):
384
+            to_yield = '<?xml version="1.0" encoding="UTF-8"?>\n'
385
+        else:
386
+            to_yield = ' '
383 387
         separator = ''
384 388
         failed_files = []
385 389
         resp_dict = {'Response Status': HTTPOk().status,
@@ -390,8 +394,6 @@ class Bulk(object):
390 394
         try:
391 395
             if not out_content_type:
392 396
                 raise HTTPNotAcceptable(request=req)
393
-            if out_content_type.endswith('/xml'):
394
-                yield '<?xml version="1.0" encoding="UTF-8"?>\n'
395 397
 
396 398
             try:
397 399
                 vrs, account, _junk = req.split_path(2, 3, True)
@@ -452,9 +454,9 @@ class Bulk(object):
452 454
                     for resp, obj_name, retry in pile.asyncstarmap(
453 455
                             do_delete, names_to_delete):
454 456
                         if last_yield + self.yield_frequency < time():
455
-                            separator = '\r\n\r\n'
456 457
                             last_yield = time()
457
-                            yield ' '
458
+                            yield to_yield
459
+                            to_yield, separator = ' ', '\r\n\r\n'
458 460
                         self._process_delete(resp, pile, obj_name,
459 461
                                              resp_dict, failed_files,
460 462
                                              failed_file_response, retry)
@@ -462,9 +464,9 @@ class Bulk(object):
462 464
                             # Abort, but drain off the in-progress deletes
463 465
                             for resp, obj_name, retry in pile:
464 466
                                 if last_yield + self.yield_frequency < time():
465
-                                    separator = '\r\n\r\n'
466 467
                                     last_yield = time()
467
-                                    yield ' '
468
+                                    yield to_yield
469
+                                    to_yield, separator = ' ', '\r\n\r\n'
468 470
                                 # Don't pass in the pile, as we shouldn't retry
469 471
                                 self._process_delete(
470 472
                                     resp, None, obj_name, resp_dict,
@@ -508,14 +510,16 @@ class Bulk(object):
508 510
                      'Response Body': '', 'Number Files Created': 0}
509 511
         failed_files = []
510 512
         last_yield = time()
513
+        if out_content_type and out_content_type.endswith('/xml'):
514
+            to_yield = '<?xml version="1.0" encoding="UTF-8"?>\n'
515
+        else:
516
+            to_yield = ' '
511 517
         separator = ''
512 518
         containers_accessed = set()
513 519
         req.environ['eventlet.minimum_write_chunk_size'] = 0
514 520
         try:
515 521
             if not out_content_type:
516 522
                 raise HTTPNotAcceptable(request=req)
517
-            if out_content_type.endswith('/xml'):
518
-                yield '<?xml version="1.0" encoding="UTF-8"?>\n'
519 523
 
520 524
             if req.content_length is None and \
521 525
                     req.headers.get('transfer-encoding',
@@ -533,9 +537,9 @@ class Bulk(object):
533 537
             containers_created = 0
534 538
             while True:
535 539
                 if last_yield + self.yield_frequency < time():
536
-                    separator = '\r\n\r\n'
537 540
                     last_yield = time()
538
-                    yield ' '
541
+                    yield to_yield
542
+                    to_yield, separator = ' ', '\r\n\r\n'
539 543
                 tar_info = next(tar)
540 544
                 if tar_info is None or \
541 545
                         len(failed_files) >= self.max_failed_extractions:

+ 9
- 10
swift/common/middleware/copy.py View File

@@ -114,12 +114,11 @@ greater than 5GB.
114 114
 
115 115
 """
116 116
 
117
-from six.moves.urllib.parse import quote, unquote
118
-
119 117
 from swift.common.utils import get_logger, config_true_value, FileLikeIter, \
120 118
     close_if_possible
121 119
 from swift.common.swob import Request, HTTPPreconditionFailed, \
122
-    HTTPRequestEntityTooLarge, HTTPBadRequest, HTTPException
120
+    HTTPRequestEntityTooLarge, HTTPBadRequest, HTTPException, \
121
+    wsgi_quote, wsgi_unquote
123 122
 from swift.common.http import HTTP_MULTIPLE_CHOICES, is_success, HTTP_OK
124 123
 from swift.common.constraints import check_account_format, MAX_FILE_SIZE
125 124
 from swift.common.request_helpers import copy_header_subset, remove_items, \
@@ -183,7 +182,7 @@ class ServerSideCopyWebContext(WSGIContext):
183 182
 
184 183
     def get_source_resp(self, req):
185 184
         sub_req = make_subrequest(
186
-            req.environ, path=quote(req.path_info), headers=req.headers,
185
+            req.environ, path=wsgi_quote(req.path_info), headers=req.headers,
187 186
             swift_source='SSC')
188 187
         return sub_req.get_response(self.app)
189 188
 
@@ -257,9 +256,9 @@ class ServerSideCopyMiddleware(object):
257 256
                                           )(req.environ, start_response)
258 257
         dest_account = account
259 258
         if 'Destination-Account' in req.headers:
260
-            dest_account = unquote(req.headers.get('Destination-Account'))
259
+            dest_account = wsgi_unquote(req.headers.get('Destination-Account'))
261 260
             dest_account = check_account_format(req, dest_account)
262
-            req.headers['X-Copy-From-Account'] = quote(account)
261
+            req.headers['X-Copy-From-Account'] = wsgi_quote(account)
263 262
             account = dest_account
264 263
             del req.headers['Destination-Account']
265 264
         dest_container, dest_object = _check_destination_header(req)
@@ -275,7 +274,7 @@ class ServerSideCopyMiddleware(object):
275 274
         req.path_info = '/%s/%s/%s/%s' % (
276 275
             ver, dest_account, dest_container, dest_object)
277 276
         req.headers['Content-Length'] = 0
278
-        req.headers['X-Copy-From'] = quote(source)
277
+        req.headers['X-Copy-From'] = wsgi_quote(source)
279 278
         del req.headers['Destination']
280 279
         return self.handle_PUT(req, start_response)
281 280
 
@@ -312,8 +311,8 @@ class ServerSideCopyMiddleware(object):
312 311
     def _create_response_headers(self, source_path, source_resp, sink_req):
313 312
         resp_headers = dict()
314 313
         acct, path = source_path.split('/', 3)[2:4]
315
-        resp_headers['X-Copied-From-Account'] = quote(acct)
316
-        resp_headers['X-Copied-From'] = quote(path)
314
+        resp_headers['X-Copied-From-Account'] = wsgi_quote(acct)
315
+        resp_headers['X-Copied-From'] = wsgi_quote(path)
317 316
         if 'last-modified' in source_resp.headers:
318 317
             resp_headers['X-Copied-From-Last-Modified'] = \
319 318
                 source_resp.headers['last-modified']
@@ -334,7 +333,7 @@ class ServerSideCopyMiddleware(object):
334 333
         src_account_name = req.headers.get('X-Copy-From-Account')
335 334
         if src_account_name:
336 335
             src_account_name = check_account_format(
337
-                req, unquote(src_account_name))
336
+                req, wsgi_unquote(src_account_name))
338 337
         else:
339 338
             src_account_name = acct
340 339
         src_container_name, src_obj_name = _check_copy_from_header(req)

+ 32
- 6
swift/common/middleware/crypto/keymaster.py View File

@@ -18,7 +18,8 @@ import hmac
18 18
 from swift.common.exceptions import UnknownSecretIdError
19 19
 from swift.common.middleware.crypto.crypto_utils import CRYPTO_KEY_CALLBACK
20 20
 from swift.common.swob import Request, HTTPException, wsgi_to_bytes
21
-from swift.common.utils import readconf, strict_b64decode, get_logger
21
+from swift.common.utils import readconf, strict_b64decode, get_logger, \
22
+    split_path
22 23
 from swift.common.wsgi import WSGIContext
23 24
 
24 25
 
@@ -77,29 +78,54 @@ class KeyMasterContext(WSGIContext):
77 78
             version = key_id['v']
78 79
             if version not in ('1', '2'):
79 80
                 raise ValueError('Unknown key_id version: %s' % version)
81
+            if version == '1' and not key_id['path'].startswith(
82
+                    '/' + self.account + '/'):
83
+                # Well shoot. This was the bug that made us notice we needed
84
+                # a v2! Hope the current account/container was the original!
85
+                key_acct, key_cont, key_obj = (
86
+                    self.account, self.container, key_id['path'])
87
+            else:
88
+                key_acct, key_cont, key_obj = split_path(
89
+                    key_id['path'], 1, 3, True)
90
+
91
+            check_path = (
92
+                self.account, self.container or key_cont, self.obj or key_obj)
93
+            if (key_acct, key_cont, key_obj) != check_path:
94
+                self.keymaster.logger.info(
95
+                    "Path stored in meta (%r) does not match path from "
96
+                    "request (%r)! Using path from meta.",
97
+                    key_id['path'],
98
+                    '/' + '/'.join(x for x in [
99
+                        self.account, self.container, self.obj] if x))
80 100
         else:
81 101
             secret_id = self.keymaster.active_secret_id
82 102
             # v1 had a bug where we would claim the path was just the object
83 103
             # name if the object started with a slash. Bump versions to
84 104
             # establish that we can trust the path.
85 105
             version = '2'
106
+            key_acct, key_cont, key_obj = (
107
+                self.account, self.container, self.obj)
108
+
86 109
         if (secret_id, version) in self._keys:
87 110
             return self._keys[(secret_id, version)]
88 111
 
89 112
         keys = {}
90
-        account_path = '/' + self.account
113
+        account_path = '/' + key_acct
91 114
 
92 115
         try:
116
+            # self.account/container/obj reflect the level of the *request*,
117
+            # which may be different from the level of the key_id-path. Only
118
+            # fetch the keys that the request needs.
93 119
             if self.container:
94
-                path = account_path + '/' + self.container
120
+                path = account_path + '/' + key_cont
95 121
                 keys['container'] = self.keymaster.create_key(
96 122
                     path, secret_id=secret_id)
97 123
 
98 124
                 if self.obj:
99
-                    if self.obj.startswith('/') and version == '1':
100
-                        path = self.obj
125
+                    if key_obj.startswith('/') and version == '1':
126
+                        path = key_obj
101 127
                     else:
102
-                        path = path + '/' + self.obj
128
+                        path = path + '/' + key_obj
103 129
                     keys['object'] = self.keymaster.create_key(
104 130
                         path, secret_id=secret_id)
105 131
 

+ 2
- 1
swift/common/middleware/dlo.py View File

@@ -144,7 +144,8 @@ class GetContext(WSGIContext):
144 144
     def _get_container_listing(self, req, version, account, container,
145 145
                                prefix, marker=''):
146 146
         con_req = make_subrequest(
147
-            req.environ, path='/'.join(['', version, account, container]),
147
+            req.environ,
148
+            path=quote('/'.join(['', version, account, container])),
148 149
             method='GET',
149 150
             headers={'x-auth-token': req.headers.get('x-auth-token')},
150 151
             agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')

+ 17
- 4
swift/common/middleware/formpost.py View File

@@ -126,7 +126,9 @@ import hmac
126 126
 from hashlib import sha1
127 127
 from time import time
128 128
 
129
+import six
129 130
 from six.moves.urllib.parse import quote
131
+
130 132
 from swift.common.exceptions import MimeInvalid
131 133
 from swift.common.middleware.tempurl import get_tempurl_keys_from_metadata
132 134
 from swift.common.utils import streq_const_time, register_swift_info, \
@@ -229,7 +231,7 @@ class FormPost(object):
229 231
                     start_response(status, headers)
230 232
                     return [body]
231 233
             except MimeInvalid:
232
-                body = 'FormPost: invalid starting boundary'
234
+                body = b'FormPost: invalid starting boundary'
233 235
                 start_response(
234 236
                     '400 Bad Request',
235 237
                     (('Content-Type', 'text/plain'),
@@ -237,6 +239,8 @@ class FormPost(object):
237 239
                 return [body]
238 240
             except (FormInvalid, EOFError) as err:
239 241
                 body = 'FormPost: %s' % err
242
+                if six.PY3:
243
+                    body = body.encode('utf-8')
240 244
                 start_response(
241 245
                     '400 Bad Request',
242 246
                     (('Content-Type', 'text/plain'),
@@ -258,6 +262,8 @@ class FormPost(object):
258 262
         :returns: status_line, headers_list, body
259 263
         """
260 264
         keys = self._get_keys(env)
265
+        if six.PY3:
266
+            boundary = boundary.encode('utf-8')
261 267
         status = message = ''
262 268
         attributes = {}
263 269
         subheaders = []
@@ -282,14 +288,13 @@ class FormPost(object):
282 288
                         hdrs['Content-Type'] or 'application/octet-stream'
283 289
                 if 'content-encoding' not in attributes and \
284 290
                         'content-encoding' in hdrs:
285
-                    attributes['content-encoding'] = \
286
-                        hdrs['Content-Encoding']
291
+                    attributes['content-encoding'] = hdrs['Content-Encoding']
287 292
                 status, subheaders = \
288 293
                     self._perform_subrequest(env, attributes, fp, keys)
289 294
                 if not status.startswith('2'):
290 295
                     break
291 296
             else:
292
-                data = ''
297
+                data = b''
293 298
                 mxln = MAX_VALUE_LENGTH
294 299
                 while mxln:
295 300
                     chunk = fp.read(mxln)
@@ -299,6 +304,8 @@ class FormPost(object):
299 304
                     data += chunk
300 305
                 while fp.read(READ_CHUNK_SIZE):
301 306
                     pass
307
+                if six.PY3:
308
+                    data = data.decode('utf-8')
302 309
                 if 'name' in attrs:
303 310
                     attributes[attrs['name'].lower()] = data.rstrip('\r\n--')
304 311
         if not status:
@@ -315,6 +322,8 @@ class FormPost(object):
315 322
                 body = status + '\r\nFormPost: ' + message.title()
316 323
             headers.extend([('Content-Type', 'text/plain'),
317 324
                             ('Content-Length', len(body))])
325
+            if six.PY3:
326
+                body = body.encode('utf-8')
318 327
             return status, headers, body
319 328
         status = status.split(' ', 1)[0]
320 329
         if '?' in redirect:
@@ -324,6 +333,8 @@ class FormPost(object):
324 333
         redirect += 'status=%s&message=%s' % (quote(status), quote(message))
325 334
         body = '<html><body><p><a href="%s">' \
326 335
                'Click to continue...</a></p></body></html>' % redirect
336
+        if six.PY3:
337
+            body = body.encode('utf-8')
327 338
         headers.extend(
328 339
             [('Location', redirect), ('Content-Length', str(len(body)))])
329 340
         return '303 See Other', headers, body
@@ -385,6 +396,8 @@ class FormPost(object):
385 396
             attributes.get('max_file_size') or '0',
386 397
             attributes.get('max_file_count') or '0',
387 398
             attributes.get('expires') or '0')
399
+        if six.PY3:
400
+            hmac_body = hmac_body.encode('utf-8')
388 401
 
389 402
         has_valid_sig = False
390 403
         for key in keys:

+ 1
- 1
swift/common/middleware/s3api/controllers/s3_acl.py View File

@@ -13,7 +13,7 @@
13 13
 # See the License for the specific language governing permissions and
14 14
 # limitations under the License.
15 15
 
16
-from urllib import quote
16
+from six.moves.urllib.parse import quote
17 17
 from swift.common.utils import public
18 18
 
19 19
 from swift.common.middleware.s3api.controllers.base import Controller

+ 53
- 7
swift/common/middleware/s3api/s3request.py View File

@@ -24,7 +24,8 @@ import six
24 24
 from six.moves.urllib.parse import quote, unquote, parse_qsl
25 25
 import string
26 26
 
27
-from swift.common.utils import split_path, json, get_swift_info
27
+from swift.common.utils import split_path, json, get_swift_info, \
28
+    close_if_possible
28 29
 from swift.common import swob
29 30
 from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \
30 31
     HTTP_NO_CONTENT, HTTP_UNAUTHORIZED, HTTP_FORBIDDEN, HTTP_NOT_FOUND, \
@@ -110,6 +111,34 @@ def _header_acl_property(resource):
110 111
                     doc='Get and set the %s acl property' % resource)
111 112
 
112 113
 
114
+class HashingInput(object):
115
+    """
116
+    wsgi.input wrapper to verify the hash of the input as it's read.
117
+    """
118
+    def __init__(self, reader, content_length, hasher, expected_hex_hash):
119
+        self._input = reader
120
+        self._to_read = content_length
121
+        self._hasher = hasher()
122
+        self._expected = expected_hex_hash
123
+
124
+    def read(self, size=None):
125
+        chunk = self._input.read(size)
126
+        self._hasher.update(chunk)
127
+        self._to_read -= len(chunk)
128
+        if self._to_read < 0 or (size > len(chunk) and self._to_read) or (
129
+                self._to_read == 0 and
130
+                self._hasher.hexdigest() != self._expected):
131
+            self.close()
132
+            # Since we don't return the last chunk, the PUT never completes
133
+            raise swob.HTTPUnprocessableEntity(
134
+                'The X-Amz-Content-SHA56 you specified did not match '
135
+                'what we received.')
136
+        return chunk
137
+
138
+    def close(self):
139
+        close_if_possible(self._input)
140
+
141
+
113 142
 class SigV4Mixin(object):
114 143
     """
115 144
     A request class mixin to provide S3 signature v4 functionality
@@ -401,6 +430,20 @@ class SigV4Mixin(object):
401 430
             raise InvalidRequest(msg)
402 431
         else:
403 432
             hashed_payload = self.headers['X-Amz-Content-SHA256']
433
+            if self.content_length == 0:
434
+                if hashed_payload != sha256().hexdigest():
435
+                    raise BadDigest(
436
+                        'The X-Amz-Content-SHA56 you specified did not match '
437
+                        'what we received.')
438
+            elif self.content_length:
439
+                self.environ['wsgi.input'] = HashingInput(
440
+                    self.environ['wsgi.input'],
441
+                    self.content_length,
442
+                    sha256,
443
+                    hashed_payload)
444
+            # else, not provided -- Swift will kick out a 411 Length Required
445
+            # which will get translated back to a S3-style response in
446
+            # S3Request._swift_error_codes
404 447
         cr.append(hashed_payload)
405 448
         return '\n'.join(cr).encode('utf-8')
406 449
 
@@ -1267,12 +1310,15 @@ class S3Request(swob.Request):
1267 1310
         sw_req = self.to_swift_req(method, container, obj, headers=headers,
1268 1311
                                    body=body, query=query)
1269 1312
 
1270
-        sw_resp = sw_req.get_response(app)
1271
-
1272
-        # reuse account and tokens
1273
-        _, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
1274
-                                        2, 3, True)
1275
-        self.account = utf8encode(self.account)
1313
+        try:
1314
+            sw_resp = sw_req.get_response(app)
1315
+        except swob.HTTPException as err:
1316
+            sw_resp = err
1317
+        else:
1318
+            # reuse account and tokens
1319
+            _, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
1320
+                                            2, 3, True)
1321
+            self.account = utf8encode(self.account)
1276 1322
 
1277 1323
         resp = S3Response.from_swift_resp(sw_resp)
1278 1324
         status = resp.status_int  # pylint: disable-msg=E1101

+ 19
- 0
swift/common/middleware/s3api/s3token.py View File

@@ -33,6 +33,25 @@ This middleware:
33 33
 * Optionally can retrieve and cache secret from keystone
34 34
   to validate signature locally
35 35
 
36
+.. note::
37
+   If upgrading from swift3, the ``auth_version`` config option has been
38
+   removed, and the ``auth_uri`` option now includes the Keystone API
39
+   version. If you previously had a configuration like
40
+
41
+   .. code-block:: ini
42
+
43
+      [filter:s3token]
44
+      use = egg:swift3#s3token
45
+      auth_uri = https://keystonehost:35357
46
+      auth_version = 3
47
+
48
+   you should now use
49
+
50
+   .. code-block:: ini
51
+
52
+      [filter:s3token]
53
+      use = egg:swift#s3token
54
+      auth_uri = https://keystonehost:35357/v3
36 55
 """
37 56
 
38 57
 import base64

+ 5
- 5
swift/common/middleware/slo.py View File

@@ -457,8 +457,8 @@ def parse_and_validate_input(req_body, req_path):
457 457
                     continue
458 458
 
459 459
             obj_path = '/'.join(['', vrs, account,
460
-                                 seg_dict['path'].lstrip('/')])
461
-            if req_path == quote(obj_path):
460
+                                 quote(seg_dict['path'].lstrip('/'))])
461
+            if req_path == obj_path:
462 462
                 errors.append(
463 463
                     b"Index %d: manifest must not include itself as a segment"
464 464
                     % seg_index)
@@ -526,7 +526,7 @@ class SloGetContext(WSGIContext):
526 526
         Raise exception on failures.
527 527
         """
528 528
         sub_req = make_subrequest(
529
-            req.environ, path='/'.join(['', version, acc, con, obj]),
529
+            req.environ, path=quote('/'.join(['', version, acc, con, obj])),
530 530
             method='GET',
531 531
             headers={'x-auth-token': req.headers.get('x-auth-token')},
532 532
             agent='%(orig)s SLO MultipartGET', swift_source='SLO')
@@ -1109,8 +1109,8 @@ class StaticLargeObject(object):
1109 1109
                 path2indices[seg_dict['path']].append(index)
1110 1110
 
1111 1111
         def do_head(obj_name):
1112
-            obj_path = '/'.join(['', vrs, account,
1113
-                                 get_valid_utf8_str(obj_name).lstrip('/')])
1112
+            obj_path = quote('/'.join([
1113
+                '', vrs, account, get_valid_utf8_str(obj_name).lstrip('/')]))
1114 1114
 
1115 1115
             sub_req = make_subrequest(
1116 1116
                 req.environ, path=obj_path + '?',  # kill the query string

+ 4
- 1
swift/common/middleware/tempauth.py View File

@@ -184,7 +184,7 @@ from eventlet import Timeout
184 184
 import six
185 185
 from swift.common.swob import Response, Request, wsgi_to_str
186 186
 from swift.common.swob import HTTPBadRequest, HTTPForbidden, HTTPNotFound, \
187
-    HTTPUnauthorized
187
+    HTTPUnauthorized, HTTPMethodNotAllowed
188 188
 
189 189
 from swift.common.request_helpers import get_sys_meta_prefix
190 190
 from swift.common.middleware.acl import (
@@ -688,6 +688,9 @@ class TempAuth(object):
688 688
         """
689 689
         req.start_time = time()
690 690
         handler = None
691
+        if req.method != 'GET':
692
+            req.response = HTTPMethodNotAllowed(request=req)
693
+            return req.response
691 694
         try:
692 695
             version, account, user, _junk = split_path(req.path_info,
693 696
                                                        1, 4, True)

+ 7
- 9
swift/common/request_helpers.py View File

@@ -26,7 +26,6 @@ import sys
26 26
 import time
27 27
 
28 28
 import six
29
-from six.moves.urllib.parse import unquote
30 29
 from swift.common.header_key_dict import HeaderKeyDict
31 30
 
32 31
 from swift import gettext_ as _
@@ -35,11 +34,11 @@ from swift.common.exceptions import ListingIterError, SegmentError
35 34
 from swift.common.http import is_success
36 35
 from swift.common.swob import HTTPBadRequest, \
37 36
     HTTPServiceUnavailable, Range, is_chunked, multi_range_iterator, \
38
-    HTTPPreconditionFailed, wsgi_to_bytes
37
+    HTTPPreconditionFailed, wsgi_to_bytes, wsgi_unquote, wsgi_to_str
39 38
 from swift.common.utils import split_path, validate_device_partition, \
40 39
     close_if_possible, maybe_multipart_byteranges_to_document_iters, \
41 40
     multipart_byteranges_to_document_iters, parse_content_type, \
42
-    parse_content_range, csv_append, list_from_csv, Spliterator
41
+    parse_content_range, csv_append, list_from_csv, Spliterator, quote
43 42
 
44 43
 from swift.common.wsgi import make_subrequest
45 44
 
@@ -115,14 +114,13 @@ def split_and_validate_path(request, minsegs=1, maxsegs=None,
115 114
     Utility function to split and validate the request path.
116 115
 
117 116
     :returns: result of :meth:`~swift.common.utils.split_path` if
118
-              everything's okay
117
+              everything's okay, as native strings
119 118
     :raises HTTPBadRequest: if something's not okay
120 119
     """
121 120
     try:
122
-        segs = split_path(unquote(request.path),
123
-                          minsegs, maxsegs, rest_with_last)
121
+        segs = request.split_path(minsegs, maxsegs, rest_with_last)
124 122
         validate_device_partition(segs[0], segs[1])
125
-        return segs
123
+        return [wsgi_to_str(seg) for seg in segs]
126 124
     except ValueError as err:
127 125
         raise HTTPBadRequest(body=str(err), request=request,
128 126
                              content_type='text/plain')
@@ -308,7 +306,7 @@ def check_path_header(req, name, length, error_msg):
308 306
     :raise: HTTPPreconditionFailed if header value
309 307
             is not well formatted.
310 308
     """
311
-    hdr = unquote(req.headers.get(name))
309
+    hdr = wsgi_unquote(req.headers.get(name))
312 310
     if not hdr.startswith('/'):
313 311
         hdr = '/' + hdr
314 312
     try:
@@ -391,7 +389,7 @@ class SegmentedIterable(object):
391 389
                 # segment is a plain old object, not some flavor of large
392 390
                 # object; therefore, its etag is its MD5sum and hence we can
393 391
                 # check it.
394
-                path = seg_path + '?multipart-manifest=get'
392
+                path = quote(seg_path) + '?multipart-manifest=get'
395 393
                 seg_req = make_subrequest(
396 394
                     self.req.environ, path=path, method='GET',
397 395
                     headers={'x-auth-token': self.req.headers.get(

+ 44
- 0
swift/common/swob.py View File

@@ -309,6 +309,50 @@ def str_to_wsgi(native_str):
309 309
     return bytes_to_wsgi(native_str.encode('utf8', errors='surrogateescape'))
310 310
 
311 311
 
312
+def wsgi_quote(wsgi_str):
313
+    if six.PY2:
314
+        if not isinstance(wsgi_str, bytes):
315
+            raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
316
+        return urllib.parse.quote(wsgi_str)
317
+
318
+    if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
319
+        raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
320
+    return urllib.parse.quote(wsgi_str, encoding='latin-1')
321
+
322
+
323
+def wsgi_unquote(wsgi_str):
324
+    if six.PY2:
325
+        if not isinstance(wsgi_str, bytes):
326
+            raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
327
+        return urllib.parse.unquote(wsgi_str)
328
+
329
+    if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
330
+        raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
331
+    return urllib.parse.unquote(wsgi_str, encoding='latin-1')
332
+
333
+
334
+def wsgi_quote_plus(wsgi_str):
335
+    if six.PY2:
336
+        if not isinstance(wsgi_str, bytes):
337
+            raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
338
+        return urllib.parse.quote_plus(wsgi_str)
339
+
340
+    if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
341
+        raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
342
+    return urllib.parse.quote_plus(wsgi_str, encoding='latin-1')
343
+
344
+
345
+def wsgi_unquote_plus(wsgi_str):
346
+    if six.PY2:
347
+        if not isinstance(wsgi_str, bytes):
348
+            raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
349
+        return urllib.parse.unquote_plus(wsgi_str)
350
+
351
+    if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
352
+        raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
353
+    return urllib.parse.unquote_plus(wsgi_str, encoding='latin-1')
354
+
355
+
312 356
 def _resp_status_property():
313 357
     """
314 358
     Set and retrieve the value of Response.status

+ 1
- 2
swift/common/utils.py View File

@@ -4273,8 +4273,7 @@ def iter_multipart_mime_documents(wsgi_input, boundary, read_chunk_size=4096):
4273 4273
     for doing that if necessary.
4274 4274
 
4275 4275
     :param wsgi_input: The file-like object to read from.
4276
-    :param boundary: The mime boundary to separate new file-like
4277
-                     objects on.
4276
+    :param boundary: The mime boundary to separate new file-like objects on.
4278 4277
     :returns: A generator of file-like objects for each part.
4279 4278
     :raises MimeInvalid: if the document is malformed
4280 4279
     """

+ 14
- 31
swift/common/wsgi.py View File

@@ -32,13 +32,10 @@ from eventlet.green import socket, ssl, os as green_os
32 32
 import six
33 33
 from six import BytesIO
34 34
 from six import StringIO
35
-from six.moves.urllib.parse import unquote
36
-if six.PY2:
37
-    import mimetools
38 35
 
39 36
 from swift.common import utils, constraints
40 37
 from swift.common.storage_policy import BindPortsCache
41
-from swift.common.swob import Request
38
+from swift.common.swob import Request, wsgi_unquote
42 39
 from swift.common.utils import capture_stdio, disable_fallocate, \
43 40
     drop_privileges, get_logger, NullLogger, config_true_value, \
44 41
     validate_configuration, get_hub, config_auto_int_value, \
@@ -148,31 +145,6 @@ def wrap_conf_type(f):
148 145
 appconfig = wrap_conf_type(loadwsgi.appconfig)
149 146
 
150 147
 
151
-def monkey_patch_mimetools():
152
-    """
153
-    mimetools.Message defaults content-type to "text/plain"
154
-    This changes it to default to None, so we can detect missing headers.
155
-    """
156
-    if six.PY3:
157
-        # The mimetools has been removed from Python 3
158
-        return
159
-
160
-    orig_parsetype = mimetools.Message.parsetype
161
-
162
-    def parsetype(self):
163
-        if not self.typeheader:
164
-            self.type = None
165
-            self.maintype = None
166
-            self.subtype = None
167
-            self.plisttext = ''
168
-        else:
169
-            orig_parsetype(self)
170
-    parsetype.patched = True
171
-
172
-    if not getattr(mimetools.Message.parsetype, 'patched', None):
173
-        mimetools.Message.parsetype = parsetype
174
-
175
-
176 148
 def get_socket(conf):
177 149
     """Bind socket to bind ip:port in conf
178 150
 
@@ -448,6 +420,18 @@ class SwiftHttpProtocol(wsgi.HttpProtocol):
448 420
             # versions the output from error is same as info anyway
449 421
             self.server.log.info('ERROR WSGI: ' + f, *a)
450 422
 
423
+    class MessageClass(wsgi.HttpProtocol.MessageClass):
424
+        '''Subclass to see when the client didn't provide a Content-Type'''
425
+        # for py2:
426
+        def parsetype(self):
427
+            if self.typeheader is None:
428
+                self.typeheader = ''
429
+            wsgi.HttpProtocol.MessageClass.parsetype(self)
430
+
431
+        # for py3:
432
+        def get_default_type(self):
433
+            return ''
434
+
451 435
 
452 436
 class SwiftHttpProxiedProtocol(SwiftHttpProtocol):
453 437
     """
@@ -1156,7 +1140,6 @@ def _initrp(conf_path, app_section, *args, **kwargs):
1156 1140
     if config_true_value(conf.get('disable_fallocate', 'no')):
1157 1141
         disable_fallocate()
1158 1142
 
1159
-    monkey_patch_mimetools()
1160 1143
     return (conf, logger, log_name)
1161 1144
 
1162 1145
 
@@ -1319,7 +1302,7 @@ def make_subrequest(env, method=None, path=None, body=None, headers=None,
1319 1302
     path = path or ''
1320 1303
     if path and '?' in path:
1321 1304
         path, query_string = path.split('?', 1)
1322
-    newenv = make_env(env, method, path=unquote(path), agent=agent,
1305
+    newenv = make_env(env, method, path=wsgi_unquote(path), agent=agent,
1323 1306
                       query_string=query_string, swift_source=swift_source)
1324 1307
     if not headers:
1325 1308
         headers = {}

+ 1
- 1
swift/container/backend.py View File

@@ -338,7 +338,7 @@ class ContainerBroker(DatabaseBroker):
338 338
         self._db_files = None
339 339
 
340 340
     @classmethod
341
-    def create_broker(self, device_path, part, account, container, logger=None,
341
+    def create_broker(cls, device_path, part, account, container, logger=None,
342 342
                       epoch=None, put_timestamp=None,
343 343
                       storage_policy_index=None):
344 344
         """

+ 7
- 48
swift/obj/diskfile.py View File

@@ -131,14 +131,6 @@ def get_tmp_dir(policy_or_index):
131 131
     return get_policy_string(TMP_BASE, policy_or_index)
132 132
 
133 133
 
134
-def _unlink_if_present(filename):
135
-    try:
136
-        os.unlink(filename)
137
-    except OSError as err:
138
-        if err.errno != errno.ENOENT:
139
-            raise
140
-
141
-
142 134
 def _get_filename(fd):
143 135
     """
144 136
     Helper function to get to file name from a file descriptor or filename.
@@ -1623,12 +1615,10 @@ class BaseDiskFileManager(object):
1623 1615
         partition_path = get_part_path(dev_path, policy, partition)
1624 1616
         if suffixes is None:
1625 1617
             suffixes = self.yield_suffixes(device, partition, policy)
1626
-            considering_all_suffixes = True
1627 1618
         else:
1628 1619
             suffixes = (
1629 1620
                 (os.path.join(partition_path, suffix), suffix)
1630 1621
                 for suffix in suffixes)
1631
-            considering_all_suffixes = False
1632 1622
 
1633 1623
         key_preference = (
1634 1624
             ('ts_meta', 'meta_info', 'timestamp'),
@@ -1637,19 +1627,18 @@ class BaseDiskFileManager(object):
1637 1627
             ('ts_ctype', 'ctype_info', 'ctype_timestamp'),
1638 1628
         )
1639 1629
 
1640
-        # We delete as many empty directories as we can.
1641
-        # cleanup_ondisk_files() takes care of the hash dirs, and we take
1642
-        # care of the suffix dirs and possibly even the partition dir.
1643
-        have_nonempty_suffix = False
1630
+        # cleanup_ondisk_files() will remove empty hash dirs, and we'll
1631
+        # invalidate any empty suffix dirs so they'll get cleaned up on
1632
+        # the next rehash
1644 1633
         for suffix_path, suffix in suffixes:
1645
-            have_nonempty_hashdir = False
1634
+            found_files = False
1646 1635
             for object_hash in self._listdir(suffix_path):
1647 1636
                 object_path = os.path.join(suffix_path, object_hash)
1648 1637
                 try:
1649 1638
                     results = self.cleanup_ondisk_files(
1650 1639
                         object_path, **kwargs)
1651 1640
                     if results['files']:
1652
-                        have_nonempty_hashdir = True
1641
+                        found_files = True
1653 1642
                     timestamps = {}
1654 1643
                     for ts_key, info_key, info_ts_key in key_preference:
1655 1644
                         if info_key not in results:
@@ -1669,38 +1658,8 @@ class BaseDiskFileManager(object):
1669 1658
                         'Invalid diskfile filename in %r (%s)' % (
1670 1659
                             object_path, err))
1671 1660
 
1672
-            if have_nonempty_hashdir:
1673
-                have_nonempty_suffix = True
1674
-            else:
1675
-                try:
1676
-                    os.rmdir(suffix_path)
1677
-                except OSError as err:
1678
-                    if err.errno not in (errno.ENOENT, errno.ENOTEMPTY):
1679
-                        self.logger.debug(
1680
-                            'Error cleaning up empty suffix directory %s: %s',
1681
-                            suffix_path, err)
1682
-                    # cleanup_ondisk_files tries to remove empty hash dirs,
1683
-                    # but if it fails, so will we. An empty directory
1684
-                    # structure won't cause errors (just slowdown), so we
1685
-                    # ignore the exception.
1686
-        if considering_all_suffixes and not have_nonempty_suffix:
1687
-            # There's nothing of interest in the partition, so delete it
1688
-            try:
1689
-                # Remove hashes.pkl *then* hashes.invalid; otherwise, if we
1690
-                # remove hashes.invalid but leave hashes.pkl, that makes it
1691
-                # look as though the invalidations in hashes.invalid never
1692
-                # occurred.
1693
-                _unlink_if_present(os.path.join(partition_path, HASH_FILE))
1694
-                _unlink_if_present(os.path.join(partition_path,
1695
-                                                HASH_INVALIDATIONS_FILE))
1696
-                # This lock is only held by people dealing with the hashes
1697
-                # or the hash invalidations, and we've just removed those.
1698
-                _unlink_if_present(os.path.join(partition_path, ".lock"))
1699
-                _unlink_if_present(os.path.join(partition_path,
1700
-                                                ".lock-replication"))
1701
-                os.rmdir(partition_path)
1702
-            except OSError as err:
1703
-                self.logger.debug("Error cleaning up empty partition: %s", err)
1661
+            if not found_files:
1662
+                self.invalidate_hash(suffix_path)
1704 1663
 
1705 1664
 
1706 1665
 class BaseDiskFileWriter(object):

+ 3
- 0
swift/obj/mem_diskfile.py View File

@@ -412,6 +412,9 @@ class DiskFile(object):
412 412
             raise DiskFileNotOpen()
413 413
         return self._metadata
414 414
 
415
+    get_datafile_metadata = get_metadata
416
+    get_metafile_metadata = get_metadata
417
+
415 418
     def read_metadata(self, current_time=None):
416 419
         """
417 420
         Return the metadata for an object.

+ 16
- 41
swift/proxy/controllers/obj.py View File

@@ -25,6 +25,7 @@
25 25
 # collected. We've seen objects hang around forever otherwise.
26 26
 
27 27
 from six.moves.urllib.parse import unquote
28
+from six.moves import zip
28 29
 
29 30
 import collections
30 31
 import itertools
@@ -697,7 +698,8 @@ class BaseObjectController(Controller):
697 698
         """
698 699
         raise NotImplementedError()
699 700
 
700
-    def _delete_object(self, req, obj_ring, partition, headers):
701
+    def _delete_object(self, req, obj_ring, partition, headers,
702
+                       node_count=None, node_iterator=None):
701 703
         """Delete object considering write-affinity.
702 704
 
703 705
         When deleting object in write affinity deployment, also take configured
@@ -711,37 +713,12 @@ class BaseObjectController(Controller):
711 713
         :param headers: system headers to storage nodes
712 714
         :return: Response object
713 715
         """
714
-        policy_index = req.headers.get('X-Backend-Storage-Policy-Index')
715
-        policy = POLICIES.get_by_index(policy_index)
716
-
717
-        node_count = None
718
-        node_iterator = None
719
-
720
-        policy_options = self.app.get_policy_options(policy)
721
-        is_local = policy_options.write_affinity_is_local_fn
722
-        if is_local is not None:
723
-            primaries = obj_ring.get_part_nodes(partition)
724
-            node_count = len(primaries)
725
-
726
-            local_handoffs = policy_options.write_affinity_handoff_delete_count
727
-            if local_handoffs is None:
728
-                local_primaries = [node for node in primaries
729
-                                   if is_local(node)]
730
-                local_handoffs = len(primaries) - len(local_primaries)
731
-
732
-            node_count += local_handoffs
733
-
734
-            node_iterator = self.iter_nodes_local_first(
735
-                obj_ring, partition, policy=policy, local_handoffs_first=True
736
-            )
737
-
738 716
         status_overrides = {404: 204}
739 717
         resp = self.make_requests(req, obj_ring,
740 718
                                   partition, 'DELETE', req.swift_entity_path,
741 719
                                   headers, overrides=status_overrides,
742 720
                                   node_count=node_count,
743 721
                                   node_iterator=node_iterator)
744
-
745 722
         return resp
746 723
 
747 724
     def _post_object(self, req, obj_ring, partition, headers):
@@ -861,6 +838,7 @@ class BaseObjectController(Controller):
861 838
 
862 839
         # Include local handoff nodes if write-affinity is enabled.
863 840
         node_count = len(nodes)
841
+        node_iterator = None
864 842
         policy = POLICIES.get_by_index(policy_index)
865 843
         policy_options = self.app.get_policy_options(policy)
866 844
         is_local = policy_options.write_affinity_is_local_fn
@@ -870,11 +848,16 @@ class BaseObjectController(Controller):
870 848
                 local_primaries = [node for node in nodes if is_local(node)]
871 849
                 local_handoffs = len(nodes) - len(local_primaries)
872 850
             node_count += local_handoffs
851
+            node_iterator = self.iter_nodes_local_first(
852
+                obj_ring, partition, policy=policy, local_handoffs_first=True
853
+            )
873 854
 
874 855
         headers = self._backend_requests(
875 856
             req, node_count, container_partition, container_nodes,
876 857
             container_path=container_path)
877
-        return self._delete_object(req, obj_ring, partition, headers)
858
+        return self._delete_object(req, obj_ring, partition, headers,
859
+                                   node_count=node_count,
860
+                                   node_iterator=node_iterator)
878 861
 
879 862
 
880 863
 @ObjectControllerRouter.register(REPL_POLICY)
@@ -1112,18 +1095,14 @@ class ECAppIter(object):
1112 1095
             resp.content_type = self.learned_content_type
1113 1096
         resp.content_length = self.obj_length
1114 1097
 
1115
-    def _next_range(self):
1098
+    def _next_ranges(self):
1116 1099
         # Each FA part should have approximately the same headers. We really
1117 1100
         # only care about Content-Range and Content-Type, and that'll be the
1118 1101
         # same for all the different FAs.
1119
-        frag_iters = []
1120
-        headers = None
1121
-        for parts_iter in self.internal_parts_iters:
1122
-            part_info = next(parts_iter)
1123
-            frag_iters.append(part_info['part_iter'])
1124
-            headers = part_info['headers']
1125
-        headers = HeaderKeyDict(headers)
1126
-        return headers, frag_iters
1102
+        for part_infos in zip(*self.internal_parts_iters):
1103
+            frag_iters = [pi['part_iter'] for pi in part_infos]
1104
+            headers = HeaderKeyDict(part_infos[0]['headers'])
1105
+            yield headers, frag_iters
1127 1106
 
1128 1107
     def _actual_range(self, req_start, req_end, entity_length):
1129 1108
         # Takes 3 args: (requested-first-byte, requested-last-byte,
@@ -1272,11 +1251,7 @@ class ECAppIter(object):
1272 1251
             seen_first_headers = False
1273 1252
             ranges_for_resp = {}
1274 1253
 
1275
-            while True:
1276
-                # this'll raise StopIteration and exit the loop
1277
-                next_range = self._next_range()
1278
-
1279
-                headers, frag_iters = next_range
1254
+            for headers, frag_iters in self._next_ranges():
1280 1255
                 content_type = headers['Content-Type']
1281 1256
 
1282 1257
                 content_range = headers.get('Content-Range')

+ 13
- 4
test/functional/__init__.py View File

@@ -53,8 +53,7 @@ from test.unit import SkipTest
53 53
 
54 54
 from swift.common import constraints, utils, ring, storage_policy
55 55
 from swift.common.ring import Ring
56
-from swift.common.wsgi import (
57
-    monkey_patch_mimetools, loadapp, SwiftHttpProtocol)
56
+from swift.common.wsgi import loadapp, SwiftHttpProtocol
58 57
 from swift.common.utils import config_true_value, split_path
59 58
 from swift.account import server as account_server
60 59
 from swift.container import server as container_server
@@ -493,8 +492,6 @@ def in_process_setup(the_object_server=object_server):
493 492
     swift_conf_src = _in_process_find_conf_file(conf_src_dir, 'swift.conf')
494 493
     _info('Using swift config from %s' % swift_conf_src)
495 494
 
496
-    monkey_patch_mimetools()
497
-
498 495
     global _testdir
499 496
     _testdir = os.path.join(mkdtemp(), 'tmp_functional')
500 497
     utils.mkdirs(_testdir)
@@ -1292,3 +1289,15 @@ def requires_policies(f):
1292 1289
         return f(self, *args, **kwargs)
1293 1290
 
1294 1291
     return wrapper
1292
+
1293
+
1294
+def requires_bulk(f):
1295
+    @functools.wraps(f)
1296
+    def wrapper(*args, **kwargs):
1297
+        if skip or not cluster_info:
1298
+            raise SkipTest('Requires bulk middleware')
1299
+        # Determine whether this cluster has bulk middleware; if not, skip test
1300
+        if not cluster_info.get('bulk_upload', {}):
1301
+            raise SkipTest('Requires bulk middleware')
1302
+        return f(*args, **kwargs)
1303
+    return wrapper

+ 3
- 2
test/functional/test_dlo.py View File

@@ -49,7 +49,8 @@ class TestDloEnv(BaseEnv):
49 49
             file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter))
50 50
             file_item.write(letter * 10)
51 51
 
52
-            file_item = cls.container.file("%s/seg_upper%s" % (prefix, letter))
52
+            file_item = cls.container.file(
53
+                "%s/seg_upper_%%ff%s" % (prefix, letter))
53 54
             file_item.write(letter.upper() * 10)
54 55
 
55 56
         for letter in ('f', 'g', 'h', 'i', 'j'):
@@ -64,7 +65,7 @@ class TestDloEnv(BaseEnv):
64 65
 
65 66
         man2 = cls.container.file("man2")
66 67
         man2.write('man2-contents',
67
-                   hdrs={"X-Object-Manifest": "%s/%s/seg_upper" %
68
+                   hdrs={"X-Object-Manifest": "%s/%s/seg_upper_%%25ff" %
68 69
                          (cls.container.name, prefix)})
69 70
 
70 71
         manall = cls.container.file("manall")

+ 33
- 1
test/functional/test_object.py View File

@@ -20,11 +20,12 @@ import json
20 20
 import unittest2
21 21
 from uuid import uuid4
22 22
 import time
23
+from xml.dom import minidom
23 24
 
24 25
 from six.moves import range
25 26
 
26 27
 from test.functional import check_response, retry, requires_acls, \
27
-    requires_policies, SkipTest
28
+    requires_policies, SkipTest, requires_bulk
28 29
 import test.functional as tf
29 30
 
30 31
 
@@ -1646,6 +1647,37 @@ class TestObject(unittest2.TestCase):
1646 1647
             self.assertEqual(resp.status, 200)
1647 1648
             self.assertEqual(body, resp.read())
1648 1649
 
1650
+    @requires_bulk
1651
+    def test_bulk_delete(self):
1652
+
1653
+        def bulk_delete(url, token, parsed, conn):
1654
+            # try to bulk delete the object that was created during test setup
1655
+            conn.request('DELETE', '%s/%s/%s?bulk-delete' % (
1656
+                parsed.path, self.container, self.obj),
1657
+                '%s/%s' % (self.container, self.obj),
1658
+                {'X-Auth-Token': token,
1659
+                 'Accept': 'application/xml',
1660
+                 'Expect': '100-continue',
1661
+                 'Content-Type': 'text/plain'})
1662
+            return check_response(conn)
1663
+        resp = retry(bulk_delete)
1664
+        self.assertEqual(resp.status, 200)
1665