Browse Source

Merge branch 'master' into feature/crypto

Conflicts:
	swift/common/wsgi.py

Change-Id: Ic37a285c008b01c7babf29110cfe0180bfde69c9
changes/01/272201/2
Alistair Coles 6 years ago
parent
commit
f9b7fd3074
  1. 6
      .functests
  2. 2
      .gitignore
  3. 6
      .mailmap
  4. 4
      .testr.conf
  5. 32
      AUTHORS
  6. 89
      CHANGELOG
  7. 5
      bin/swift-dispersion-report
  8. 5
      bin/swift-init
  9. 3
      doc/manpages/proxy-server.conf.5
  10. 1
      doc/manpages/swift-init.1
  11. 7
      doc/source/admin_guide.rst
  12. 11
      doc/source/conf.py
  13. 249
      doc/source/deployment_guide.rst
  14. 9
      doc/source/development_auth.rst
  15. 6
      doc/source/development_saio.rst
  16. 73
      doc/source/overview_container_sync.rst
  17. 14
      doc/source/overview_erasure_code.rst
  18. 26
      doc/source/overview_policies.rst
  19. 4
      doc/source/policies_saio.rst
  20. 1
      etc/memcache.conf-sample
  21. 8
      etc/object-server.conf-sample
  22. 12
      etc/proxy-server.conf-sample
  23. 15
      etc/swift.conf-sample
  24. 2
      requirements.txt
  25. 4
      swift/account/reaper.py
  26. 20
      swift/cli/recon.py
  27. 29
      swift/cli/ringbuilder.py
  28. 13
      swift/common/db_replicator.py
  29. 6
      swift/common/direct_client.py
  30. 12
      swift/common/internal_client.py
  31. 30
      swift/common/manager.py
  32. 108
      swift/common/memcached.py
  33. 2
      swift/common/middleware/decrypter.py
  34. 5
      swift/common/middleware/dlo.py
  35. 2
      swift/common/middleware/fake_footers.py
  36. 4
      swift/common/middleware/keymaster.py
  37. 6
      swift/common/middleware/keystoneauth.py
  38. 65
      swift/common/middleware/slo.py
  39. 17
      swift/common/middleware/tempauth.py
  40. 4
      swift/common/middleware/tempurl.py
  41. 6
      swift/common/request_helpers.py
  42. 18
      swift/common/ring/builder.py
  43. 23
      swift/common/ring/ring.py
  44. 199
      swift/common/storage_policy.py
  45. 66
      swift/common/utils.py
  46. 2
      swift/common/wsgi.py
  47. 18
      swift/container/replicator.py
  48. 15
      swift/container/server.py
  49. 40
      swift/container/sync.py
  50. 177
      swift/container/sync_store.py
  51. 2
      swift/container/updater.py
  52. 17
      swift/locale/de/LC_MESSAGES/swift.po
  53. 17
      swift/locale/es/LC_MESSAGES/swift.po
  54. 17
      swift/locale/fr/LC_MESSAGES/swift.po
  55. 17
      swift/locale/it/LC_MESSAGES/swift.po
  56. 17
      swift/locale/ja/LC_MESSAGES/swift.po
  57. 19
      swift/locale/ko_KR/LC_MESSAGES/swift.po
  58. 19
      swift/locale/pt_BR/LC_MESSAGES/swift.po
  59. 128
      swift/locale/ru/LC_MESSAGES/swift.po
  60. 437
      swift/locale/swift.pot
  61. 19
      swift/locale/tr_TR/LC_MESSAGES/swift.po
  62. 19
      swift/locale/zh_CN/LC_MESSAGES/swift.po
  63. 19
      swift/locale/zh_TW/LC_MESSAGES/swift.po
  64. 26
      swift/obj/auditor.py
  65. 638
      swift/obj/diskfile.py
  66. 3
      swift/obj/mem_diskfile.py
  67. 4
      swift/obj/reconstructor.py
  68. 22
      swift/obj/replicator.py
  69. 2
      swift/obj/server.py
  70. 9
      swift/obj/updater.py
  71. 14
      swift/proxy/controllers/base.py
  72. 17
      swift/proxy/controllers/obj.py
  73. 7
      swift/proxy/server.py
  74. 1
      test-requirements.txt
  75. 52
      test/functional/__init__.py
  76. 2
      test/functional/swift_test_client.py
  77. 18
      test/functional/test_account.py
  78. 18
      test/functional/test_container.py
  79. 16
      test/functional/test_object.py
  80. 136
      test/functional/tests.py
  81. 78
      test/probe/test_container_sync.py
  82. 16
      test/unit/__init__.py
  83. 4
      test/unit/account/test_backend.py
  84. 14
      test/unit/cli/test_recon.py
  85. 61
      test/unit/cli/test_ringbuilder.py
  86. 9
      test/unit/common/middleware/helpers.py
  87. 389
      test/unit/common/middleware/test_slo.py
  88. 27
      test/unit/common/middleware/test_tempauth.py
  89. 228
      test/unit/common/ring/test_builder.py
  90. 5
      test/unit/common/ring/test_ring.py
  91. 9
      test/unit/common/test_internal_client.py
  92. 90
      test/unit/common/test_manager.py
  93. 153
      test/unit/common/test_memcached.py
  94. 245
      test/unit/common/test_storage_policy.py
  95. 114
      test/unit/common/test_swob.py
  96. 64
      test/unit/common/test_utils.py
  97. 7
      test/unit/common/test_wsgi.py
  98. 4
      test/unit/container/test_backend.py
  99. 2
      test/unit/container/test_reconciler.py
  100. 133
      test/unit/container/test_replicator.py

6
.functests

@ -1,9 +1,11 @@
#!/bin/bash
SRC_DIR=$(python -c "import os; print os.path.dirname(os.path.realpath('$0'))")
set -e
cd ${SRC_DIR}/test/functional
nosetests --exe $@
cd ${SRC_DIR}
export TESTS_DIR=${SRC_DIR}/test/functional
ostestr --serial --pretty $@
rvalue=$?
cd -

2
.gitignore

@ -15,4 +15,6 @@ pycscope.*
.idea
MANIFEST
.testrepository/*
subunit.log
test/probe/.noseids

6
.mailmap

@ -87,3 +87,9 @@ Donagh McCabe <donagh.mccabe@hpe.com> <donagh.mccabe@hp.com>
Eamonn O'Toole <eamonn.otoole@hpe.com> <eamonn.otoole@hp.com>
Gerry Drudy <gerry.drudy@hpe.com> <gerry.drudy@hp.com>
Mark Seger <mark.seger@hpe.com> <mark.seger@hp.com>
Timur Alperovich <timur.alperovich@gmail.com> <timuralp@swiftstack.com>
Mehdi Abaakouk <sileht@redhat.com> <mehdi.abaakouk@enovance.com>
Richard Hawkins <richard.hawkins@rackspace.com> <hurricanerix@gmail.com>
Ondrej Novy <ondrej.novy@firma.seznam.cz>
Peter Lisak <peter.lisak@firma.seznam.cz>
Ke Liang <ke.liang@easystack.cn>

4
.testr.conf

@ -0,0 +1,4 @@
[DEFAULT]
test_command=SWIFT_TEST_DEBUG_LOGS=${SWIFT_TEST_DEBUG_LOGS} ${PYTHON:-python} -m subunit.run discover -t ./ ${TESTS_DIR:-./test/functional/} $LISTOPT $IDOPTION
test_id_option=--load-list $IDFILE
test_list_option=--list

32
AUTHORS

@ -25,7 +25,7 @@ Chuck Thier (cthier@gmail.com)
Contributors
------------
Mehdi Abaakouk (mehdi.abaakouk@enovance.com)
Mehdi Abaakouk (sileht@redhat.com)
Timur Alperovich (timur.alperovich@gmail.com)
Jesse Andrews (anotherjesse@gmail.com)
Joe Arnold (joe@swiftstack.com)
@ -41,7 +41,7 @@ James E. Blair (jeblair@openstack.org)
Fabien Boucher (fabien.boucher@enovance.com)
Clark Boylan (clark.boylan@gmail.com)
Pádraig Brady (pbrady@redhat.com)
Lorcan Browne (lorcan.browne@hp.com)
Lorcan Browne (lorcan.browne@hpe.com)
Russell Bryant (rbryant@redhat.com)
Jay S. Bryant (jsbryant@us.ibm.com)
Tim Burke (tim.burke@gmail.com)
@ -56,15 +56,17 @@ François Charlier (francois.charlier@enovance.com)
Ray Chen (oldsharp@163.com)
Harshit Chitalia (harshit@acelio.com)
Brian Cline (bcline@softlayer.com)
Alistair Coles (alistair.coles@hp.com)
Alistair Coles (alistair.coles@hpe.com)
Clément Contini (ccontini@cloudops.com)
Brian Curtin (brian.curtin@rackspace.com)
Thiago da Silva (thiago@redhat.com)
Julien Danjou (julien@danjou.info)
Paul Dardeau (paul.dardeau@intel.com)
Zack M. Davis (zdavis@swiftstack.com)
Ksenia Demina (kdemina@mirantis.com)
Dan Dillinger (dan.dillinger@sonian.net)
Cedric Dos Santos (cedric.dos.sant@gmail.com)
Gerry Drudy (gerry.drudy@hp.com)
Gerry Drudy (gerry.drudy@hpe.com)
Morgan Fainberg (morgan.fainberg@gmail.com)
ZhiQiang Fan (aji.zqfan@gmail.com)
Oshrit Feder (oshritf@il.ibm.com)
@ -85,6 +87,7 @@ David Goetz (david.goetz@rackspace.com)
Tushar Gohad (tushar.gohad@intel.com)
Jonathan Gonzalez V (jonathan.abdiel@gmail.com)
Joe Gordon (jogo@cloudscaling.com)
ChangBo Guo(gcb) (eric.guo@easystack.cn)
David Hadas (davidh@il.ibm.com)
Andrew Hale (andy@wwwdata.eu)
Soren Hansen (soren@linux2go.dk)
@ -92,9 +95,12 @@ Richard Hawkins (richard.hawkins@rackspace.com)
Gregory Haynes (greg@greghaynes.net)
Doug Hellmann (doug.hellmann@dreamhost.com)
Dan Hersam (dan.hersam@hp.com)
hgangwx (hgangwx@cn.ibm.com)
Derek Higgins (derekh@redhat.com)
Jonathan Hinson (jlhinson@us.ibm.com)
Alex Holden (alex@alexjonasholden.com)
Edward Hope-Morley (opentastic@gmail.com)
Ferenc Horváth (hferenc@inf.u-szeged.hu)
Charles Hsu (charles0126@gmail.com)
Joanna H. Huang (joanna.huitzu.huang@gmail.com)
Kun Huang (gareth@unitedstack.com)
@ -111,6 +117,7 @@ Jason Johnson (jajohnson@softlayer.com)
Brian K. Jones (bkjones@gmail.com)
Arnaud JOST (arnaud.jost@ovh.net)
Kiyoung Jung (kiyoung.jung@kt.com)
Harshada Mangesh Kakad (harshadak@metsi.co.uk)
Takashi Kajinami (kajinamit@nttdata.co.jp)
Matt Kassawara (mkassawara@gmail.com)
Morita Kazutaka (morita.kazutaka@gmail.com)
@ -136,6 +143,8 @@ Eohyung Lee (liquidnuker@gmail.com)
Zhao Lei (zhaolei@cn.fujitsu.com)
Jamie Lennox (jlennox@redhat.com)
Tong Li (litong01@us.ibm.com)
Ke Liang (ke.liang@easystack.cn)
Peter Lisak (peter.lisak@firma.seznam.cz)
Changbin Liu (changbin.liu@gmail.com)
Jing Liuqing (jing.liuqing@99cloud.net)
Victor Lowther (victor.lowther@gmail.com)
@ -143,6 +152,7 @@ Sergey Lukjanov (slukjanov@mirantis.com)
Zhongyue Luo (zhongyue.nah@intel.com)
Paul Luse (paul.e.luse@intel.com)
Christopher MacGown (chris@pistoncloud.com)
Ganesh Maharaj Mahalingam (ganesh.mahalingam@intel.com)
Dragos Manolescu (dragosm@hp.com)
Ben Martin (blmartin@us.ibm.com)
Steve Martinelli (stevemar@ca.ibm.com)
@ -152,7 +162,7 @@ Nakagawa Masaaki (nakagawamsa@nttdata.co.jp)
Dolph Mathews (dolph.mathews@gmail.com)
Kenichiro Matsuda (matsuda_kenichi@jp.fujitsu.com)
Michael Matur (michael.matur@gmail.com)
Donagh McCabe (donagh.mccabe@hp.com)
Donagh McCabe (donagh.mccabe@hpe.com)
Andy McCrae (andy.mccrae@gmail.com)
Paul McMillan (paul.mcmillan@nebula.com)
Ewan Mellor (ewan.mellor@citrix.com)
@ -168,19 +178,22 @@ Maru Newby (mnewby@internap.com)
Newptone (xingchao@unitedstack.com)
Colin Nicholson (colin.nicholson@iomart.com)
Zhenguo Niu (zhenguo@unitedstack.com)
Catherine Northcott (catherine@northcott.nz)
Ondrej Novy (ondrej.novy@firma.seznam.cz)
Timothy Okwii (tokwii@cisco.com)
Matthew Oliver (matt@oliver.net.au)
Hisashi Osanai (osanai.hisashi@jp.fujitsu.com)
Eamonn O'Toole (eamonn.otoole@hp.com)
Eamonn O'Toole (eamonn.otoole@hpe.com)
James Page (james.page@ubuntu.com)
Prashanth Pai (ppai@redhat.com)
Venkateswarlu Pallamala (p.venkatesh551@gmail.com)
Pawel Palucki (pawel.palucki@gmail.com)
Alex Pecoraro (alex.pecoraro@emc.com)
Sascha Peilicke (saschpe@gmx.de)
Constantine Peresypkin (constantine.peresypk@rackspace.com)
Dieter Plaetinck (dieter@vimeo.com)
Dan Prince (dprince@redhat.com)
Sivasathurappan Radhakrishnan (siva.radhakrishnan@intel.com)
Sarvesh Ranjan (saranjan@cisco.com)
Falk Reimann (falk.reimann@sap.com)
Brian Reitz (brian.reitz@oracle.com)
@ -198,7 +211,7 @@ Shilla Saebi (shilla.saebi@gmail.com)
Atsushi Sakai (sakaia@jp.fujitsu.com)
Cristian A Sanchez (cristian.a.sanchez@intel.com)
Christian Schwede (cschwede@redhat.com)
Mark Seger (Mark.Seger@hp.com)
Mark Seger (mark.seger@hpe.com)
Azhagu Selvan SP (tamizhgeek@gmail.com)
Alexandra Settle (alexandra.settle@rackspace.com)
Andrew Clay Shafer (acs@parvuscaptus.com)
@ -212,6 +225,7 @@ Pradeep Kumar Singh (pradeep.singh@nectechnologies.in)
Liu Siqi (meizu647@gmail.com)
Adrian Smith (adrian_f_smith@dell.com)
Jon Snitow (otherjon@swiftstack.com)
Emile Snyder (emile.snyder@gmail.com)
Emett Speer (speer.emett@gmail.com)
TheSriram (sriram@klusterkloud.com)
Jeremy Stanley (fungi@yuggoth.org)
@ -234,7 +248,9 @@ Dmitry Ukov (dukov@mirantis.com)
Vincent Untz (vuntz@suse.com)
Daniele Valeriani (daniele@dvaleriani.net)
Koert van der Veer (koert@cloudvps.com)
Béla Vancsics (vancsics@inf.u-szeged.hu)
Vladimir Vechkanov (vvechkanov@mirantis.com)
venkatamahesh (venkatamaheshkotha@gmail.com)
Gil Vernik (gilv@il.ibm.com)
Hou Ming Wang (houming.wang@easystack.cn)
Shane Wang (shane.wang@intel.com)
@ -248,7 +264,7 @@ Ye Jia Xu (xyj.asmy@gmail.com)
Alex Yang (alex890714@gmail.com)
Lin Yang (lin.a.yang@intel.com)
Yee (mail.zhang.yee@gmail.com)
Guang Yee (guang.yee@hp.com)
Guang Yee (guang.yee@hpe.com)
Pete Zaitcev (zaitcev@kotori.zaitcev.us)
Hua Zhang (zhuadl@cn.ibm.com)
Jian Zhang (jian.zhang@intel.com)

89
CHANGELOG

@ -1,3 +1,92 @@
swift (2.6.0)
* Dependency changes
- Updated minimum version of eventlet to 0.17.4 to support IPv6.
- Updated the minimum version of PyECLib to 1.0.7.
* The ring rebalancing algorithm was updated to better handle edge cases
and to give better (more balanced) rings in the general case. New rings
will have better initial placement, capacity adjustments will move less
data for better balance, and existing rings that were imbalanced should
start to become better balanced as they go through rebalance cycles.
* Added container and account reverse listings.
A GET request to an account or container resource with a "reverse=true"
query parameter will return the listing in reverse order. When
iterating over pages of reverse listings, the relative order of marker
and end_marker are swapped.
* Storage policies now support having more than one name.
This allows operators to fix a typo without breaking existing clients,
or, alternatively, have "short names" for policies. This is implemented
with the "aliases" config key in the storage policy config in
swift.conf. The aliases value is a list of names that the storage
policy may also be identified by. The storage policy "name" is used to
report the policy to users (eg in container headers). The aliases have
the same naming restrictions as the policy's primary name.
* The object auditor learned the "interval" config value to control the
time between each audit pass.
* `swift-recon --all` now includes the config checksum check.
* `swift-init` learned the --kill-after-timeout option to force a service
to quit (SIGKILL) after a designated time.
* `swift-recon` now correctly shows timestamps in UTC instead of local
time.
* Fixed bug where `swift-ring-builder` couldn't select device id 0.
* Documented the previously undocumented
`swift-ring-builder pretend_min_part_hours_passed` command.
* The "node_timeout" config value now accepts decimal values.
* `swift-ring-builder` now properly removes devices with zero weight.
* `swift-init` return codes are updated via "--strict" and "--non-strict"
options. Please see the usage string for more information.
* `swift-ring-builder` now reports the min_part_hours lockout time
remaining
* Container sync has been improved to more quickly find and iterate over
the containers to be synced. This reduced server load and lowers the
time required to see data propagate between two clusters. Please see
http://swift.openstack.org/overview_container_sync.html for more details
about the new on-disk structure for tracking synchronized containers.
* A container POST will now update that container's put-timestamp value.
* TempURL header restrictions are now exposed in /info.
* Error messages on static large object manifest responses have been
greatly improved.
* Closed a bug where an unfinished read of a large object would leak a
socket file descriptor and a small amount of memory. (CVE-2016-0738)
* Fixed an issue where a zero-byte object PUT with an incorrect Etag
would return a 503.
* Fixed an error when a static large object manifest references the same
object more than once.
* Improved performance of finding handoff nodes if a zone is empty.
* Fixed duplication of headers in Access-Control-Expose-Headers on CORS
requests.
* Fixed handling of IPv6 connections to memcache pools.
* Continued work towards python 3 compatibility.
* Various other minor bug fixes and improvements.
swift (2.5.0, OpenStack Liberty)
* Added the ability to specify ranges for Static Large Object (SLO)

5
bin/swift-dispersion-report

@ -23,7 +23,6 @@ from time import time
from eventlet import GreenPool, hubs, patcher, Timeout
from eventlet.pools import Pool
from eventlet.green import urllib2
from swift.common import direct_client
try:
@ -174,8 +173,8 @@ def object_dispersion_report(coropool, connpool, account, object_ring,
try:
objects = [o['name'] for o in conn.get_container(
container, prefix='dispersion_', full_listing=True)[1]]
except urllib2.HTTPError as err:
if err.getcode() != 404:
except ClientException as err:
if err.http_status != 404:
raise
print >>stderr, 'No objects to query. Has ' \

5
bin/swift-init

@ -74,6 +74,11 @@ def main():
help="Return zero status code even if some config is "
"missing. Default mode if any server is a glob or "
"one of aliases `all`, `main` or `rest`.")
# SIGKILL daemon after kill_wait period
parser.add_option('--kill-after-timeout', dest='kill_after_timeout',
action='store_true',
help="Kill daemon and all childs after kill-wait "
"period.")
options, args = parser.parse_args()

3
doc/manpages/proxy-server.conf.5

@ -384,7 +384,8 @@ Sets the maximum number of connections to each memcached server per worker.
If not set in the configuration file, the value for memcache_servers will be
read from /etc/swift/memcache.conf (see memcache.conf-sample) or lacking that
file, it will default to 127.0.0.1:11211. You can specify multiple servers
separated with commas, as in: 10.1.2.3:11211,10.1.2.4:11211.
separated with commas, as in: 10.1.2.3:11211,10.1.2.4:11211. (IPv6
addresses must follow rfc3986 section-3.2.2, i.e. [::1]:11211)
.IP \fBmemcache_serialization_support\fR
This sets how memcache values are serialized and deserialized:
.RE

1
doc/manpages/swift-init.1

@ -111,6 +111,7 @@ allows one to use the keywords such as "all", "main" and "rest" for the <server>
.IP "-r RUN_DIR, --run-dir=RUN_DIR directory where the pids will be stored (default /var/run/swift)
.IP "--strict return non-zero status code if some config is missing. Default mode if server is explicitly named."
.IP "--non-strict return zero status code even if some config is missing. Default mode if server is one of aliases `all`, `main` or `rest`."
.IP "--kill-after-timeout kill daemon and all childs after kill-wait period."
.PD
.RE

7
doc/source/admin_guide.rst

@ -463,7 +463,12 @@ Example::
Assuming 3 replicas, this configuration will make object PUTs try
storing the object's replicas on up to 6 disks ("2 * replicas") in
region 1 ("r1").
region 1 ("r1"). Proxy server tries to find 3 devices for storing the
object. While a device is unavailable, it queries the ring for the 4th
device and so on until 6th device. If the 6th disk is still unavailable,
the last replica will be sent to other region. It doesn't mean there'll
have 6 replicas in region 1.
You should be aware that, if you have data coming into SF faster than
your link to NY can transfer it, then your cluster's data distribution

11
doc/source/conf.py

@ -13,9 +13,10 @@
# All configuration values have a default; values that are commented out
# serve to show the default.
import sys
import os
import datetime
import os
import subprocess
import sys
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
@ -144,8 +145,10 @@ modindex_common_prefix = ['swift.']
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'
git_cmd = "git log --pretty=format:'%ad, commit %h' --date=local -n1"
html_last_updated_fmt = os.popen(git_cmd).read()
git_cmd = ["git", "log", "--pretty=format:'%ad, commit %h'", "--date=local",
"-n1"]
html_last_updated_fmt = subprocess.Popen(
git_cmd, stdout=subprocess.PIPE).communicate()[0]
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.

249
doc/source/deployment_guide.rst

@ -510,9 +510,9 @@ container_update_timeout 1 Time to wait while sending a contai
[object-server]
============================= ====================== =================================
============================= ====================== ===============================================
Option Default Description
----------------------------- ---------------------- ---------------------------------
----------------------------- ---------------------- -----------------------------------------------
use paste.deploy entry point for the
object server. For most cases,
this should be
@ -537,9 +537,9 @@ keep_cache_private false Allow non-public objects t
in kernel's buffer cache
allowed_headers Content-Disposition, Comma separated list of headers
Content-Encoding, that can be set in metadata on an object.
X-Delete-At, This list is in addition to X-Object-Meta-* headers and cannot include
X-Object-Manifest, Content-Type, etag, Content-Length, or deleted
X-Static-Large-Object
X-Delete-At, This list is in addition to
X-Object-Manifest, X-Object-Meta-* headers and cannot include
X-Static-Large-Object Content-Type, etag, Content-Length, or deleted
auto_create_account_prefix . Prefix used when automatically
creating accounts.
threads_per_disk 0 Size of the per-disk thread pool
@ -596,98 +596,99 @@ splice no Use splice() for zero-copy
will appear in the object server
logs at startup, but your object
servers should continue to function.
============================= ====================== =================================
============================= ====================== ===============================================
[object-replicator]
================== ======================== ================================
Option Default Description
------------------ ------------------------ --------------------------------
log_name object-replicator Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
log_address /dev/log Logging directory
daemonize yes Whether or not to run replication
as a daemon
interval 30 Time in seconds to wait between
replication passes
concurrency 1 Number of replication workers to
spawn
sync_method rsync The sync method to use; default is
rsync but you can use ssync to try the
EXPERIMENTAL all-swift-code-no-rsync-callouts
method. Once ssync is verified
as having performance comparable to,
or better than, rsync, we plan to
deprecate rsync so we can move on
with more features for replication.
rsync_timeout 900 Max duration of a partition rsync
rsync_bwlimit 0 Bandwidth limit for rsync in kB/s.
0 means unlimited.
rsync_io_timeout 30 Timeout value sent to rsync
--timeout and --contimeout
options
rsync_compress no Allow rsync to compress data
which is transmitted to destination
node during sync. However, this
is applicable only when destination
node is in a different region
than the local one.
NOTE: Objects that are already
compressed (for example: .tar.gz,
.mp3) might slow down the syncing
process.
stats_interval 300 Interval in seconds between
logging replication statistics
reclaim_age 604800 Time elapsed in seconds before an
object can be reclaimed
handoffs_first false If set to True, partitions that
are not supposed to be on the
node will be replicated first.
The default setting should not be
changed, except for extreme
situations.
handoff_delete auto By default handoff partitions
will be removed when it has
successfully replicated to all
the canonical nodes. If set to an
integer n, it will remove the
partition if it is successfully
replicated to n nodes. The
default setting should not be
changed, except for extreme
situations.
node_timeout DEFAULT or 10 Request timeout to external
services. This uses what's set
here, or what's set in the
DEFAULT section, or 10 (though
other sections use 3 as the final
default).
http_timeout 60 Max duration of an http request.
This is for REPLICATE finalization
calls and so should be longer
than node_timeout.
lockup_timeout 1800 Attempts to kill all workers if
nothing replicates for
lockup_timeout seconds
rsync_module {replication_ip}::object Format of the rsync module where
the replicator will send data.
The configuration value can
include some variables that will
be extracted from the ring.
Variables must follow the format
{NAME} where NAME is one of: ip,
port, replication_ip,
replication_port, region, zone,
device, meta. See
etc/rsyncd.conf-sample for some
examples.
rsync_error_log_line_length 0 Limits how long rsync error log
lines are
ring_check_interval 15 Interval for checking new ring
file
recon_cache_path /var/cache/swift Path to recon cache
================== ======================== ================================
=========================== ======================== ================================
Option Default Description
--------------------------- ------------------------ --------------------------------
log_name object-replicator Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
log_address /dev/log Logging directory
daemonize yes Whether or not to run replication
as a daemon
interval 30 Time in seconds to wait between
replication passes
concurrency 1 Number of replication workers to
spawn
sync_method rsync The sync method to use; default
is rsync but you can use ssync to
try the EXPERIMENTAL
all-swift-code-no-rsync-callouts
method. Once ssync is verified as
or better than, rsync, we plan to
deprecate rsync so we can move on
with more features for
replication.
rsync_timeout 900 Max duration of a partition rsync
rsync_bwlimit 0 Bandwidth limit for rsync in kB/s.
0 means unlimited.
rsync_io_timeout 30 Timeout value sent to rsync
--timeout and --contimeout
options
rsync_compress no Allow rsync to compress data
which is transmitted to destination
node during sync. However, this
is applicable only when destination
node is in a different region
than the local one.
NOTE: Objects that are already
compressed (for example: .tar.gz,
.mp3) might slow down the syncing
process.
stats_interval 300 Interval in seconds between
logging replication statistics
reclaim_age 604800 Time elapsed in seconds before an
object can be reclaimed
handoffs_first false If set to True, partitions that
are not supposed to be on the
node will be replicated first.
The default setting should not be
changed, except for extreme
situations.
handoff_delete auto By default handoff partitions
will be removed when it has
successfully replicated to all
the canonical nodes. If set to an
integer n, it will remove the
partition if it is successfully
replicated to n nodes. The
default setting should not be
changed, except for extreme
situations.
node_timeout DEFAULT or 10 Request timeout to external
services. This uses what's set
here, or what's set in the
DEFAULT section, or 10 (though
other sections use 3 as the final
default).
http_timeout 60 Max duration of an http request.
This is for REPLICATE finalization
calls and so should be longer
than node_timeout.
lockup_timeout 1800 Attempts to kill all workers if
nothing replicates for
lockup_timeout seconds
rsync_module {replication_ip}::object Format of the rsync module where
the replicator will send data.
The configuration value can
include some variables that will
be extracted from the ring.
Variables must follow the format
{NAME} where NAME is one of: ip,
port, replication_ip,
replication_port, region, zone,
device, meta. See
etc/rsyncd.conf-sample for some
examples.
rsync_error_log_line_length 0 Limits how long rsync error log
lines are
ring_check_interval 15 Interval for checking new ring
file
recon_cache_path /var/cache/swift Path to recon cache
=========================== ======================== ================================
[object-updater]
@ -718,6 +719,8 @@ log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
log_address /dev/log Logging directory
log_time 3600 Frequency of status logs in seconds.
interval 30 Time in seconds to wait between
auditor passes
disk_chunk_size 65536 Size of chunks read during auditing
files_per_second 20 Maximum files audited per second per
auditor process. Should be tuned according
@ -822,7 +825,7 @@ set log_address /dev/log Logging directory
node_timeout 3 Request timeout to external services
conn_timeout 0.5 Connection timeout to external services
allow_versions false Enable/Disable object versioning feature
auto_create_account_prefix . Prefix used when automatically
auto_create_account_prefix . Prefix used when automatically
replication_server Configure parameter for creating
specific server. To handle all verbs,
including replication verbs, do not
@ -887,15 +890,15 @@ rsync_module {replication_ip}::container Format of the rsync module
etc/rsyncd.conf-sample for
some examples.
rsync_compress no Allow rsync to compress data
which is transmitted to destination
node during sync. However, this
is applicable only when destination
node is in a different region
than the local one.
NOTE: Objects that are already
compressed (for example: .tar.gz,
.mp3) might slow down the syncing
process.
which is transmitted to
destination node during sync.
However, this is applicable
only when destination node is
in a different region than the
local one. NOTE: Objects that
are already compressed (for
example: .tar.gz, mp3) might
slow down the syncing process.
recon_cache_path /var/cache/swift Path to recon cache
================== =========================== =============================
@ -1090,15 +1093,15 @@ rsync_module {replication_ip}::account Format of the rsync module where
etc/rsyncd.conf-sample for some
examples.
rsync_compress no Allow rsync to compress data
which is transmitted to destination
node during sync. However, this
is applicable only when destination
node is in a different region
than the local one.
NOTE: Objects that are already
compressed (for example: .tar.gz,
.mp3) might slow down the syncing
process.
which is transmitted to
destination node during sync.
However, this is applicable only
when destination node is in a
different region than the local
one. NOTE: Objects that are
already compressed (for example:
.tar.gz, mp3) might slow down
the syncing process.
recon_cache_path /var/cache/swift Path to recon cache
================== ========================= ===============================
@ -1159,9 +1162,9 @@ The following configuration options are available:
[DEFAULT]
==================================== ======================== =============================
==================================== ======================== ========================================
Option Default Description
------------------------------------ ------------------------ -----------------------------
------------------------------------ ------------------------ ----------------------------------------
bind_ip 0.0.0.0 IP Address for server to
bind to
bind_port 80 Port for server to bind to
@ -1205,11 +1208,12 @@ cors_allow_origin This is a list o
strict_cors_mode True
client_timeout 60
trans_id_suffix This optional suffix (default is empty)
that would be appended to the swift transaction
id allows one to easily figure out from
which cluster that X-Trans-Id belongs to.
This is very useful when one is managing
more than one swift cluster.
that would be appended to the swift
transaction id allows one to easily
figure out from which cluster that
X-Trans-Id belongs to. This is very
useful when one is managing more than
one swift cluster.
log_name swift Label used when logging
log_facility LOG_LOCAL0 Syslog log facility
log_level INFO Logging level
@ -1246,7 +1250,7 @@ disallowed_sections swift.valid_api_versions Allows the abili
the dict level with a ".".
expiring_objects_container_divisor 86400
expiring_objects_account_name expiring_objects
==================================== ======================== =============================
==================================== ======================== ========================================
[proxy-server]
@ -1276,7 +1280,8 @@ object_chunk_size 65536 Chunk size to read from
client_chunk_size 65536 Chunk size to read from
clients
memcache_servers 127.0.0.1:11211 Comma separated list of
memcached servers ip:port
memcached servers
ip:port or [ipv6addr]:port
memcache_max_connections 2 Max number of connections to
each memcached server per
worker

9
doc/source/development_auth.rst

@ -375,7 +375,7 @@ folks a start on their own code if they want to use repoze.what::
expiration = float(resp.getheader('x-auth-ttl'))
user = resp.getheader('x-auth-user')
memcache_client.set(key, (time(), expiration, user),
timeout=expiration)
time=expiration)
return user
return None
@ -487,7 +487,8 @@ folks a start on their own code if they want to use repoze.what::
Allowing CORS with Auth
-----------------------
Cross Origin RequestS require that the auth system allow the OPTIONS method to
pass through without a token. The preflight request will make an OPTIONS call
against the object or container and will not work if the auth system stops it.
Cross Origin Resource Sharing (CORS) require that the auth system allow the
OPTIONS method to pass through without a token. The preflight request will
make an OPTIONS call against the object or container and will not work if
the auth system stops it.
See TempAuth for an example of how OPTIONS requests are handled.

6
doc/source/development_saio.rst

@ -37,7 +37,8 @@ Installing dependencies
sudo apt-get update
sudo apt-get install curl gcc memcached rsync sqlite3 xfsprogs \
git-core libffi-dev python-setuptools
git-core libffi-dev python-setuptools \
liberasurecode-dev
sudo apt-get install python-coverage python-dev python-nose \
python-xattr python-eventlet \
python-greenlet python-pastedeploy \
@ -48,7 +49,8 @@ Installing dependencies
sudo yum update
sudo yum install curl gcc memcached rsync sqlite xfsprogs git-core \
libffi-devel xinetd python-setuptools \
libffi-devel xinetd liberasurecode-devel \
python-setuptools \
python-coverage python-devel python-nose \
pyxattr python-eventlet \
python-greenlet python-paste-deploy \

73
doc/source/overview_container_sync.rst

@ -29,7 +29,7 @@ synchronization key.
Configuring Container Sync
--------------------------
Create a container-sync-realms.conf file specifying the allowable clusters
Create a ``container-sync-realms.conf`` file specifying the allowable clusters
and their information::
[realm1]
@ -50,18 +50,18 @@ clusters that have agreed to allow container syncing with each other. Realm
names will be considered case insensitive.
The key is the overall cluster-to-cluster key used in combination with the
external users' key that they set on their containers' X-Container-Sync-Key
metadata header values. These keys will be used to sign each request the
container sync daemon makes and used to validate each incoming container sync
request.
external users' key that they set on their containers'
``X-Container-Sync-Key`` metadata header values. These keys will be used to
sign each request the container sync daemon makes and used to validate each
incoming container sync request.
The key2 is optional and is an additional key incoming requests will be checked
against. This is so you can rotate keys if you wish; you move the existing key
to key2 and make a new key value.
Any values in the realm section whose names begin with cluster\_ will indicate
the name and endpoint of a cluster and will be used by external users in
their containers' X-Container-Sync-To metadata header values with the format
Any values in the realm section whose names begin with ``cluster_`` will
indicate the name and endpoint of a cluster and will be used by external users in
their containers' ``X-Container-Sync-To`` metadata header values with the format
"//realm_name/cluster_name/account_name/container_name". Realm and cluster
names are considered case insensitive.
@ -71,7 +71,7 @@ container servers, since that is where the container sync daemon runs. Note
that the endpoint ends with /v1/ and that the container sync daemon will then
add the account/container/obj name after that.
Distribute this container-sync-realms.conf file to all your proxy servers
Distribute this ``container-sync-realms.conf`` file to all your proxy servers
and container servers.
You also need to add the container_sync middleware to your proxy pipeline. It
@ -95,7 +95,7 @@ section, Configuring Container Sync, for the new-style.
With the old-style, the Swift cluster operator must allow synchronization with
a set of hosts before the user can enable container synchronization. First, the
backend container server needs to be given this list of hosts in the
container-server.conf file::
``container-server.conf`` file::
[DEFAULT]
# This is a comma separated list of hosts allowed in the
@ -170,8 +170,8 @@ we'll make next::
The ``-t`` indicates the cluster to sync to, which is the realm name of the
section from container-sync-realms.conf, followed by the cluster name from
that section (without the cluster\_ prefix), followed by the account and container names we want to sync to.
The ``-k`` specifies the secret key the two containers will share for
that section (without the cluster\_ prefix), followed by the account and container
names we want to sync to. The ``-k`` specifies the secret key the two containers will share for
synchronization; this is the user key, the cluster key in
container-sync-realms.conf will also be used behind the scenes.
@ -195,8 +195,18 @@ as it gets synchronized over to the second::
list container2
[Nothing there yet, so we wait a bit...]
[If you're an operator running SAIO and just testing, you may need to
run 'swift-init container-sync once' to perform a sync scan.]
.. note::
If you're an operator running SAIO and just testing, each time you
configure a container for synchronization and place objects in the
source container you will need to ensure that container-sync runs
before attempting to retrieve objects from the target container.
That is, you need to run::
swift-init container-sync once
Now expect to see objects copied from the first container to the second::
$ swift -A http://cluster2/auth/v1.0 -U test2:tester2 -K testing2 \
list container2
@ -340,13 +350,34 @@ synchronize to the second, we could have used this curl command::
What's going on behind the scenes, in the cluster?
--------------------------------------------------
The swift-container-sync does the job of sending updates to the remote
container.
This is done by scanning the local devices for container databases and
checking for x-container-sync-to and x-container-sync-key metadata values.
If they exist, newer rows since the last sync will trigger PUTs or DELETEs
to the other container.
Container ring devices have a directory called ``containers``, where container
databases reside. In addition to ``containers``, each container ring device
also has a directory called ``sync-containers``. ``sync-containers`` holds
symlinks to container databases that were configured for container sync using
``x-container-sync-to`` and ``x-container-sync-key`` metadata keys.
The swift-container-sync process does the job of sending updates to the remote
container. This is done by scanning ``sync-containers`` for container
databases. For each container db found, newer rows since the last sync will
trigger PUTs or DELETEs to the other container.
``sync-containers`` is maintained as follows:
Whenever the container-server processes a PUT or a POST request that carries
``x-container-sync-to`` and ``x-container-sync-key`` metadata keys the server
creates a symlink to the container database in ``sync-containers``. Whenever
the container server deletes a synced container, the appropriate symlink
is deleted from ``sync-containers``.
In addition to the container-server, the container-replicator process does the
job of identifying containers that should be synchronized. This is done by
scanning the local devices for container databases and checking for
x-container-sync-to and x-container-sync-key metadata values. If they exist
then a symlink to the container database is created in a sync-containers
sub-directory on the same device.
Similarly, when the container sync metadata keys are deleted, the container
server and container-replicator would take care of deleting the symlinks
from ``sync-containers``.
.. note::

14
doc/source/overview_erasure_code.rst

@ -2,20 +2,6 @@
Erasure Code Support
====================
--------------------------
Beta: Not production ready
--------------------------
The erasure code support in Swift is considered "beta" at this point.
Most major functionality is included, but it has not been tested or validated
at large scale. This feature relies on ssync for durability. Deployers are
urged to do extensive testing and not deploy production data using an
erasure code storage policy.
If any bugs are found during testing, please report them to
https://bugs.launchpad.net/swift
-------------------------------
History and Theory of Operation
-------------------------------

26
doc/source/overview_policies.rst

@ -57,7 +57,7 @@ deployers. Each container has a new special immutable metadata element called
the storage policy index. Note that internally, Swift relies on policy
indexes and not policy names. Policy names exist for human readability and
translation is managed in the proxy. When a container is created, one new
optional header is supported to specify the policy name. If nothing is
optional header is supported to specify the policy name. If no name is
specified, the default policy is used (and if no other policies defined,
Policy-0 is considered the default). We will be covering the difference
between default and Policy-0 in the next section.
@ -170,12 +170,13 @@ Storage Policies is a versatile feature intended to support both new and
pre-existing clusters with the same level of flexibility. For that reason, we
introduce the ``Policy-0`` concept which is not the same as the "default"
policy. As you will see when we begin to configure policies, each policy has
both a name (human friendly, configurable) as well as an index (or simply
policy number). Swift reserves index 0 to map to the object ring that's
present in all installations (e.g., ``/etc/swift/object.ring.gz``). You can
name this policy anything you like, and if no policies are defined it will
report itself as ``Policy-0``, however you cannot change the index as there must
always be a policy with index 0.
a single name and an arbitrary number of aliases (human friendly,
configurable) as well as an index (or simply policy number). Swift reserves
index 0 to map to the object ring that's present in all installations
(e.g., ``/etc/swift/object.ring.gz``). You can name this policy anything you
like, and if no policies are defined it will report itself as ``Policy-0``,
however you cannot change the index as there must always be a policy with
index 0.
Another important concept is the default policy which can be any policy
in the cluster. The default policy is the policy that is automatically
@ -273,6 +274,8 @@ file:
* Policy names must contain only letters, digits or a dash
* Policy names must be unique
* The policy name 'Policy-0' can only be used for the policy with index 0
* Multiple names can be assigned to one policy using aliases. All names
must follow the Swift naming rules.
* If any policies are defined, exactly one policy must be declared default
* Deprecated policies cannot be declared the default
* If no ``policy_type`` is provided, ``replication`` is the default value.
@ -288,6 +291,7 @@ example configuration.::
[storage-policy:0]
name = gold
aliases = yellow, orange
policy_type = replication
default = yes
@ -301,8 +305,10 @@ information about the ``default`` and ``deprecated`` options.
There are some other considerations when managing policies:
* Policy names can be changed (but be sure that users are aware, aliases are
not currently supported but could be implemented in custom middleware!)
* Policy names can be changed.
* Aliases are supported and can be added and removed. If the primary name
of a policy is removed the next available alias will be adopted as the
primary name. A policy must always have at least one name.
* You cannot change the index of a policy once it has been created
* The default policy can be changed at any time, by adding the
default directive to the desired policy section
@ -399,7 +405,7 @@ The module, :ref:`storage_policy`, is responsible for parsing the
configured policies via class :class:`.StoragePolicyCollection`. This
collection is made up of policies of class :class:`.StoragePolicy`. The
collection class includes handy functions for getting to a policy either by
name or by index , getting info about the policies, etc. There's also one
name or by index , getting info about the policies, etc. There's also one
very important function, :meth:`~.StoragePolicyCollection.get_object_ring`.
Object rings are members of the :class:`.StoragePolicy` class and are
actually not instantiated until the :meth:`~.StoragePolicy.load_ring`

4
doc/source/policies_saio.rst

@ -26,6 +26,7 @@ to implement a usable set of policies.
[storage-policy:0]
name = gold
aliases = yellow, orange
default = yes
[storage-policy:1]
@ -82,7 +83,8 @@ Storage Policies effect placement of data in Swift.
You should see this: (only showing the policy output here)::
policies: [{'default': True, 'name': 'gold'}, {'name': 'silver'}]
policies: [{'aliases': 'gold, yellow, orange', 'default': True,
'name': 'gold'}, {'aliases': 'silver', 'name': 'silver'}]
3. Now create a container without specifying a policy, it will use the
default, 'gold' and then put a test object in it (create the file ``file0.txt``

1
etc/memcache.conf-sample

@ -2,6 +2,7 @@
# You can use this single conf file instead of having memcache_servers set in
# several other conf files under [filter:cache] for example. You can specify
# multiple servers separated with commas, as in: 10.1.2.3:11211,10.1.2.4:11211
# (IPv6 addresses must follow rfc3986 section-3.2.2, i.e. [::1]:11211)
# memcache_servers = 127.0.0.1:11211
#
# Sets how memcache values are serialized and deserialized:

8
etc/object-server.conf-sample

@ -172,10 +172,7 @@ use = egg:swift#recon
# concurrency = 1
# stats_interval = 300
#
# The sync method to use; default is rsync but you can use ssync to try the
# EXPERIMENTAL all-swift-code-no-rsync-callouts method. Once ssync is verified
# as having performance comparable to, or better than, rsync, we plan to
# deprecate rsync so we can move on with more features for replication.
# default is rsync, alternative is ssync
# sync_method = rsync
#
# max duration of a partition rsync
@ -285,6 +282,9 @@ use = egg:swift#recon
# log_level = INFO
# log_address = /dev/log
#
# Time in seconds to wait between auditor passes
# interval = 30
#
# You can set the disk chunk size that the auditor uses making it larger if
# you like for more efficient local auditing of larger objects
# disk_chunk_size = 65536

12
etc/proxy-server.conf-sample

@ -388,7 +388,8 @@ use = egg:swift#memcache
# If not set here, the value for memcache_servers will be read from
# memcache.conf (see memcache.conf-sample) or lacking that file, it will
# default to the value below. You can specify multiple servers separated with
# commas, as in: 10.1.2.3:11211,10.1.2.4:11211
# commas, as in: 10.1.2.3:11211,10.1.2.4:11211 (IPv6 addresses must
# follow rfc3986 section-3.2.2, i.e. [::1]:11211)
# memcache_servers = 127.0.0.1:11211
#
# Sets how memcache values are serialized and deserialized:
@ -628,14 +629,17 @@ use = egg:swift#bulk
use = egg:swift#slo
# max_manifest_segments = 1000
# max_manifest_size = 2097152
# min_segment_size = 1048576
# Start rate-limiting SLO segment serving after the Nth segment of a
#
# Rate limiting applies only to segments smaller than this size (bytes).
# rate_limit_under_size = 1048576
#
# Start rate-limiting SLO segment serving after the Nth small segment of a
# segmented object.
# rate_limit_after_segment = 10
#
# Once segment rate-limiting kicks in for an object, limit segments served
# to N per second. 0 means no rate-limiting.
# rate_limit_segments_per_sec = 0
# rate_limit_segments_per_sec = 1
#
# Time limit on GET requests (seconds)
# max_get_time = 86400

15
etc/swift.conf-sample

@ -21,7 +21,7 @@ swift_hash_path_prefix = changeme
# policy with index 0 will be declared the default. If multiple policies are
# defined you must define a policy with index 0 and you must specify a
# default. It is recommended you always define a section for
# storage-policy:0.
# storage-policy:0. Aliases are not required when defining a storage policy.
#
# A 'policy_type' argument is also supported but is not mandatory. Default
# policy type 'replication' is used when 'policy_type' is unspecified.
@ -29,6 +29,7 @@ swift_hash_path_prefix = changeme
name = Policy-0
default = yes
#policy_type = replication
aliases = yellow, orange
# the following section would declare a policy called 'silver', the number of
# replicas will be determined by how the ring is built. In this example the
@ -40,7 +41,10 @@ default = yes
# this config has specified it as the default. However if a legacy container
# (one created with a pre-policy version of swift) is accessed, it is known
# implicitly to be assigned to the policy with index 0 as opposed to the
# current default.
# current default. Note that even without specifying any aliases, a policy
# always has at least the default name stored in aliases because this field is
# used to contain all human readable names for a storage policy.
#
#[storage-policy:1]
#name = silver
#policy_type = replication
@ -67,12 +71,13 @@ default = yes
# refer to Swift documentation for details on how to configure EC policies.
#
# The example 'deepfreeze10-4' policy defined below is a _sample_
# configuration with 10 'data' and 4 'parity' fragments. 'ec_type'
# defines the Erasure Coding scheme. 'jerasure_rs_vand' (Reed-Solomon
# Vandermonde) is used as an example below.
# configuration with an alias of 'df10-4' as well as 10 'data' and 4 'parity'
# fragments. 'ec_type' defines the Erasure Coding scheme.
# 'jerasure_rs_vand' (Reed-Solomon Vandermonde) is used as an example below.
#
#[storage-policy:2]
#name = deepfreeze10-4
#aliases = df10-4
#policy_type = erasure_coding
#ec_type = jerasure_rs_vand
#ec_num_data_fragments = 10

2
requirements.txt

@ -4,7 +4,7 @@
dnspython>=1.12.0;python_version<'3.0'
dnspython3>=1.12.0;python_version>='3.0'
eventlet>=0.16.1,!=0.17.0
eventlet>=0.17.4 # MIT
greenlet>=0.3.1
netifaces>=0.5,!=0.10.0,!=0.10.1
pastedeploy>=1.3.3

4
swift/account/reaper.py

@ -311,8 +311,8 @@ class AccountReaper(Daemon):
delete_timestamp = Timestamp(info['delete_timestamp'])
if self.stats_containers_remaining and \
begin - float(delete_timestamp) >= self.reap_not_done_after:
self.logger.warn(_('Account %s has not been reaped since %s') %
(account, delete_timestamp.isoformat))
self.logger.warning(_('Account %s has not been reaped since %s') %
(account, delete_timestamp.isoformat))
return True
def reap_container(self, account, account_partition, account_nodes,

20
swift/cli/recon.py

@ -181,12 +181,12 @@ class SwiftRecon(object):
def _ptime(self, timev=None):
"""
:param timev: a unix timestamp or None
:returns: a pretty string of the current time or provided time
:returns: a pretty string of the current time or provided time in UTC
"""
if timev:
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timev))
return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(timev))
else:
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
def _md5_file(self, path):
"""
@ -495,16 +495,14 @@ class SwiftRecon(object):
elapsed = time.time() - least_recent_time
elapsed, elapsed_unit = seconds2timeunit(elapsed)
print('Oldest completion was %s (%d %s ago) by %s.' % (
time.strftime('%Y-%m-%d %H:%M:%S',
time.gmtime(least_recent_time)),
self._ptime(least_recent_time),
elapsed, elapsed_unit, host))
if most_recent_url is not None:
host = urlparse(most_recent_url).netloc
elapsed = time.time() - most_recent_time
elapsed, elapsed_unit = seconds2timeunit(elapsed)
print('Most recent completion was %s (%d %s ago) by %s.' % (
time.strftime('%Y-%m-%d %H:%M:%S',
time.gmtime(most_recent_time)),
self._ptime(most_recent_time),
elapsed, elapsed_unit, host))
print("=" * 79)
@ -899,12 +897,8 @@ class SwiftRecon(object):
continue
if (ts_remote < ts_start or ts_remote > ts_end):
diff = abs(ts_end - ts_remote)
ts_end_f = time.strftime(
"%Y-%m-%d %H:%M:%S",
time.localtime(ts_end))
ts_remote_f = time.strftime(
"%Y-%m-%d %H:%M:%S",
time.localtime(ts_remote))
ts_end_f = self._ptime(ts_end)
ts_remote_f = self._ptime(ts_remote)
print("!! %s current time is %s, but remote is %s, "
"differs by %.2f sec" % (

29
swift/cli/ringbuilder.py

@ -25,6 +25,7 @@ from os.path import basename, abspath, dirname, exists, join as pathjoin
from sys import argv as sys_argv, exit, stderr, stdout
from textwrap import wrap
from time import time
from datetime import timedelta
import optparse
import math
@ -444,7 +445,9 @@ swift-ring-builder <builder_file>
builder.parts, builder.replicas, regions, zones, dev_count,
balance, dispersion_trailer))
print('The minimum number of hours before a partition can be '
'reassigned is %s' % builder.min_part_hours)
'reassigned is %s (%s remaining)' % (
builder.min_part_hours,
timedelta(seconds=builder.min_part_seconds_left)))
print('The overload factor is %0.2f%% (%.6f)' % (
builder.overload * 100, builder.overload))
if builder.devs:
@ -787,6 +790,14 @@ swift-ring-builder <builder_file> rebalance [options]
handler.setFormatter(formatter)
logger.addHandler(handler)
if builder.min_part_seconds_left > 0 and not options.force:
print('No partitions could be reassigned.')
print('The time between rebalances must be at least '
'min_part_hours: %s hours (%s remaining)' % (
builder.min_part_hours,
timedelta(seconds=builder.min_part_seconds_left)))
exit(EXIT_WARNING)
devs_changed = builder.devs_changed
try:
last_balance = builder.get_balance()
@ -802,8 +813,7 @@ swift-ring-builder <builder_file> rebalance [options]
exit(EXIT_ERROR)
if not (parts or options.force or removed_devs):
print('No partitions could be reassigned.')
print('Either none need to be or none can be due to '
'min_part_hours [%s].' % builder.min_part_hours)
print('There is no need to do so at this time')
exit(EXIT_WARNING)
# If we set device's weight to zero, currently balance will be set
# special value(MAX_BALANCE) until zero weighted device return all
@ -1029,6 +1039,19 @@ swift-ring-builder <ring_file> write_builder [min_part_hours]
builder.save(builder_file)
def pretend_min_part_hours_passed():
"""
swift-ring-builder <builder_file> pretend_min_part_hours_passed
Resets the clock on the last time a rebalance happened, thus
circumventing the min_part_hours check.
*****************************
USE THIS WITH EXTREME CAUTION
*****************************
If you run this command and deploy rebalanced rings before a replication
pass completes, you may introduce unavailability in your cluster. This
has an end-user impact.
"""
builder.pretend_min_part_hours_passed()
builder.save(builder_file)
exit(EXIT_SUCCESS)

13
swift/common/db_replicator.py

@ -174,11 +174,12 @@ class Replicator(Daemon):
if not self.rsync_module:
self.rsync_module = '{replication_ip}::%s' % self.server_type
if config_true_value(conf.get('vm_test_mode', 'no')):
self.logger.warn('Option %(type)s-replicator/vm_test_mode is '
'deprecated and will be removed in a future '
'version. Update your configuration to use '
'option %(type)s-replicator/rsync_module.'
% {'type': self.server_type})
self.logger.warning('Option %(type)s-replicator/vm_test_mode '
'is deprecated and will be removed in a '
'future version. Update your configuration'
' to use option %(type)s-replicator/'
'rsync_module.'
% {'type': self.server_type})
self.rsync_module += '{replication_port}'
self.reclaim_age = float(conf.get('reclaim_age', 86400 * 7))
swift.common.db.DB_PREALLOCATION = \
@ -632,7 +633,7 @@ class Replicator(Daemon):
[(failure_dev['replication_ip'],
failure_dev['device'])
for failure_dev in self.ring.devs if failure_dev])
self.logger.warn(
self.logger.warning(
_('Skipping %(device)s as it is not mounted') % node)
continue
unlink_older_than(

6
swift/common/direct_client.py

@ -50,7 +50,7 @@ class DirectClientException(ClientException):
def _get_direct_account_container(path, stype, node, part,
account, marker=None, limit=None,
marker=None, limit=None,
prefix=None, delimiter=None, conn_timeout=5,
response_timeout=15):
"""Base class for get direct account and container.
@ -113,7 +113,7 @@ def direct_get_account(node, part, account, marker=None, limit=None,
"""
path = '/' + account
return _get_direct_account_container(path, "Account", node, part,
account, marker=marker,
marker=marker,
limit=limit, prefix=prefix,
delimiter=delimiter,
conn_timeout=conn_timeout,
@ -189,7 +189,7 @@ def direct_get_container(node, part, account, container, marker=None,
"""
path = '/%s/%s' % (account, container)
return _get_direct_account_container(path, "Container", node,
part, account, marker=marker,
part, marker=marker,
limit=limit, prefix=prefix,
delimiter=delimiter,
conn_timeout=conn_timeout,

12
swift/common/internal_client.py

@ -26,9 +26,10 @@ from swift import gettext_ as _
from time import gmtime, strftime, time
from zlib import compressobj
from swift.common.utils import quote
from swift.common.exceptions import ClientException
from swift.common.http import HTTP_NOT_FOUND, HTTP_MULTIPLE_CHOICES
from swift.common.swob import Request
from swift.common.utils import quote
from swift.common.wsgi import loadapp, pipeline_property
@ -807,9 +808,14 @@ class SimpleClient(object):
self.attempts += 1
try:
return self.base_request(method, **kwargs)
except (socket.error, httplib.HTTPException, urllib2.URLError):
except (socket.error, httplib.HTTPException, urllib2.URLError) \
as err:
if self.attempts > retries:
raise
if isinstance(err, urllib2.HTTPError):
raise ClientException('Raise too many retries',
http_status=err.getcode())
else:
raise
sleep(backoff)
backoff = min(backoff * 2, self.max_backoff)

30
swift/common/manager.py

@ -162,6 +162,16 @@ def safe_kill(pid, sig, name):
os.kill(pid, sig)
def kill_group(pid, sig):
"""Send signal to process group
: param pid: process id
: param sig: signal to send
"""
# Negative PID means process group
os.kill(-pid, sig)
class UnknownCommandError(Exception):
pass
@ -285,11 +295,27 @@ class Manager(object):
return 0
# reached interval n watch_pids w/o killing all servers
kill_after_timeout = kwargs.get('kill_after_timeout', False)
for server, pids in server_pids.items():
if not killed_pids.issuperset(pids):
# some pids of this server were not killed
print(_('Waited %s seconds for %s to die; giving up') % (
kill_wait, server))
if kill_after_timeout:
print(_('Waited %s seconds for %s to die; killing') % (
kill_wait, server))
# Send SIGKILL to all remaining pids
for pid in set(pids.keys()) - killed_pids:
print(_('Signal %s pid: %s signal: %s') % (
server, pid, signal.SIGKILL))
# Send SIGKILL to process group
try:
kill_group(pid, signal.SIGKILL)
except OSError as e:
# PID died before kill_group can take action?
if e.errno != errno.ESRCH:
raise e
else:
print(_('Waited %s seconds for %s to die; giving up') % (
kill_wait, server))
return 1
@command

108
swift/common/memcached.py

@ -47,6 +47,7 @@ http://github.com/memcached/memcached/blob/1.4.2/doc/protocol.txt
import six.moves.cPickle as pickle
import json
import logging
import <