Merge branch 'master' into feature/hummingbird
Change-Id: Ib4a2d12a47f023235c1dc1d67d3c458cc967a7b4
This commit is contained in:
commit
eb8f1f83f1
3
.mailmap
3
.mailmap
|
@ -78,3 +78,6 @@ Jaivish Kothari <jaivish.kothari@nectechnologies.in> <janonymous.codevulture@gma
|
|||
Michael Matur <michael.matur@gmail.com>
|
||||
Kazuhiro Miyahara <miyahara.kazuhiro@lab.ntt.co.jp>
|
||||
Alexandra Settle <alexandra.settle@rackspace.com>
|
||||
Kenichiro Matsuda <matsuda_kenichi@jp.fujitsu.com>
|
||||
Atsushi Sakai <sakaia@jp.fujitsu.com>
|
||||
Takashi Natsume <natsume.takashi@lab.ntt.co.jp>
|
||||
|
|
19
AUTHORS
19
AUTHORS
|
@ -26,6 +26,7 @@ Chuck Thier (cthier@gmail.com)
|
|||
Contributors
|
||||
------------
|
||||
Mehdi Abaakouk (mehdi.abaakouk@enovance.com)
|
||||
Timur Alperovich (timur.alperovich@gmail.com)
|
||||
Jesse Andrews (anotherjesse@gmail.com)
|
||||
Joe Arnold (joe@swiftstack.com)
|
||||
Ionuț Arțăriși (iartarisi@suse.cz)
|
||||
|
@ -47,6 +48,7 @@ Tim Burke (tim.burke@gmail.com)
|
|||
Brian D. Burns (iosctr@gmail.com)
|
||||
Devin Carlen (devin.carlen@gmail.com)
|
||||
Thierry Carrez (thierry@openstack.org)
|
||||
Carlos Cavanna (ccavanna@ca.ibm.com)
|
||||
Emmanuel Cazenave (contact@emcaz.fr)
|
||||
Mahati Chamarthy (mahati.chamarthy@gmail.com)
|
||||
Zap Chang (zapchang@gmail.com)
|
||||
|
@ -55,6 +57,7 @@ Ray Chen (oldsharp@163.com)
|
|||
Harshit Chitalia (harshit@acelio.com)
|
||||
Brian Cline (bcline@softlayer.com)
|
||||
Alistair Coles (alistair.coles@hp.com)
|
||||
Clément Contini (ccontini@cloudops.com)
|
||||
Brian Curtin (brian.curtin@rackspace.com)
|
||||
Thiago da Silva (thiago@redhat.com)
|
||||
Julien Danjou (julien@danjou.info)
|
||||
|
@ -64,6 +67,7 @@ Cedric Dos Santos (cedric.dos.sant@gmail.com)
|
|||
Gerry Drudy (gerry.drudy@hp.com)
|
||||
Morgan Fainberg (morgan.fainberg@gmail.com)
|
||||
ZhiQiang Fan (aji.zqfan@gmail.com)
|
||||
Oshrit Feder (oshritf@il.ibm.com)
|
||||
Mike Fedosin (mfedosin@mirantis.com)
|
||||
Ricardo Ferreira (ricardo.sff@gmail.com)
|
||||
Flaper Fesp (flaper87@gmail.com)
|
||||
|
@ -91,8 +95,10 @@ Dan Hersam (dan.hersam@hp.com)
|
|||
Derek Higgins (derekh@redhat.com)
|
||||
Alex Holden (alex@alexjonasholden.com)
|
||||
Edward Hope-Morley (opentastic@gmail.com)
|
||||
Charles Hsu (charles0126@gmail.com)
|
||||
Joanna H. Huang (joanna.huitzu.huang@gmail.com)
|
||||
Kun Huang (gareth@unitedstack.com)
|
||||
Bill Huber (wbhuber@us.ibm.com)
|
||||
Matthieu Huin (mhu@enovance.com)
|
||||
Hodong Hwang (hodong.hwang@kt.com)
|
||||
Motonobu Ichimura (motonobu@gmail.com)
|
||||
|
@ -126,6 +132,7 @@ John Leach (john@johnleach.co.uk)
|
|||
Ed Leafe (ed.leafe@rackspace.com)
|
||||
Thomas Leaman (thomas.leaman@hp.com)
|
||||
Eohyung Lee (liquidnuker@gmail.com)
|
||||
Zhao Lei (zhaolei@cn.fujitsu.com)
|
||||
Jamie Lennox (jlennox@redhat.com)
|
||||
Tong Li (litong01@us.ibm.com)
|
||||
Changbin Liu (changbin.liu@gmail.com)
|
||||
|
@ -136,10 +143,12 @@ Zhongyue Luo (zhongyue.nah@intel.com)
|
|||
Paul Luse (paul.e.luse@intel.com)
|
||||
Christopher MacGown (chris@pistoncloud.com)
|
||||
Dragos Manolescu (dragosm@hp.com)
|
||||
Ben Martin (blmartin@us.ibm.com)
|
||||
Steve Martinelli (stevemar@ca.ibm.com)
|
||||
Juan J. Martinez (juan@memset.com)
|
||||
Marcelo Martins (btorch@gmail.com)
|
||||
Dolph Mathews (dolph.mathews@gmail.com)
|
||||
Kenichiro Matsuda (matsuda_kenichi@jp.fujitsu.com)
|
||||
Michael Matur (michael.matur@gmail.com)
|
||||
Donagh McCabe (donagh.mccabe@hp.com)
|
||||
Andy McCrae (andy.mccrae@gmail.com)
|
||||
|
@ -151,11 +160,13 @@ Jola Mirecka (jola.mirecka@hp.com)
|
|||
Kazuhiro Miyahara (miyahara.kazuhiro@lab.ntt.co.jp)
|
||||
Daisuke Morita (morita.daisuke@lab.ntt.co.jp)
|
||||
Dirk Mueller (dirk@dmllr.de)
|
||||
Takashi Natsume (natsume.takashi@lab.ntt.co.jp)
|
||||
Russ Nelson (russ@crynwr.com)
|
||||
Maru Newby (mnewby@internap.com)
|
||||
Newptone (xingchao@unitedstack.com)
|
||||
Colin Nicholson (colin.nicholson@iomart.com)
|
||||
Zhenguo Niu (zhenguo@unitedstack.com)
|
||||
Ondrej Novy (ondrej.novy@firma.seznam.cz)
|
||||
Timothy Okwii (tokwii@cisco.com)
|
||||
Matthew Oliver (matt@oliver.net.au)
|
||||
Hisashi Osanai (osanai.hisashi@jp.fujitsu.com)
|
||||
|
@ -169,18 +180,24 @@ Constantine Peresypkin (constantine.peresypk@rackspace.com)
|
|||
Dieter Plaetinck (dieter@vimeo.com)
|
||||
Dan Prince (dprince@redhat.com)
|
||||
Sarvesh Ranjan (saranjan@cisco.com)
|
||||
Falk Reimann (falk.reimann@sap.com)
|
||||
Brian Reitz (brian.reitz@oracle.com)
|
||||
Felipe Reyes (freyes@tty.cl)
|
||||
Janie Richling (jrichli@us.ibm.com)
|
||||
Matt Riedemann (mriedem@us.ibm.com)
|
||||
Li Riqiang (lrqrun@gmail.com)
|
||||
Rafael Rivero (rafael@cloudscaling.com)
|
||||
Victor Rodionov (victor.rodionov@nexenta.com)
|
||||
Eran Rom (eranr@il.ibm.com)
|
||||
Aaron Rosen (arosen@nicira.com)
|
||||
Brent Roskos (broskos@internap.com)
|
||||
Hamdi Roumani (roumani@ca.ibm.com)
|
||||
Shilla Saebi (shilla.saebi@gmail.com)
|
||||
Atsushi Sakai (sakaia@jp.fujitsu.com)
|
||||
Cristian A Sanchez (cristian.a.sanchez@intel.com)
|
||||
Christian Schwede (cschwede@redhat.com)
|
||||
Mark Seger (Mark.Seger@hp.com)
|
||||
Azhagu Selvan SP (tamizhgeek@gmail.com)
|
||||
Alexandra Settle (alexandra.settle@rackspace.com)
|
||||
Andrew Clay Shafer (acs@parvuscaptus.com)
|
||||
Mitsuhiro SHIGEMATSU (shigematsu.mitsuhiro@lab.ntt.co.jp)
|
||||
|
@ -198,6 +215,7 @@ Jeremy Stanley (fungi@yuggoth.org)
|
|||
Mauro Stettler (mauro.stettler@gmail.com)
|
||||
Tobias Stevenson (tstevenson@vbridges.com)
|
||||
Victor Stinner (vstinner@redhat.com)
|
||||
Akihito Takai (takaiak@nttdata.co.jp)
|
||||
Pearl Yajing Tan (pearl.y.tan@seagate.com)
|
||||
Yuriy Taraday (yorik.sar@gmail.com)
|
||||
Monty Taylor (mordred@inaugust.com)
|
||||
|
@ -231,5 +249,6 @@ Guang Yee (guang.yee@hp.com)
|
|||
Pete Zaitcev (zaitcev@kotori.zaitcev.us)
|
||||
Hua Zhang (zhuadl@cn.ibm.com)
|
||||
Jian Zhang (jian.zhang@intel.com)
|
||||
Kai Zhang (zakir.exe@gmail.com)
|
||||
Ning Zhang (ning@zmanda.com)
|
||||
Yuan Zhou (yuan.zhou@intel.com)
|
||||
|
|
165
CHANGELOG
165
CHANGELOG
|
@ -1,4 +1,133 @@
|
|||
swift (2.3.0)
|
||||
swift (2.4.0)
|
||||
|
||||
* Dependency changes
|
||||
|
||||
- Added six requirement. This is part of an ongoing effort to add
|
||||
support for Python 3.
|
||||
|
||||
- Dropped support for Python 2.6.
|
||||
|
||||
* Config changes
|
||||
|
||||
- Recent versions of Python restrict the number of headers allowed in a
|
||||
request to 100. This number may be too low for custom middleware. The
|
||||
new "extra_header_count" config value in swift.conf can be used to
|
||||
increase the number of headers allowed.
|
||||
|
||||
- Renamed "run_pause" setting to "interval" (current configs with
|
||||
run_pause still work). Future versions of Swift may remove the
|
||||
run_pause setting.
|
||||
|
||||
* Versioned writes middleware
|
||||
|
||||
The versioned writes feature has been refactored and reimplemented as
|
||||
middleware. You should explicitly add the versioned_writes middleware to
|
||||
your proxy pipeline, but do not remove or disable the existing container
|
||||
server config setting ("allow_versions"), if it is currently enabled.
|
||||
The existing container server config setting enables existing
|
||||
containers to continue being versioned. Please see
|
||||
http://swift.openstack.org/middleware.html#how-to-enable-object-versioning-in-a-swift-cluster
|
||||
for further upgrade notes.
|
||||
|
||||
* Allow 1+ object-servers-per-disk deployment
|
||||
|
||||
Enabled by a new > 0 integer config value, "servers_per_port" in the
|
||||
[DEFAULT] config section for object-server and/or replication server
|
||||
configs. The setting's integer value determines how many different
|
||||
object-server workers handle requests for any single unique local port
|
||||
in the ring. In this mode, the parent swift-object-server process
|
||||
continues to run as the original user (i.e. root if low-port binding
|
||||
is required), binds to all ports as defined in the ring, and forks off
|
||||
the specified number of workers per listen socket. The child, per-port
|
||||
servers drop privileges and behave pretty much how object-server workers
|
||||
always have, except that because the ring has unique ports per disk, the
|
||||
object-servers will only be handling requests for a single disk. The
|
||||
parent process detects dead servers and restarts them (with the correct
|
||||
listen socket), starts missing servers when an updated ring file is
|
||||
found with a device on the server with a new port, and kills extraneous
|
||||
servers when their port is found to no longer be in the ring. The ring
|
||||
files are stat'ed at most every "ring_check_interval" seconds, as
|
||||
configured in the object-server config (same default of 15s).
|
||||
|
||||
In testing, this deployment configuration (with a value of 3) lowers
|
||||
request latency, improves requests per second, and isolates slow disk
|
||||
IO as compared to the existing "workers" setting. To use this, each
|
||||
device must be added to the ring using a different port.
|
||||
|
||||
* Do container listing updates in another (green)thread
|
||||
|
||||
The object server has learned the "container_update_timeout" setting
|
||||
(with a default of 1 second). This value is the number of seconds that
|
||||
the object server will wait for the container server to update the
|
||||
listing before returning the status of the object PUT operation.
|
||||
|
||||
Previously, the object server would wait up to 3 seconds for the
|
||||
container server response. The new behavior dramatically lowers object
|
||||
PUT latency when container servers in the cluster are busy (e.g. when
|
||||
the container is very large). Setting the value too low may result in a
|
||||
client PUT'ing an object and not being able to immediately find it in
|
||||
listings. Setting it too high will increase latency for clients when
|
||||
container servers are busy.
|
||||
|
||||
* TempURL fixes (closes CVE-2015-5223)
|
||||
|
||||
Do not allow PUT tempurls to create pointers to other data.
|
||||
Specifically, disallow the creation of DLO object manifests via a PUT
|
||||
tempurl. This prevents discoverability attacks which can use any PUT
|
||||
tempurl to probe for private data by creating a DLO object manifest and
|
||||
then using the PUT tempurl to head the object.
|
||||
|
||||
* Ring changes
|
||||
|
||||
- Partition placement no longer uses the port number to place
|
||||
partitions. This improves dispersion in small clusters running one
|
||||
object server per drive, and it does not affect dispersion in
|
||||
clusters running one object server per server.
|
||||
|
||||
- Added ring-builder-analyzer tool to more easily test and analyze a
|
||||
series of ring management operations.
|
||||
|
||||
- Stop moving partitions unnecessarily when overload is on.
|
||||
|
||||
* Significant improvements and bug fixes have been made to erasure code
|
||||
support. This feature is suitable for beta testing, but it is not yet
|
||||
ready for broad production usage.
|
||||
|
||||
* Bulk upload now treats user xattrs on files in the given archive as
|
||||
object metadata on the resulting created objects.
|
||||
|
||||
* Emit warning log in object replicator if "handoffs_first" or
|
||||
"handoff_delete" is set.
|
||||
|
||||
* Enable object replicator's failure count in swift-recon.
|
||||
|
||||
* Added storage policy support to dispersion tools.
|
||||
|
||||
* Support keystone v3 domains in swift-dispersion.
|
||||
|
||||
* Added domain_remap information to the /info endpoint.
|
||||
|
||||
* Added support for a "default_reseller_prefix" in domain_remap
|
||||
middleware config.
|
||||
|
||||
* Allow SLO PUTs to forgo per-segment integrity checks. Previously, each
|
||||
segment referenced in the manifest also needed the correct etag and
|
||||
bytes setting. These fields now allow the "null" value to skip those
|
||||
particular checks on the given segment.
|
||||
|
||||
* Allow rsync to use compression via a "rsync_compress" config. If set to
|
||||
true, compression is only enabled for an rsync to a device in a
|
||||
different region. In some cases, this can speed up cross-region
|
||||
replication data transfer.
|
||||
|
||||
* Added time synchronization check in swift-recon (the --time option).
|
||||
|
||||
* The account reaper now runs faster on large accounts.
|
||||
|
||||
* Various other minor bug fixes and improvements.
|
||||
|
||||
|
||||
swift (2.3.0, OpenStack Kilo)
|
||||
|
||||
* Erasure Code support (beta)
|
||||
|
||||
|
@ -58,6 +187,7 @@ swift (2.3.0)
|
|||
|
||||
* Various other minor bug fixes and improvements.
|
||||
|
||||
|
||||
swift (2.2.2)
|
||||
|
||||
* Data placement changes
|
||||
|
@ -117,6 +247,7 @@ swift (2.2.2)
|
|||
|
||||
* Various other minor bug fixes and improvements.
|
||||
|
||||
|
||||
swift (2.2.1)
|
||||
|
||||
* Swift now rejects object names with Unicode surrogates.
|
||||
|
@ -164,7 +295,7 @@ swift (2.2.1)
|
|||
* Various other minor bug fixes and improvements.
|
||||
|
||||
|
||||
swift (2.2.0)
|
||||
swift (2.2.0, OpenStack Juno)
|
||||
|
||||
* Added support for Keystone v3 auth.
|
||||
|
||||
|
@ -338,7 +469,7 @@ swift (2.0.0)
|
|||
|
||||
* Various other minor bug fixes and improvements
|
||||
|
||||
swift (1.13.1)
|
||||
swift (1.13.1, OpenStack Icehouse)
|
||||
|
||||
* Change the behavior of CORS responses to better match the spec
|
||||
|
||||
|
@ -605,7 +736,7 @@ swift (1.11.0)
|
|||
|
||||
* Various other bug fixes and improvements
|
||||
|
||||
swift (1.10.0)
|
||||
swift (1.10.0, OpenStack Havana)
|
||||
|
||||
* Added support for pooling memcache connections
|
||||
|
||||
|
@ -776,7 +907,7 @@ swift (1.9.0)
|
|||
|
||||
* Various other minor bug fixes and improvements
|
||||
|
||||
swift (1.8.0)
|
||||
swift (1.8.0, OpenStack Grizzly)
|
||||
|
||||
* Make rings' replica count adjustable
|
||||
|
||||
|
@ -947,7 +1078,7 @@ swift (1.7.5)
|
|||
|
||||
* Various other minor bug fixes and improvements
|
||||
|
||||
swift (1.7.4)
|
||||
swift (1.7.4, OpenStack Folsom)
|
||||
|
||||
* Fix issue where early client disconnects may have caused a memory leak
|
||||
|
||||
|
@ -962,14 +1093,14 @@ swift (1.7.0)
|
|||
|
||||
Serialize RingData in a versioned, custom format which is a combination
|
||||
of a JSON-encoded header and .tostring() dumps of the
|
||||
replica2part2dev_id arrays. This format deserializes hundreds of times
|
||||
replica2part2dev_id arrays. This format deserializes hundreds of times
|
||||
faster than rings serialized with Python 2.7's pickle (a significant
|
||||
performance regression for ring loading between Python 2.6 and Python
|
||||
2.7). Fixes bug 1031954.
|
||||
2.7). Fixes bug 1031954.
|
||||
|
||||
The new implementation is backward-compatible; if a ring
|
||||
does not begin with a new-style magic string, it is assumed to be an
|
||||
old-style pickle-dumped ring and is handled as before. So new Swift
|
||||
old-style pickle-dumped ring and is handled as before. So new Swift
|
||||
code can read old rings, but old Swift code will not be able to read
|
||||
newly-serialized rings.
|
||||
|
||||
|
@ -1153,7 +1284,7 @@ swift (1.5.0)
|
|||
|
||||
* Various other minor bug fixes and improvements
|
||||
|
||||
swift (1.4.8)
|
||||
swift (1.4.8, OpenStack Essex)
|
||||
|
||||
* Added optional max_containers_per_account restriction
|
||||
|
||||
|
@ -1296,7 +1427,7 @@ swift (1.4.4)
|
|||
|
||||
* Query only specific zone via swift-recon.
|
||||
|
||||
swift (1.4.3)
|
||||
swift (1.4.3, OpenStack Diablo)
|
||||
|
||||
* Additional quarantine catching code.
|
||||
|
||||
|
@ -1421,3 +1552,15 @@ swift (1.4.0)
|
|||
|
||||
* Stats uploaders now allow overrides for source_filename_pattern and
|
||||
new_log_cutoff values.
|
||||
|
||||
---
|
||||
|
||||
Changelog entries for previous versions are incomplete
|
||||
|
||||
swift (1.3.0, OpenStack Cactus)
|
||||
|
||||
swift (1.2.0, OpenStack Bexar)
|
||||
|
||||
swift (1.1.0, OpenStack Austin)
|
||||
|
||||
swift (1.0.0, Initial Release)
|
||||
|
|
|
@ -0,0 +1,149 @@
|
|||
# optional: after how many files to update progress
|
||||
#show_progress_every: 100
|
||||
|
||||
# optional: plugins directory name
|
||||
#plugins_dir: 'plugins'
|
||||
|
||||
# optional: plugins discovery name pattern
|
||||
plugin_name_pattern: '*.py'
|
||||
|
||||
# optional: terminal escape sequences to display colors
|
||||
#output_colors:
|
||||
# DEFAULT: '\033[0m'
|
||||
# HEADER: '\033[95m'
|
||||
# LOW: '\033[94m'
|
||||
# MEDIUM: '\033[93m'
|
||||
# HIGH: '\033[91m'
|
||||
|
||||
# optional: log format string
|
||||
#log_format: "[%(module)s]\t%(levelname)s\t%(message)s"
|
||||
|
||||
# globs of files which should be analyzed
|
||||
include:
|
||||
- '*.py'
|
||||
|
||||
# a list of strings, which if found in the path will cause files to be
|
||||
# excluded
|
||||
# for example /tests/ - to remove all all files in tests directory
|
||||
#exclude_dirs:
|
||||
# - '/tests/'
|
||||
|
||||
#configured for swift
|
||||
profiles:
|
||||
gate:
|
||||
include:
|
||||
- blacklist_calls
|
||||
- blacklist_imports
|
||||
- exec_used
|
||||
- linux_commands_wildcard_injection
|
||||
- request_with_no_cert_validation
|
||||
- set_bad_file_permissions
|
||||
- subprocess_popen_with_shell_equals_true
|
||||
- ssl_with_bad_version
|
||||
- password_config_option_not_marked_secret
|
||||
|
||||
# - any_other_function_with_shell_equals_true
|
||||
# - ssl_with_bad_defaults
|
||||
# - jinja2_autoescape_false
|
||||
# - use_of_mako_templates
|
||||
# - subprocess_without_shell_equals_true
|
||||
# - any_other_function_with_shell_equals_true
|
||||
# - start_process_with_a_shell
|
||||
# - start_process_with_no_shell
|
||||
# - hardcoded_sql_expressions
|
||||
# - hardcoded_tmp_director
|
||||
# - linux_commands_wildcard_injection
|
||||
#For now some items are commented which could be included as per use later.
|
||||
blacklist_calls:
|
||||
bad_name_sets:
|
||||
# - pickle:
|
||||
# qualnames: [pickle.loads, pickle.load, pickle.Unpickler,
|
||||
# cPickle.loads, cPickle.load, cPickle.Unpickler]
|
||||
# level: LOW
|
||||
# message: "Pickle library appears to be in use, possible security
|
||||
#issue."
|
||||
|
||||
- marshal:
|
||||
qualnames: [marshal.load, marshal.loads]
|
||||
message: "Deserialization with the marshal module is possibly
|
||||
dangerous."
|
||||
# - md5:
|
||||
# qualnames: [hashlib.md5]
|
||||
# level: LOW
|
||||
# message: "Use of insecure MD5 hash function."
|
||||
- mktemp_q:
|
||||
qualnames: [tempfile.mktemp]
|
||||
message: "Use of insecure and deprecated function (mktemp)."
|
||||
# - eval:
|
||||
# qualnames: [eval]
|
||||
# level: LOW
|
||||
# message: "Use of possibly insecure function - consider using safer
|
||||
#ast.literal_eval."
|
||||
- mark_safe:
|
||||
names: [mark_safe]
|
||||
message: "Use of mark_safe() may expose cross-site scripting
|
||||
vulnerabilities and should be reviewed."
|
||||
- httpsconnection:
|
||||
qualnames: [httplib.HTTPSConnection]
|
||||
message: "Use of HTTPSConnection does not provide security, see
|
||||
https://wiki.openstack.org/wiki/OSSN/OSSN-0033"
|
||||
- yaml_load:
|
||||
qualnames: [yaml.load]
|
||||
message: "Use of unsafe yaml load. Allows instantiation of
|
||||
arbitrary objects. Consider yaml.safe_load()."
|
||||
- urllib_urlopen:
|
||||
qualnames: [urllib.urlopen, urllib.urlretrieve, urllib.URLopener,
|
||||
urllib.FancyURLopener, urllib2.urlopen, urllib2.Request]
|
||||
message: "Audit url open for permitted schemes. Allowing use of
|
||||
file:/ or custom schemes is often unexpected."
|
||||
- paramiko_injection:
|
||||
qualnames: [paramiko.exec_command, paramiko.invoke_shell]
|
||||
message: "Paramiko exec_command() and invoke_shell() usage may
|
||||
expose command injection vulnerabilities and should be reviewed."
|
||||
|
||||
shell_injection:
|
||||
# Start a process using the subprocess module, or one of its wrappers.
|
||||
subprocess: [subprocess.Popen, subprocess.call, subprocess.check_call,
|
||||
subprocess.check_output, utils.execute,
|
||||
utils.execute_with_timeout]
|
||||
# Start a process with a function vulnerable to shell injection.
|
||||
shell: [os.system, os.popen, os.popen2, os.popen3, os.popen4,
|
||||
popen2.popen2, popen2.popen3, popen2.popen4, popen2.Popen3,
|
||||
popen2.Popen4, commands.getoutput, commands.getstatusoutput]
|
||||
# Start a process with a function that is not vulnerable to shell
|
||||
# injection.
|
||||
no_shell: [os.execl, os.execle, os.execlp, os.execlpe, os.execv,os.execve,
|
||||
os.execvp, os.execvpe, os.spawnl, os.spawnle, os.spawnlp,
|
||||
os.spawnlpe, os.spawnv, os.spawnve, os.spawnvp, os.spawnvpe,
|
||||
os.startfile]
|
||||
|
||||
blacklist_imports:
|
||||
bad_import_sets:
|
||||
- telnet:
|
||||
imports: [telnetlib]
|
||||
level: HIGH
|
||||
message: "Telnet is considered insecure. Use SSH or some other
|
||||
encrypted protocol."
|
||||
- info_libs:
|
||||
imports: [Crypto]
|
||||
level: LOW
|
||||
message: "Consider possible security implications associated with
|
||||
#{module} module."
|
||||
|
||||
hardcoded_password:
|
||||
word_list: "wordlist/default-passwords"
|
||||
|
||||
ssl_with_bad_version:
|
||||
bad_protocol_versions:
|
||||
- 'PROTOCOL_SSLv2'
|
||||
- 'SSLv2_METHOD'
|
||||
- 'SSLv23_METHOD'
|
||||
- 'PROTOCOL_SSLv3' # strict option
|
||||
- 'PROTOCOL_TLSv1' # strict option
|
||||
- 'SSLv3_METHOD' # strict option
|
||||
- 'TLSv1_METHOD' # strict option
|
||||
|
||||
password_config_option_not_marked_secret:
|
||||
function_names:
|
||||
- oslo.config.cfg.StrOpt
|
||||
- oslo_config.cfg.StrOpt
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/python
|
||||
#!/usr/bin/env python
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
||||
# use this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/python
|
||||
#!/usr/bin/env python
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
||||
# use this file except in compliance with the License. You may obtain a copy
|
||||
# of the License at
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/python
|
||||
#!/usr/bin/env python
|
||||
# Copyright (c) 2010-2012 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
|
@ -16,13 +16,13 @@
|
|||
|
||||
import traceback
|
||||
from ConfigParser import ConfigParser
|
||||
from cStringIO import StringIO
|
||||
from optparse import OptionParser
|
||||
from sys import exit, stdout
|
||||
from time import time
|
||||
from six.moves import range
|
||||
from eventlet import GreenPool, patcher, sleep
|
||||
from eventlet.pools import Pool
|
||||
from six.moves import cStringIO as StringIO
|
||||
|
||||
try:
|
||||
from swiftclient import get_auth
|
||||
|
@ -76,8 +76,9 @@ def report(success):
|
|||
return
|
||||
next_report = time() + 5
|
||||
eta, eta_unit = compute_eta(begun, created, need_to_create)
|
||||
print '\r\x1B[KCreating %s: %d of %d, %d%s left, %d retries' % (item_type,
|
||||
created, need_to_create, round(eta), eta_unit, retries_done),
|
||||
print ('\r\x1B[KCreating %s: %d of %d, %d%s left, %d retries'
|
||||
% (item_type, created, need_to_create, round(eta), eta_unit,
|
||||
retries_done)),
|
||||
stdout.flush()
|
||||
|
||||
|
||||
|
@ -132,6 +133,9 @@ Usage: %%prog [options] [conf_file]
|
|||
retries = int(conf.get('retries', 5))
|
||||
concurrency = int(conf.get('concurrency', 25))
|
||||
endpoint_type = str(conf.get('endpoint_type', 'publicURL'))
|
||||
user_domain_name = str(conf.get('user_domain_name', ''))
|
||||
project_domain_name = str(conf.get('project_domain_name', ''))
|
||||
project_name = str(conf.get('project_name', ''))
|
||||
insecure = options.insecure \
|
||||
or config_true_value(conf.get('keystone_api_insecure', 'no'))
|
||||
container_populate = config_true_value(
|
||||
|
@ -146,6 +150,12 @@ Usage: %%prog [options] [conf_file]
|
|||
retries_done = 0
|
||||
|
||||
os_options = {'endpoint_type': endpoint_type}
|
||||
if user_domain_name:
|
||||
os_options['user_domain_name'] = user_domain_name
|
||||
if project_domain_name:
|
||||
os_options['project_domain_name'] = project_domain_name
|
||||
if project_name:
|
||||
os_options['project_name'] = project_name
|
||||
|
||||
url, token = get_auth(conf['auth_url'], conf['auth_user'],
|
||||
conf['auth_key'],
|
||||
|
|
|
@ -26,6 +26,7 @@ except ImportError:
|
|||
|
||||
from eventlet import GreenPool, hubs, patcher, Timeout
|
||||
from eventlet.pools import Pool
|
||||
from eventlet.green import urllib2
|
||||
|
||||
from swift.common import direct_client
|
||||
try:
|
||||
|
@ -126,7 +127,7 @@ def container_dispersion_report(coropool, connpool, account, container_ring,
|
|||
if not json_output:
|
||||
print '\r\x1B[KQuerying containers: %d of %d, %d%s left, %d ' \
|
||||
'retries' % (containers_queried[0], containers_listed,
|
||||
round(eta), eta_unit, retries_done[0]),
|
||||
round(eta), eta_unit, retries_done[0]),
|
||||
stdout.flush()
|
||||
container_parts = {}
|
||||
for container in containers:
|
||||
|
@ -145,7 +146,7 @@ def container_dispersion_report(coropool, connpool, account, container_ring,
|
|||
if not json_output:
|
||||
print '\r\x1B[KQueried %d containers for dispersion reporting, ' \
|
||||
'%d%s, %d retries' % (containers_listed, round(elapsed),
|
||||
elapsed_unit, retries_done[0])
|
||||
elapsed_unit, retries_done[0])
|
||||
if containers_listed - distinct_partitions:
|
||||
print 'There were %d overlapping partitions' % (
|
||||
containers_listed - distinct_partitions)
|
||||
|
@ -176,9 +177,10 @@ def object_dispersion_report(coropool, connpool, account, object_ring,
|
|||
try:
|
||||
objects = [o['name'] for o in conn.get_container(
|
||||
container, prefix='dispersion_', full_listing=True)[1]]
|
||||
except ClientException as err:
|
||||
if err.http_status != 404:
|
||||
except urllib2.HTTPError as err:
|
||||
if err.getcode() != 404:
|
||||
raise
|
||||
|
||||
print >>stderr, 'No objects to query. Has ' \
|
||||
'swift-dispersion-populate been run?'
|
||||
stderr.flush()
|
||||
|
@ -255,7 +257,7 @@ def object_dispersion_report(coropool, connpool, account, object_ring,
|
|||
if not json_output:
|
||||
print '\r\x1B[KQueried %d objects for dispersion reporting, ' \
|
||||
'%d%s, %d retries' % (objects_listed, round(elapsed),
|
||||
elapsed_unit, retries_done[0])
|
||||
elapsed_unit, retries_done[0])
|
||||
if objects_listed - distinct_partitions:
|
||||
print 'There were %d overlapping partitions' % (
|
||||
objects_listed - distinct_partitions)
|
||||
|
@ -363,6 +365,9 @@ Usage: %%prog [options] [conf_file]
|
|||
and not options.container_only
|
||||
if not (object_report or container_report):
|
||||
exit("Neither container or object report is set to run")
|
||||
user_domain_name = str(conf.get('user_domain_name', ''))
|
||||
project_domain_name = str(conf.get('project_domain_name', ''))
|
||||
project_name = str(conf.get('project_name', ''))
|
||||
insecure = options.insecure \
|
||||
or config_true_value(conf.get('keystone_api_insecure', 'no'))
|
||||
if options.debug:
|
||||
|
@ -371,6 +376,12 @@ Usage: %%prog [options] [conf_file]
|
|||
coropool = GreenPool(size=concurrency)
|
||||
|
||||
os_options = {'endpoint_type': endpoint_type}
|
||||
if user_domain_name:
|
||||
os_options['user_domain_name'] = user_domain_name
|
||||
if project_domain_name:
|
||||
os_options['project_domain_name'] = project_domain_name
|
||||
if project_name:
|
||||
os_options['project_name'] = project_name
|
||||
|
||||
url, token = get_auth(conf['auth_url'], conf['auth_user'],
|
||||
conf['auth_key'],
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/python
|
||||
#!/usr/bin/env python
|
||||
# Copyright (c) 2014 Christian Schwede <christian.schwede@enovance.com>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/python
|
||||
#!/usr/bin/env python
|
||||
# Copyright (c) 2014 Christian Schwede <christian.schwede@enovance.com>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/python
|
||||
#!/usr/bin/env python
|
||||
# Copyright (c) 2015 Samuel Merritt <sam@swiftstack.com>
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
|
|
|
@ -188,12 +188,6 @@ Number of replication workers to spawn. The default is 8.
|
|||
Time in seconds to wait between replication passes. The default is 30.
|
||||
.IP \fBinterval\fR
|
||||
Replaces run_pause with the more standard "interval", which means the replicator won't pause unless it takes less than the interval set. The default is 30.
|
||||
.IP \fBerror_suppression_interval\fR
|
||||
How long without an error before a node's error count is reset. This will also be how long before a node is re-enabled after suppression is triggered.
|
||||
The default is 60 seconds.
|
||||
.IP \fBerror_suppression_limit\fR
|
||||
How many errors can accumulate before a node is temporarily ignored. The default
|
||||
is 10 seconds.
|
||||
.IP \fBnode_timeout\fR
|
||||
Request timeout to external services. The default is 10 seconds.
|
||||
.IP \fBconn_timeout\fR
|
||||
|
|
|
@ -44,6 +44,12 @@ Authentication system URL
|
|||
Authentication system account/user name
|
||||
.IP "\fBauth_key\fR"
|
||||
Authentication system account/user password
|
||||
.IP "\fBproject_name\fR"
|
||||
Project name in case of keystone auth version 3
|
||||
.IP "\fBproject_domain_name\fR"
|
||||
Project domain name in case of keystone auth version 3
|
||||
.IP "\fBuser_domain_name\fR"
|
||||
User domain name in case of keystone auth version 3
|
||||
.IP "\fBswift_dir\fR"
|
||||
Location of openstack-swift configuration and ring files
|
||||
.IP "\fBdispersion_coverage\fR"
|
||||
|
@ -70,6 +76,9 @@ Whether to run the object report. The default is yes.
|
|||
.IP "auth_key = dpstats"
|
||||
.IP "swift_dir = /etc/swift"
|
||||
.IP "# keystone_api_insecure = no"
|
||||
.IP "# project_name = dpstats"
|
||||
.IP "# project_domain_name = default"
|
||||
.IP "# user_domain_name = default"
|
||||
.IP "# dispersion_coverage = 1.0"
|
||||
.IP "# retries = 5"
|
||||
.IP "# concurrency = 25"
|
||||
|
|
|
@ -129,6 +129,8 @@ Logging address. The default is /dev/log.
|
|||
Request timeout to external services. The default is 3 seconds.
|
||||
.IP \fBconn_timeout\fR
|
||||
Connection timeout to external services. The default is 0.5 seconds.
|
||||
.IP \fBcontainer_update_timeout\fR
|
||||
Time to wait while sending a container update on object update. The default is 1 second.
|
||||
.RE
|
||||
.PD
|
||||
|
||||
|
|
|
@ -85,6 +85,9 @@ Example \fI/etc/swift/dispersion.conf\fR:
|
|||
.IP "auth_user = dpstats:dpstats"
|
||||
.IP "auth_key = dpstats"
|
||||
.IP "swift_dir = /etc/swift"
|
||||
.IP "# project_name = dpstats"
|
||||
.IP "# project_domain_name = default"
|
||||
.IP "# user_domain_name = default"
|
||||
.IP "# dispersion_coverage = 1.0"
|
||||
.IP "# retries = 5"
|
||||
.IP "# concurrency = 25"
|
||||
|
|
|
@ -101,6 +101,9 @@ Example \fI/etc/swift/dispersion.conf\fR:
|
|||
.IP "auth_user = dpstats:dpstats"
|
||||
.IP "auth_key = dpstats"
|
||||
.IP "swift_dir = /etc/swift"
|
||||
.IP "# project_name = dpstats"
|
||||
.IP "# project_domain_name = default"
|
||||
.IP "# user_domain_name = default"
|
||||
.IP "# dispersion_coverage = 1.0"
|
||||
.IP "# retries = 5"
|
||||
.IP "# concurrency = 25"
|
||||
|
|
|
@ -25,7 +25,7 @@
|
|||
.SH SYNOPSIS
|
||||
.LP
|
||||
.B swift-recon
|
||||
\ <server_type> [-v] [--suppress] [-a] [-r] [-u] [-d] [-l] [--md5] [--auditor] [--updater] [--expirer] [--sockstat]
|
||||
\ <server_type> [-v] [--suppress] [-a] [-r] [-u] [-d] [-l] [-T] [--md5] [--auditor] [--updater] [--expirer] [--sockstat]
|
||||
|
||||
.SH DESCRIPTION
|
||||
.PP
|
||||
|
@ -80,8 +80,10 @@ Get md5sum of servers ring and compare to local copy
|
|||
Get cluster socket usage stats
|
||||
.IP "\fB--driveaudit\fR"
|
||||
Get drive audit error stats
|
||||
.IP "\fB-T, --time\fR"
|
||||
Check time synchronization
|
||||
.IP "\fB--all\fR"
|
||||
Perform all checks. Equivalent to \-arudlq \-\-md5
|
||||
Perform all checks. Equivalent to \-arudlqT \-\-md5
|
||||
.IP "\fB--region=REGION\fR"
|
||||
Only query servers in specified region
|
||||
.IP "\fB-z ZONE, --zone=ZONE\fR"
|
||||
|
|
|
@ -9,7 +9,6 @@ user = <your-user-name>
|
|||
log_facility = LOG_LOCAL2
|
||||
recon_cache_path = /var/cache/swift
|
||||
eventlet_debug = true
|
||||
allow_versions = true
|
||||
|
||||
[pipeline:main]
|
||||
pipeline = recon container-server
|
||||
|
|
|
@ -9,7 +9,6 @@ user = <your-user-name>
|
|||
log_facility = LOG_LOCAL3
|
||||
recon_cache_path = /var/cache/swift2
|
||||
eventlet_debug = true
|
||||
allow_versions = true
|
||||
|
||||
[pipeline:main]
|
||||
pipeline = recon container-server
|
||||
|
|
|
@ -9,7 +9,6 @@ user = <your-user-name>
|
|||
log_facility = LOG_LOCAL4
|
||||
recon_cache_path = /var/cache/swift3
|
||||
eventlet_debug = true
|
||||
allow_versions = true
|
||||
|
||||
[pipeline:main]
|
||||
pipeline = recon container-server
|
||||
|
|
|
@ -9,7 +9,6 @@ user = <your-user-name>
|
|||
log_facility = LOG_LOCAL5
|
||||
recon_cache_path = /var/cache/swift4
|
||||
eventlet_debug = true
|
||||
allow_versions = true
|
||||
|
||||
[pipeline:main]
|
||||
pipeline = recon container-server
|
||||
|
|
|
@ -0,0 +1,5 @@
|
|||
[saio]
|
||||
key = changeme
|
||||
key2 = changeme
|
||||
cluster_saio_endpoint = http://127.0.0.1:8080/v1/
|
||||
|
|
@ -37,7 +37,7 @@ interval = 300
|
|||
# config value
|
||||
# processes = 0
|
||||
# process is which of the parts a particular process will work on
|
||||
# process can also be specified on the command line and will overide the config
|
||||
# process can also be specified on the command line and will override the config
|
||||
# value
|
||||
# process is "zero based", if you want to use 3 processes, you should run
|
||||
# processes with process set to 0, 1, and 2
|
||||
|
|
|
@ -9,7 +9,7 @@ eventlet_debug = true
|
|||
[pipeline:main]
|
||||
# Yes, proxy-logging appears twice. This is so that
|
||||
# middleware-originated requests get logged too.
|
||||
pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl ratelimit crossdomain tempauth staticweb container-quotas account-quotas slo dlo proxy-logging proxy-server
|
||||
pipeline = catch_errors gatekeeper healthcheck proxy-logging cache bulk tempurl ratelimit crossdomain container_sync tempauth staticweb container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server
|
||||
|
||||
[filter:catch_errors]
|
||||
use = egg:swift#catch_errors
|
||||
|
@ -35,6 +35,10 @@ use = egg:swift#dlo
|
|||
[filter:slo]
|
||||
use = egg:swift#slo
|
||||
|
||||
[filter:container_sync]
|
||||
use = egg:swift#container_sync
|
||||
current = //saio/saio_endpoint
|
||||
|
||||
[filter:tempurl]
|
||||
use = egg:swift#tempurl
|
||||
|
||||
|
@ -60,6 +64,10 @@ use = egg:swift#memcache
|
|||
[filter:gatekeeper]
|
||||
use = egg:swift#gatekeeper
|
||||
|
||||
[filter:versioned_writes]
|
||||
use = egg:swift#versioned_writes
|
||||
allow_versioned_writes = true
|
||||
|
||||
[app:proxy-server]
|
||||
use = egg:swift#proxy
|
||||
allow_account_management = true
|
||||
|
|
|
@ -154,6 +154,10 @@ until it has been resolved. If the drive is going to be replaced immediately,
|
|||
then it is just best to replace the drive, format it, remount it, and let
|
||||
replication fill it up.
|
||||
|
||||
After the drive is unmounted, make sure the mount point is owned by root
|
||||
(root:root 755). This ensures that rsync will not try to replicate into the
|
||||
root drive once the failed drive is unmounted.
|
||||
|
||||
If the drive can't be replaced immediately, then it is best to leave it
|
||||
unmounted, and set the device weight to 0. This will allow all the
|
||||
replicas that were on that drive to be replicated elsewhere until the drive
|
||||
|
@ -270,7 +274,8 @@ configuration file, /etc/swift/dispersion.conf. Example conf file::
|
|||
|
||||
There are also options for the conf file for specifying the dispersion coverage
|
||||
(defaults to 1%), retries, concurrency, etc. though usually the defaults are
|
||||
fine.
|
||||
fine. If you want to use keystone v3 for authentication there are options like
|
||||
auth_version, user_domain_name, project_domain_name and project_name.
|
||||
|
||||
Once the configuration is in place, run `swift-dispersion-populate` to populate
|
||||
the containers and objects throughout the cluster.
|
||||
|
@ -544,18 +549,22 @@ Request URI Description
|
|||
/recon/sockstat returns consumable info from /proc/net/sockstat|6
|
||||
/recon/devices returns list of devices and devices dir i.e. /srv/node
|
||||
/recon/async returns count of async pending
|
||||
/recon/replication returns object replication times (for backward compatibility)
|
||||
/recon/replication returns object replication info (for backward compatibility)
|
||||
/recon/replication/<type> returns replication info for given type (account, container, object)
|
||||
/recon/auditor/<type> returns auditor stats on last reported scan for given type (account, container, object)
|
||||
/recon/updater/<type> returns last updater sweep times for given type (container, object)
|
||||
========================= ========================================================================================
|
||||
|
||||
Note that 'object_replication_last' and 'object_replication_time' in object
|
||||
replication info are considered to be transitional and will be removed in
|
||||
the subsequent releases. Use 'replication_last' and 'replication_time' instead.
|
||||
|
||||
This information can also be queried via the swift-recon command line utility::
|
||||
|
||||
fhines@ubuntu:~$ swift-recon -h
|
||||
Usage:
|
||||
usage: swift-recon <server_type> [-v] [--suppress] [-a] [-r] [-u] [-d]
|
||||
[-l] [--md5] [--auditor] [--updater] [--expirer] [--sockstat]
|
||||
[-l] [-T] [--md5] [--auditor] [--updater] [--expirer] [--sockstat]
|
||||
|
||||
<server_type> account|container|object
|
||||
Defaults to object server.
|
||||
|
@ -578,7 +587,8 @@ This information can also be queried via the swift-recon command line utility::
|
|||
-q, --quarantined Get cluster quarantine stats
|
||||
--md5 Get md5sum of servers ring and compare to local copy
|
||||
--sockstat Get cluster socket usage stats
|
||||
--all Perform all checks. Equal to -arudlq --md5 --sockstat
|
||||
-T, --time Check time synchronization
|
||||
--all Perform all checks. Equal to -arudlqT --md5 --sockstat
|
||||
-z ZONE, --zone=ZONE Only query servers in specified zone
|
||||
-t SECONDS, --timeout=SECONDS
|
||||
Time to wait for a response from a server
|
||||
|
|
|
@ -7,7 +7,7 @@ metadata by using the Object Storage API, which is implemented as a set
|
|||
of Representational State Transfer (REST) web services.
|
||||
|
||||
For an introduction to OpenStack Object Storage, see `Object
|
||||
Storage <http://docs.openstack.org/admin-guide-cloud/content/ch_admin-openstack-object-storage.html>`__
|
||||
Storage <http://docs.openstack.org/admin-guide-cloud/objectstorage.html>`
|
||||
in the *OpenStack Cloud Administrator Guide*.
|
||||
|
||||
You use the HTTPS (SSL) protocol to interact with Object Storage, and
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
CORS
|
||||
====
|
||||
|
||||
CORS_ is a mechanisim to allow code running in a browser (Javascript for
|
||||
CORS_ is a mechanism to allow code running in a browser (Javascript for
|
||||
example) make requests to a domain other then the one from where it originated.
|
||||
|
||||
Swift supports CORS requests to containers and objects.
|
||||
|
|
|
@ -340,7 +340,7 @@ paste.deploy works (at least at the time of this writing.)
|
|||
|
||||
`name3` got the local value from the `app:myapp` subsection because it is using
|
||||
the special paste.deploy syntax of ``set option_name = value``. So, if you want
|
||||
a default value for most app/filters but want to overridde it in one
|
||||
a default value for most app/filters but want to override it in one
|
||||
subsection, this is how you do it.
|
||||
|
||||
`name4` got the global value from `DEFAULT` since it's only in that section
|
||||
|
@ -390,6 +390,13 @@ max_header_size 8192 max_header_size is the max number of bytes in
|
|||
See also include_service_catalog in
|
||||
proxy-server.conf-sample (documented in
|
||||
overview_auth.rst).
|
||||
extra_header_count 0 By default the maximum number of allowed
|
||||
headers depends on the number of max
|
||||
allowed metadata settings plus a default
|
||||
value of 32 for regular http headers.
|
||||
If for some reason this is not enough (custom
|
||||
middleware for example) it can be increased
|
||||
with the extra_header_count constraint.
|
||||
=================== ========== =============================================
|
||||
|
||||
---------------------------
|
||||
|
@ -405,76 +412,86 @@ The following configuration options are available:
|
|||
|
||||
[DEFAULT]
|
||||
|
||||
=================== ========== =============================================
|
||||
Option Default Description
|
||||
------------------- ---------- ---------------------------------------------
|
||||
swift_dir /etc/swift Swift configuration directory
|
||||
devices /srv/node Parent directory of where devices are mounted
|
||||
mount_check true Whether or not check if the devices are
|
||||
mounted to prevent accidentally writing
|
||||
to the root device
|
||||
bind_ip 0.0.0.0 IP Address for server to bind to
|
||||
bind_port 6000 Port for server to bind to
|
||||
bind_timeout 30 Seconds to attempt bind before giving up
|
||||
workers auto Override the number of pre-forked workers
|
||||
that will accept connections. If set it
|
||||
should be an integer, zero means no fork. If
|
||||
unset, it will try to default to the number
|
||||
of effective cpu cores and fallback to one.
|
||||
Increasing the number of workers helps slow
|
||||
filesystem operations in one request from
|
||||
negatively impacting other requests, but only
|
||||
the :ref:`servers_per_port
|
||||
<server-per-port-configuration>`
|
||||
option provides complete I/O isolation with
|
||||
no measurable overhead.
|
||||
servers_per_port 0 If each disk in each storage policy ring has
|
||||
unique port numbers for its "ip" value, you
|
||||
can use this setting to have each
|
||||
object-server worker only service requests
|
||||
for the single disk matching the port in the
|
||||
ring. The value of this setting determines
|
||||
how many worker processes run for each port
|
||||
(disk) in the ring. If you have 24 disks
|
||||
per server, and this setting is 4, then
|
||||
each storage node will have 1 + (24 * 4) =
|
||||
97 total object-server processes running.
|
||||
This gives complete I/O isolation, drastically
|
||||
reducing the impact of slow disks on storage
|
||||
node performance. The object-replicator and
|
||||
object-reconstructor need to see this setting
|
||||
too, so it must be in the [DEFAULT] section.
|
||||
See :ref:`server-per-port-configuration`.
|
||||
max_clients 1024 Maximum number of clients one worker can
|
||||
process simultaneously (it will actually
|
||||
accept(2) N + 1). Setting this to one (1)
|
||||
will only handle one request at a time,
|
||||
without accepting another request
|
||||
concurrently.
|
||||
disable_fallocate false Disable "fast fail" fallocate checks if the
|
||||
underlying filesystem does not support it.
|
||||
log_max_line_length 0 Caps the length of log lines to the
|
||||
value given; no limit if set to 0, the
|
||||
default.
|
||||
log_custom_handlers None Comma-separated list of functions to call
|
||||
to setup custom log handlers.
|
||||
eventlet_debug false If true, turn on debug logging for eventlet
|
||||
fallocate_reserve 0 You can set fallocate_reserve to the number of
|
||||
bytes you'd like fallocate to reserve, whether
|
||||
there is space for the given file size or not.
|
||||
This is useful for systems that behave badly
|
||||
when they completely run out of space; you can
|
||||
make the services pretend they're out of space
|
||||
early.
|
||||
conn_timeout 0.5 Time to wait while attempting to connect to
|
||||
another backend node.
|
||||
node_timeout 3 Time to wait while sending each chunk of data
|
||||
to another backend node.
|
||||
client_timeout 60 Time to wait while receiving each chunk of
|
||||
data from a client or another backend node.
|
||||
network_chunk_size 65536 Size of chunks to read/write over the network
|
||||
disk_chunk_size 65536 Size of chunks to read/write to disk
|
||||
=================== ========== =============================================
|
||||
======================== ========== ==========================================
|
||||
Option Default Description
|
||||
------------------------ ---------- ------------------------------------------
|
||||
swift_dir /etc/swift Swift configuration directory
|
||||
devices /srv/node Parent directory of where devices are
|
||||
mounted
|
||||
mount_check true Whether or not check if the devices are
|
||||
mounted to prevent accidentally writing
|
||||
to the root device
|
||||
bind_ip 0.0.0.0 IP Address for server to bind to
|
||||
bind_port 6000 Port for server to bind to
|
||||
bind_timeout 30 Seconds to attempt bind before giving up
|
||||
workers auto Override the number of pre-forked workers
|
||||
that will accept connections. If set it
|
||||
should be an integer, zero means no fork.
|
||||
If unset, it will try to default to the
|
||||
number of effective cpu cores and fallback
|
||||
to one. Increasing the number of workers
|
||||
helps slow filesystem operations in one
|
||||
request from negatively impacting other
|
||||
requests, but only the
|
||||
:ref:`servers_per_port
|
||||
<server-per-port-configuration>` option
|
||||
provides complete I/O isolation with no
|
||||
measurable overhead.
|
||||
servers_per_port 0 If each disk in each storage policy ring
|
||||
has unique port numbers for its "ip"
|
||||
value, you can use this setting to have
|
||||
each object-server worker only service
|
||||
requests for the single disk matching the
|
||||
port in the ring. The value of this
|
||||
setting determines how many worker
|
||||
processes run for each port (disk) in the
|
||||
ring. If you have 24 disks per server, and
|
||||
this setting is 4, then each storage node
|
||||
will have 1 + (24 * 4) = 97 total
|
||||
object-server processes running. This
|
||||
gives complete I/O isolation, drastically
|
||||
reducing the impact of slow disks on
|
||||
storage node performance. The
|
||||
object-replicator and object-reconstructor
|
||||
need to see this setting too, so it must
|
||||
be in the [DEFAULT] section.
|
||||
See :ref:`server-per-port-configuration`.
|
||||
max_clients 1024 Maximum number of clients one worker can
|
||||
process simultaneously (it will actually
|
||||
accept(2) N + 1). Setting this to one (1)
|
||||
will only handle one request at a time,
|
||||
without accepting another request
|
||||
concurrently.
|
||||
disable_fallocate false Disable "fast fail" fallocate checks if
|
||||
the underlying filesystem does not support
|
||||
it.
|
||||
log_max_line_length 0 Caps the length of log lines to the
|
||||
value given; no limit if set to 0, the
|
||||
default.
|
||||
log_custom_handlers None Comma-separated list of functions to call
|
||||
to setup custom log handlers.
|
||||
eventlet_debug false If true, turn on debug logging for
|
||||
eventlet
|
||||
fallocate_reserve 0 You can set fallocate_reserve to the
|
||||
number of bytes you'd like fallocate to
|
||||
reserve, whether there is space for the
|
||||
given file size or not. This is useful for
|
||||
systems that behave badly when they
|
||||
completely run out of space; you can
|
||||
make the services pretend they're out of
|
||||
space early.
|
||||
conn_timeout 0.5 Time to wait while attempting to connect
|
||||
to another backend node.
|
||||
node_timeout 3 Time to wait while sending each chunk of
|
||||
data to another backend node.
|
||||
client_timeout 60 Time to wait while receiving each chunk of
|
||||
data from a client or another backend node
|
||||
network_chunk_size 65536 Size of chunks to read/write over the
|
||||
network
|
||||
disk_chunk_size 65536 Size of chunks to read/write to disk
|
||||
container_update_timeout 1 Time to wait while sending a container
|
||||
update on object update.
|
||||
======================== ========== ==========================================
|
||||
|
||||
.. _object-server-options:
|
||||
|
||||
|
@ -1229,6 +1246,10 @@ For a standard swift install, all data drives are mounted directly under
|
|||
be sure to set the `devices` config option in all of the server configs to
|
||||
point to the correct directory.
|
||||
|
||||
The mount points for each drive in /srv/node/ should be owned by the root user
|
||||
almost exclusively (root:root 755). This is required to prevent rsync from
|
||||
syncing files into the root drive in the event a drive is unmounted.
|
||||
|
||||
Swift uses system calls to reserve space for new objects being written into
|
||||
the system. If your filesystem does not support `fallocate()` or
|
||||
`posix_fallocate()`, be sure to set the `disable_fallocate = true` config
|
||||
|
|
|
@ -42,7 +42,7 @@ To execute the unit tests:
|
|||
|
||||
Remarks:
|
||||
If you installed using: `cd ~/swift; sudo python setup.py develop`,
|
||||
you may need to do: `cd ~/swift; sudo chown -R swift:swift swift.egg-info`
|
||||
you may need to do: `cd ~/swift; sudo chown -R ${USER}:${USER} swift.egg-info`
|
||||
prior to running tox.
|
||||
|
||||
* Optionally, run only specific tox builds:
|
||||
|
@ -71,6 +71,18 @@ The endpoint and authorization credentials to be used by functional tests
|
|||
should be configured in the ``test.conf`` file as described in the section
|
||||
:ref:`setup_scripts`.
|
||||
|
||||
The environment variable ``SWIFT_TEST_POLICY`` may be set to specify a
|
||||
particular storage policy *name* that will be used for testing. When set, tests
|
||||
that would otherwise not specify a policy or choose a random policy from
|
||||
those available will instead use the policy specified. Tests that use more than
|
||||
one policy will include the specified policy in the set of policies used. The
|
||||
specified policy must be available on the cluster under test.
|
||||
|
||||
For example, this command would run the functional tests using policy
|
||||
'silver'::
|
||||
|
||||
SWIFT_TEST_POLICY=silver tox -e func
|
||||
|
||||
If the ``test.conf`` file is not found then the functional test framework will
|
||||
instantiate a set of Swift servers in the same process that executes the
|
||||
functional tests. This 'in-process test' mode may also be enabled (or disabled)
|
||||
|
@ -95,13 +107,14 @@ found in ``<custom_conf_source_dir>``, the search will then look in the
|
|||
the corresponding sample config file from ``etc/`` is used (e.g.
|
||||
``proxy-server.conf-sample`` or ``swift.conf-sample``).
|
||||
|
||||
The environment variable ``SWIFT_TEST_POLICY`` may be set to specify
|
||||
a particular storage policy *name* that will be used for testing. When set,
|
||||
this policy must exist in the ``swift.conf`` file and its corresponding ring
|
||||
file must exist in ``<custom_conf_source_dir>`` (if specified) or ``etc/``. The
|
||||
test setup will set the specified policy to be the default and use its ring
|
||||
file properties for constructing the test object ring. This allows in-process
|
||||
testing to be run against various policy types and ring files.
|
||||
When using the 'in-process test' mode ``SWIFT_TEST_POLICY`` may be set to
|
||||
specify a particular storage policy *name* that will be used for testing as
|
||||
described above. When set, this policy must exist in the ``swift.conf`` file
|
||||
and its corresponding ring file must exist in ``<custom_conf_source_dir>`` (if
|
||||
specified) or ``etc/``. The test setup will set the specified policy to be the
|
||||
default and use its ring file properties for constructing the test object ring.
|
||||
This allows in-process testing to be run against various policy types and ring
|
||||
files.
|
||||
|
||||
For example, this command would run the in-process mode functional tests
|
||||
using config files found in ``$HOME/my_tests`` and policy 'silver'::
|
||||
|
|
|
@ -4,7 +4,7 @@ Pluggable On-Disk Back-end APIs
|
|||
|
||||
The internal REST API used between the proxy server and the account, container
|
||||
and object server is almost identical to public Swift REST API, but with a few
|
||||
internal extentsions (for example, update an account with a new container).
|
||||
internal extensions (for example, update an account with a new container).
|
||||
|
||||
The pluggable back-end APIs for the three REST API servers (account,
|
||||
container, object) abstracts the needs for servicing the various REST APIs
|
||||
|
|
|
@ -95,6 +95,16 @@ another device when creating the VM, and follow these instructions:
|
|||
# **Make sure to include the trailing slash after /srv/$x/**
|
||||
for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done
|
||||
|
||||
Note: We create the mount points and mount the storage disk under
|
||||
/mnt/sdb1. This disk will contain one directory per simulated swift node,
|
||||
each owned by the current swift user.
|
||||
|
||||
We then create symlinks to these directories under /srv.
|
||||
If the disk sdb is unmounted, files will not be written under
|
||||
/srv/\*, because the symbolic link destination /mnt/sdb1/* will not
|
||||
exist. This prevents disk sync operations from writing to the root
|
||||
partition in the event a drive is unmounted.
|
||||
|
||||
#. Next, skip to :ref:`common-dev-section`.
|
||||
|
||||
|
||||
|
@ -135,6 +145,15 @@ these instructions:
|
|||
# **Make sure to include the trailing slash after /srv/$x/**
|
||||
for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done
|
||||
|
||||
Note: We create the mount points and mount the loopback file under
|
||||
/mnt/sdb1. This file will contain one directory per simulated swift node,
|
||||
each owned by the current swift user.
|
||||
|
||||
We then create symlinks to these directories under /srv.
|
||||
If the loopback file is unmounted, files will not be written under
|
||||
/srv/\*, because the symbolic link destination /mnt/sdb1/* will not
|
||||
exist. This prevents disk sync operations from writing to the root
|
||||
partition in the event a drive is unmounted.
|
||||
|
||||
.. _common-dev-section:
|
||||
|
||||
|
@ -184,7 +203,7 @@ Getting the code
|
|||
|
||||
#. Install swift's test dependencies::
|
||||
|
||||
sudo pip install -r swift/test-requirements.txt
|
||||
cd $HOME/swift; sudo pip install -r test-requirements.txt
|
||||
|
||||
----------------
|
||||
Setting up rsync
|
||||
|
@ -352,6 +371,10 @@ commands are as follows:
|
|||
|
||||
.. literalinclude:: /../saio/swift/container-reconciler.conf
|
||||
|
||||
#. ``/etc/swift/container-sync-realms.conf``
|
||||
|
||||
.. literalinclude:: /../saio/swift/container-sync-realms.conf
|
||||
|
||||
#. ``/etc/swift/account-server/1.conf``
|
||||
|
||||
.. literalinclude:: /../saio/swift/account-server/1.conf
|
||||
|
|
|
@ -0,0 +1,204 @@
|
|||
===========================
|
||||
First Contribution to Swift
|
||||
===========================
|
||||
|
||||
-------------
|
||||
Getting Swift
|
||||
-------------
|
||||
|
||||
Swift's source code is hosted on github and managed with git. The current
|
||||
trunk can be checked out like this:
|
||||
|
||||
``git clone https://github.com/openstack/swift.git``
|
||||
|
||||
This will clone the Swift repository under your account.
|
||||
|
||||
A source tarball for the latest release of Swift is available on the
|
||||
`launchpad project page <https://launchpad.net/swift>`_.
|
||||
|
||||
Prebuilt packages for Ubuntu and RHEL variants are available.
|
||||
|
||||
* `Swift Ubuntu Packages <https://launchpad.net/ubuntu/+source/swift>`_
|
||||
* `Swift RDO Packages <https://www.rdoproject.org/Repositories>`_
|
||||
|
||||
--------------------
|
||||
Source Control Setup
|
||||
--------------------
|
||||
|
||||
Swift uses `git` for source control. The OpenStack
|
||||
`Developer's Guide <http://docs.openstack.org/infra/manual/developers.html>`_
|
||||
describes the steps for setting up Git and all the necessary accounts for
|
||||
contributing code to Swift.
|
||||
|
||||
----------------
|
||||
Changes to Swift
|
||||
----------------
|
||||
|
||||
Once you have the source code and source control set up, you can make your
|
||||
changes to Swift.
|
||||
|
||||
-------
|
||||
Testing
|
||||
-------
|
||||
|
||||
The `Development Guidelines <development_guidelines>`_ describes the testing
|
||||
requirements before submitting Swift code.
|
||||
|
||||
In summary, you can execute tox from the swift home directory (where you
|
||||
checked out the source code):
|
||||
|
||||
``tox``
|
||||
|
||||
Tox will present tests results. Notice that in the beginning, it is very common
|
||||
to break many coding style guidelines.
|
||||
|
||||
--------------------------
|
||||
Proposing changes to Swift
|
||||
--------------------------
|
||||
|
||||
The OpenStack
|
||||
`Developer's Guide <http://docs.openstack.org/infra/manual/developers.html>`_
|
||||
describes the most common `git` commands that you will need.
|
||||
|
||||
Following is a list of the commands that you need to know for your first
|
||||
contribution to Swift:
|
||||
|
||||
To clone a copy of Swift:
|
||||
|
||||
``git clone https://github.com/openstack/swift.git``
|
||||
|
||||
Under the swift directory, set up the Gerrit repository. The following command
|
||||
configures the repository to know about Gerrit and makes the Change-Id commit
|
||||
hook get installed. You only need to do this once:
|
||||
|
||||
``git review -s``
|
||||
|
||||
To create your development branch (substitute branch_name for a name of your
|
||||
choice:
|
||||
|
||||
``git checkout -b <branch_name>``
|
||||
|
||||
To check the files that have been updated in your branch:
|
||||
|
||||
``git status``
|
||||
|
||||
To check the differences between your branch and the repository:
|
||||
|
||||
``git diff``
|
||||
|
||||
Assuming you have not added new files, you commit all your changes using:
|
||||
|
||||
``git commit -a``
|
||||
|
||||
Read the `Summary of Git commit message structure <https://wiki.openstack.org/wiki/GitCommitMessages?%22Summary%20of%20Git%20commit%20message%20structure%22#Summary_of_Git_commit_message_structure>`_
|
||||
for best practices on writing the commit message. When you are ready to send
|
||||
your changes for review use:
|
||||
|
||||
``git review``
|
||||
|
||||
If successful, Git response message will contain a URL you can use to track your
|
||||
changes.
|
||||
|
||||
If you need to make further changes to the same review, you can commit them
|
||||
using:
|
||||
|
||||
``git commit -a --amend``
|
||||
|
||||
This will commit the changes under the same set of changes you issued earlier.
|
||||
Notice that in order to send your latest version for review, you will still
|
||||
need to call:
|
||||
|
||||
``git review``
|
||||
|
||||
---------------------
|
||||
Tracking your changes
|
||||
---------------------
|
||||
|
||||
After you proposed your changes to Swift, you can track the review in:
|
||||
|
||||
* `<https://review.openstack.org>`_
|
||||
|
||||
.. _post-rebase-instructions:
|
||||
|
||||
------------------------
|
||||
Post rebase instructions
|
||||
------------------------
|
||||
|
||||
After rebasing, the following steps should be performed to rebuild the swift
|
||||
installation. Note that these commands should be performed from the root of the
|
||||
swift repo directory (e.g. $HOME/swift/):
|
||||
|
||||
``sudo python setup.py develop``
|
||||
|
||||
``sudo pip install -r test-requirements.txt``
|
||||
|
||||
If using TOX, depending on the changes made during the rebase, you may need to
|
||||
rebuild the TOX environment (generally this will be the case if
|
||||
test-requirements.txt was updated such that a new version of a package is
|
||||
required), this can be accomplished using the '-r' argument to the TOX cli:
|
||||
|
||||
``tox -r``
|
||||
|
||||
You can include any of the other TOX arguments as well, for example, to run the
|
||||
pep8 suite and rebuild the TOX environment the following can be used:
|
||||
|
||||
``tox -r -e pep8``
|
||||
|
||||
The rebuild option only needs to be specified once for a particular build (e.g.
|
||||
pep8), that is further invocations of the same build will not require this
|
||||
until the next rebase.
|
||||
|
||||
---------------
|
||||
Troubleshooting
|
||||
---------------
|
||||
|
||||
You may run into the following errors when starting Swift if you rebase
|
||||
your commit using:
|
||||
|
||||
``git rebase``
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/usr/local/bin/swift-init", line 5, in <module>
|
||||
from pkg_resources import require
|
||||
File "/usr/lib/python2.7/dist-packages/pkg_resources.py", line 2749, in <module>
|
||||
working_set = WorkingSet._build_master()
|
||||
File "/usr/lib/python2.7/dist-packages/pkg_resources.py", line 446, in _build_master
|
||||
return cls._build_from_requirements(__requires__)
|
||||
File "/usr/lib/python2.7/dist-packages/pkg_resources.py", line 459, in _build_from_requirements
|
||||
dists = ws.resolve(reqs, Environment())
|
||||
File "/usr/lib/python2.7/dist-packages/pkg_resources.py", line 628, in resolve
|
||||
raise DistributionNotFound(req)
|
||||
pkg_resources.DistributionNotFound: swift==2.3.1.devXXX
|
||||
(where XXX represents a dev version of Swift).
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/usr/local/bin/swift-proxy-server", line 10, in <module>
|
||||
execfile(__file__)
|
||||
File "/home/swift/swift/bin/swift-proxy-server", line 23, in <module>
|
||||
sys.exit(run_wsgi(conf_file, 'proxy-server', **options))
|
||||
File "/home/swift/swift/swift/common/wsgi.py", line 888, in run_wsgi
|
||||
loadapp(conf_path, global_conf=global_conf)
|
||||
File "/home/swift/swift/swift/common/wsgi.py", line 390, in loadapp
|
||||
func(PipelineWrapper(ctx))
|
||||
File "/home/swift/swift/swift/proxy/server.py", line 602, in modify_wsgi_pipeline
|
||||
ctx = pipe.create_filter(filter_name)
|
||||
File "/home/swift/swift/swift/common/wsgi.py", line 329, in create_filter
|
||||
global_conf=self.context.global_conf)
|
||||
File "/usr/lib/python2.7/dist-packages/paste/deploy/loadwsgi.py", line 296, in loadcontext
|
||||
global_conf=global_conf)
|
||||
File "/usr/lib/python2.7/dist-packages/paste/deploy/loadwsgi.py", line 328, in _loadegg
|
||||
return loader.get_context(object_type, name, global_conf)
|
||||
File "/usr/lib/python2.7/dist-packages/paste/deploy/loadwsgi.py", line 620, in get_context
|
||||
object_type, name=name)
|
||||
File "/usr/lib/python2.7/dist-packages/paste/deploy/loadwsgi.py", line 659, in find_egg_entry_point
|
||||
for prot in protocol_options] or '(no entry points)'))))
|
||||
LookupError: Entry point 'versioned_writes' not found in egg 'swift' (dir: /home/swift/swift; protocols: paste.filter_factory, paste.filter_app_factory; entry_points: )
|
||||
|
||||
This happens because `git rebase` will retrieve code for a different version of
|
||||
Swift in the development stream, but the start scripts under `/usr/local/bin` have
|
||||
not been updated. The solution is to follow the steps described in the
|
||||
:ref:`post-rebase-instructions` section.
|
|
@ -18,23 +18,6 @@ Swift is written in Python and has these dependencies:
|
|||
|
||||
There is no current support for Python 3.
|
||||
|
||||
-------------
|
||||
Getting Swift
|
||||
-------------
|
||||
|
||||
Swift's source code is hosted on github and managed with git. The current
|
||||
trunk can be checked out like this:
|
||||
|
||||
``git clone https://github.com/openstack/swift.git``
|
||||
|
||||
A source tarball for the latest release of Swift is available on the
|
||||
`launchpad project page <https://launchpad.net/swift>`_.
|
||||
|
||||
Prebuilt packages for Ubuntu and RHEL variants are available.
|
||||
|
||||
* `Swift Ubuntu Packages <https://launchpad.net/ubuntu/+source/swift>`_
|
||||
* `Swift RDO Packages <https://openstack.redhat.com/Repositories>`_
|
||||
|
||||
-----------
|
||||
Development
|
||||
-----------
|
||||
|
@ -42,10 +25,10 @@ Development
|
|||
To get started with development with Swift, or to just play around, the
|
||||
following docs will be useful:
|
||||
|
||||
* :doc:`Swift All in One <development_saio>` - Set up a VM with Swift
|
||||
installed
|
||||
* :doc:`Swift All in One <development_saio>` - Set up a VM with Swift installed
|
||||
* :doc:`Development Guidelines <development_guidelines>`
|
||||
* `Associated Projects <http://docs.openstack.org/developer/swift/associated_projects.html>`
|
||||
* :doc:`First Contribution to Swift <first_contribution_swift>`
|
||||
* :doc:`Associated Projects <associated_projects>`
|
||||
|
||||
--------------------------
|
||||
CLI client and SDK library
|
||||
|
|
|
@ -6,6 +6,13 @@ Please refer to the latest official
|
|||
`Openstack Installation Guides <http://docs.openstack.org/#install-guides>`_
|
||||
for the most up-to-date documentation.
|
||||
|
||||
Object Storage installation guide for Openstack Kilo
|
||||
----------------------------------------------------
|
||||
|
||||
* `openSUSE 13.2 and SUSE Linux Enterprise Server 12 <http://docs.openstack.org/kilo/install-guide/install/zypper/content/ch_swift.html>`_
|
||||
* `RHEL 7, CentOS 7, and Fedora 21 <http://docs.openstack.org/kilo/install-guide/install/yum/content/ch_swift.html>`_
|
||||
* `Ubuntu 14.04 <http://docs.openstack.org/kilo/install-guide/install/apt/content/ch_swift.html>`_
|
||||
|
||||
Object Storage installation guide for Openstack Juno
|
||||
----------------------------------------------------
|
||||
|
||||
|
|
|
@ -68,6 +68,7 @@ Developer Documentation
|
|||
|
||||
development_guidelines
|
||||
development_saio
|
||||
first_contribution_swift
|
||||
policies_saio
|
||||
development_auth
|
||||
development_middleware
|
||||
|
|
|
@ -59,7 +59,7 @@ client_etag The etag header value given by the client.
|
|||
transaction_id The transaction id of the request.
|
||||
headers The headers given in the request.
|
||||
request_time The duration of the request.
|
||||
source The "source" of the reuqest. This may be set for requests
|
||||
source The "source" of the request. This may be set for requests
|
||||
that are generated in order to fulfill client requests,
|
||||
e.g. bulk uploads.
|
||||
log_info Various info that may be useful for diagnostics, e.g. the
|
||||
|
@ -102,6 +102,7 @@ DLO :ref:`dynamic-large-objects`
|
|||
LE :ref:`list_endpoints`
|
||||
KS :ref:`keystoneauth`
|
||||
RL :ref:`ratelimit`
|
||||
VW :ref:`versioned_writes`
|
||||
======================= =============================
|
||||
|
||||
|
||||
|
|
|
@ -155,6 +155,15 @@ Name Check (Forbidden Character Filter)
|
|||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
.. _versioned_writes:
|
||||
|
||||
Object Versioning
|
||||
=================
|
||||
|
||||
.. automodule:: swift.common.middleware.versioned_writes
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
Proxy Logging
|
||||
=============
|
||||
|
||||
|
|
|
@ -13,7 +13,7 @@ architecture. For each request, it will look up the location of the account,
|
|||
container, or object in the ring (see below) and route the request accordingly.
|
||||
For Erasure Code type policies, the Proxy Server is also responsible for
|
||||
encoding and decoding object data. See :doc:`overview_erasure_code` for
|
||||
complete information on Erasure Code suport. The public API is also exposed
|
||||
complete information on Erasure Code support. The public API is also exposed
|
||||
through the Proxy Server.
|
||||
|
||||
A large number of failures are also handled in the Proxy Server. For
|
||||
|
|
|
@ -425,7 +425,7 @@ The basic flow looks like this:
|
|||
* The proxy waits for a minimal number of two object servers to respond with a
|
||||
success (2xx) status before responding to the client with a successful
|
||||
status. In this particular case it was decided that two responses was
|
||||
the mininum amount to know that the file would be propagated in case of
|
||||
the minimum amount to know that the file would be propagated in case of
|
||||
failure from other others and because a greater number would potentially
|
||||
mean more latency, which should be avoided if possible.
|
||||
|
||||
|
|
|
@ -1,89 +1,6 @@
|
|||
=================
|
||||
Object Versioning
|
||||
=================
|
||||
|
||||
--------
|
||||
Overview
|
||||
--------
|
||||
|
||||
Object versioning in swift is implemented by setting a flag on the container
|
||||
to tell swift to version all objects in the container. The flag is the
|
||||
``X-Versions-Location`` header on the container, and its value is the
|
||||
container where the versions are stored. It is recommended to use a different
|
||||
``X-Versions-Location`` container for each container that is being versioned.
|
||||
|
||||
When data is ``PUT`` into a versioned container (a container with the
|
||||
versioning flag turned on), the existing data in the file is redirected to a
|
||||
new object and the data in the ``PUT`` request is saved as the data for the
|
||||
versioned object. The new object name (for the previous version) is
|
||||
``<versions_container>/<length><object_name>/<timestamp>``, where ``length``
|
||||
is the 3-character zero-padded hexadecimal length of the ``<object_name>`` and
|
||||
``<timestamp>`` is the timestamp of when the previous version was created.
|
||||
|
||||
A ``GET`` to a versioned object will return the current version of the object
|
||||
without having to do any request redirects or metadata lookups.
|
||||
|
||||
A ``POST`` to a versioned object will update the object metadata as normal,
|
||||
but will not create a new version of the object. In other words, new versions
|
||||
are only created when the content of the object changes.
|
||||
|
||||
A ``DELETE`` to a versioned object will only remove the current version of the
|
||||
object. If you have 5 total versions of the object, you must delete the
|
||||
object 5 times to completely remove the object.
|
||||
|
||||
Note: A large object manifest file cannot be versioned, but a large object
|
||||
manifest may point to versioned segments.
|
||||
|
||||
--------------------------------------------------
|
||||
How to Enable Object Versioning in a Swift Cluster
|
||||
--------------------------------------------------
|
||||
|
||||
Set ``allow_versions`` to ``True`` in the container server config.
|
||||
|
||||
-----------------------
|
||||
Examples Using ``curl``
|
||||
-----------------------
|
||||
|
||||
First, create a container with the ``X-Versions-Location`` header or add the
|
||||
header to an existing container. Also make sure the container referenced by
|
||||
the ``X-Versions-Location`` exists. In this example, the name of that
|
||||
container is "versions"::
|
||||
|
||||
curl -i -XPUT -H "X-Auth-Token: <token>" \
|
||||
-H "X-Versions-Location: versions" http://<storage_url>/container
|
||||
curl -i -XPUT -H "X-Auth-Token: <token>" http://<storage_url>/versions
|
||||
|
||||
Create an object (the first version)::
|
||||
|
||||
curl -i -XPUT --data-binary 1 -H "X-Auth-Token: <token>" \
|
||||
http://<storage_url>/container/myobject
|
||||
|
||||
Now create a new version of that object::
|
||||
|
||||
curl -i -XPUT --data-binary 2 -H "X-Auth-Token: <token>" \
|
||||
http://<storage_url>/container/myobject
|
||||
|
||||
See a listing of the older versions of the object::
|
||||
|
||||
curl -i -H "X-Auth-Token: <token>" \
|
||||
http://<storage_url>/versions?prefix=008myobject/
|
||||
|
||||
Now delete the current version of the object and see that the older version is
|
||||
gone::
|
||||
|
||||
curl -i -XDELETE -H "X-Auth-Token: <token>" \
|
||||
http://<storage_url>/container/myobject
|
||||
curl -i -H "X-Auth-Token: <token>" \
|
||||
http://<storage_url>/versions?prefix=008myobject/
|
||||
|
||||
---------------------------------------------------
|
||||
How to Disable Object Versioning in a Swift Cluster
|
||||
---------------------------------------------------
|
||||
|
||||
If you want to disable all functionality, set ``allow_versions`` back to
|
||||
``False`` in the container server config.
|
||||
|
||||
Disable versioning a versioned container (x is any value except empty)::
|
||||
|
||||
curl -i -XPOST -H "X-Auth-Token: <token>" \
|
||||
-H "X-Remove-Versions-Location: x" http://<storage_url>/container
|
||||
.. automodule:: swift.common.middleware.versioned_writes
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
|
|
@ -168,6 +168,21 @@ on them than the disks in nodes A and B. If 80% full is the warning
|
|||
threshold for the cluster, node C's disks will reach 80% full while A
|
||||
and B's disks are only 72.7% full.
|
||||
|
||||
**********
|
||||
Dispersion
|
||||
**********
|
||||
|
||||
With each rebalance, the ring builder calculates a dispersion metric. This is
|
||||
the percentage of partitions in the ring that have too many replicas within a
|
||||
particular failure domain.
|
||||
|
||||
For example, if you have three servers in a cluster but two replicas for a
|
||||
partition get placed onto the same server, that partition will count towards the
|
||||
dispersion metric.
|
||||
|
||||
A lower dispersion value is better, and the value can be used to find the proper
|
||||
value for "overload".
|
||||
|
||||
*********************
|
||||
Partition Shift Value
|
||||
*********************
|
||||
|
|
|
@ -100,13 +100,6 @@ use = egg:swift#recon
|
|||
# run_pause is deprecated, use interval instead
|
||||
# run_pause = 30
|
||||
#
|
||||
# How long without an error before a node's error count is reset. This will
|
||||
# also be how long before a node is reenabled after suppression is triggered.
|
||||
# error_suppression_interval = 60
|
||||
#
|
||||
# How many errors can accumulate before a node is temporarily ignored.
|
||||
# error_suppression_limit = 10
|
||||
#
|
||||
# node_timeout = 10
|
||||
# conn_timeout = 0.5
|
||||
#
|
||||
|
|
|
@ -13,6 +13,16 @@ auth_key = testing
|
|||
# auth_key = password
|
||||
# auth_version = 2.0
|
||||
#
|
||||
# NOTE: If you want to use keystone (auth version 3.0), then its configuration
|
||||
# would look something like:
|
||||
# auth_url = http://localhost:5000/v3/
|
||||
# auth_user = user
|
||||
# auth_key = password
|
||||
# auth_version = 3.0
|
||||
# project_name = project
|
||||
# project_domain_name = project_domain
|
||||
# user_domain_name = user_domain
|
||||
#
|
||||
# endpoint_type = publicURL
|
||||
# keystone_api_insecure = no
|
||||
#
|
||||
|
|
|
@ -41,7 +41,7 @@
|
|||
# config value
|
||||
# processes = 0
|
||||
# process is which of the parts a particular process will work on
|
||||
# process can also be specified on the command line and will overide the config
|
||||
# process can also be specified on the command line and will override the config
|
||||
# value
|
||||
# process is "zero based", if you want to use 3 processes, you should run
|
||||
# processes with process set to 0, 1, and 2
|
||||
|
|
|
@ -60,6 +60,8 @@ bind_port = 6000
|
|||
# conn_timeout = 0.5
|
||||
# Time to wait while sending each chunk of data to another backend node.
|
||||
# node_timeout = 3
|
||||
# Time to wait while sending a container update on object update.
|
||||
# container_update_timeout = 1.0
|
||||
# Time to wait while receiving each chunk of data from a client or another
|
||||
# backend node.
|
||||
# client_timeout = 60
|
||||
|
|
|
@ -77,7 +77,7 @@ bind_port = 8080
|
|||
# eventlet_debug = false
|
||||
|
||||
[pipeline:main]
|
||||
pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit tempauth container-quotas account-quotas slo dlo proxy-logging proxy-server
|
||||
pipeline = catch_errors gatekeeper healthcheck proxy-logging cache container_sync bulk tempurl ratelimit tempauth container-quotas account-quotas slo dlo versioned_writes proxy-logging proxy-server
|
||||
|
||||
[app:proxy-server]
|
||||
use = egg:swift#proxy
|
||||
|
@ -703,3 +703,14 @@ use = egg:swift#xprofile
|
|||
#
|
||||
# unwind the iterator of applications
|
||||
# unwind = false
|
||||
|
||||
# Note: Put after slo, dlo in the pipeline.
|
||||
# If you don't put it in the pipeline, it will be inserted automatically.
|
||||
[filter:versioned_writes]
|
||||
use = egg:swift#versioned_writes
|
||||
# Enables using versioned writes middleware and exposing configuration
|
||||
# settings via HTTP GET /info.
|
||||
# WARNING: Setting this option bypasses the "allow_versions" option
|
||||
# in the container configuration file, which will be eventually
|
||||
# deprecated. See documentation for more details.
|
||||
# allow_versioned_writes = false
|
||||
|
|
|
@ -134,7 +134,7 @@ default = yes
|
|||
# headers. If for some reason this is not enough (custom middleware for
|
||||
# example) it can be increased with the extra_header_count constraint.
|
||||
|
||||
#extra_header_count = 32
|
||||
#extra_header_count = 0
|
||||
|
||||
|
||||
# max_object_name_length is the max number of bytes in the utf8 encoding
|
||||
|
|
|
@ -10,4 +10,4 @@ pastedeploy>=1.3.3
|
|||
simplejson>=2.0.9
|
||||
six>=1.9.0
|
||||
xattr>=0.4
|
||||
PyECLib>=1.0.7
|
||||
PyECLib==1.0.7 # BSD
|
||||
|
|
|
@ -95,6 +95,7 @@ paste.filter_factory =
|
|||
gatekeeper = swift.common.middleware.gatekeeper:filter_factory
|
||||
container_sync = swift.common.middleware.container_sync:filter_factory
|
||||
xprofile = swift.common.middleware.xprofile:filter_factory
|
||||
versioned_writes = swift.common.middleware.versioned_writes:filter_factory
|
||||
|
||||
[build_sphinx]
|
||||
all_files = 1
|
||||
|
|
|
@ -122,11 +122,10 @@ class AccountAuditor(Daemon):
|
|||
continue
|
||||
raise InvalidAccountInfo(_(
|
||||
'The total %(key)s for the container (%(total)s) does not '
|
||||
'match the sum of %(key)s across policies (%(sum)s)') % {
|
||||
'key': key,
|
||||
'total': info[key],
|
||||
'sum': policy_totals[key],
|
||||
})
|
||||
'match the sum of %(key)s across policies (%(sum)s)')
|
||||
% {'key': key,
|
||||
'total': info[key],
|
||||
'sum': policy_totals[key]})
|
||||
|
||||
def account_audit(self, path):
|
||||
"""
|
||||
|
|
|
@ -18,7 +18,7 @@ Pluggable Back-end for Account Server
|
|||
|
||||
from uuid import uuid4
|
||||
import time
|
||||
import cPickle as pickle
|
||||
import six.moves.cPickle as pickle
|
||||
|
||||
import sqlite3
|
||||
|
||||
|
@ -380,6 +380,7 @@ class AccountBroker(DatabaseBroker):
|
|||
|
||||
:returns: list of tuples of (name, object_count, bytes_used, 0)
|
||||
"""
|
||||
delim_force_gte = False
|
||||
(marker, end_marker, prefix, delimiter) = utf8encode(
|
||||
marker, end_marker, prefix, delimiter)
|
||||
self._commit_puts_stale_ok()
|
||||
|
@ -392,12 +393,17 @@ class AccountBroker(DatabaseBroker):
|
|||
query = """
|
||||
SELECT name, object_count, bytes_used, 0
|
||||
FROM container
|
||||
WHERE deleted = 0 AND """
|
||||
WHERE """
|
||||
query_args = []
|
||||
if end_marker:
|
||||
query += ' name < ? AND'
|
||||
query_args.append(end_marker)
|
||||
if marker and marker >= prefix:
|
||||
if delim_force_gte:
|
||||
query += ' name >= ? AND'
|
||||
query_args.append(marker)
|
||||
# Always set back to False
|
||||
delim_force_gte = False
|
||||
elif marker and marker >= prefix:
|
||||
query += ' name > ? AND'
|
||||
query_args.append(marker)
|
||||
elif prefix:
|
||||
|
@ -437,6 +443,8 @@ class AccountBroker(DatabaseBroker):
|
|||
end = name.find(delimiter, len(prefix))
|
||||
if end > 0:
|
||||
marker = name[:end] + chr(ord(delimiter) + 1)
|
||||
# we want result to be inclusive of delim+1
|
||||
delim_force_gte = True
|
||||
dir_name = name[:end + 1]
|
||||
if dir_name != orig_marker:
|
||||
results.append([dir_name, 0, 0, 1])
|
||||
|
|
|
@ -15,10 +15,12 @@
|
|||
|
||||
import os
|
||||
import random
|
||||
import socket
|
||||
from swift import gettext_ as _
|
||||
from logging import DEBUG
|
||||
from math import sqrt
|
||||
from time import time
|
||||
from hashlib import md5
|
||||
import itertools
|
||||
|
||||
from eventlet import GreenPool, sleep, Timeout
|
||||
|
@ -70,6 +72,7 @@ class AccountReaper(Daemon):
|
|||
self.node_timeout = int(conf.get('node_timeout', 10))
|
||||
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
|
||||
self.myips = whataremyips(conf.get('bind_ip', '0.0.0.0'))
|
||||
self.bind_port = int(conf.get('bind_port', 0))
|
||||
self.concurrency = int(conf.get('concurrency', 25))
|
||||
self.container_concurrency = self.object_concurrency = \
|
||||
sqrt(self.concurrency)
|
||||
|
@ -79,6 +82,7 @@ class AccountReaper(Daemon):
|
|||
self.delay_reaping = int(conf.get('delay_reaping') or 0)
|
||||
reap_warn_after = float(conf.get('reap_warn_after') or 86400 * 30)
|
||||
self.reap_not_done_after = reap_warn_after + self.delay_reaping
|
||||
self.start_time = time()
|
||||
|
||||
def get_account_ring(self):
|
||||
"""The account :class:`swift.common.ring.Ring` for the cluster."""
|
||||
|
@ -161,9 +165,16 @@ class AccountReaper(Daemon):
|
|||
if not partition.isdigit():
|
||||
continue
|
||||
nodes = self.get_account_ring().get_part_nodes(int(partition))
|
||||
if (not is_local_device(self.myips, None, nodes[0]['ip'], None)
|
||||
or not os.path.isdir(partition_path)):
|
||||
if not os.path.isdir(partition_path):
|
||||
continue
|
||||
container_shard = None
|
||||
for container_shard, node in enumerate(nodes):
|
||||
if is_local_device(self.myips, None, node['ip'], None) and \
|
||||
(not self.bind_port or self.bind_port == node['port']):
|
||||
break
|
||||
else:
|
||||
continue
|
||||
|
||||
for suffix in os.listdir(partition_path):
|
||||
suffix_path = os.path.join(partition_path, suffix)
|
||||
if not os.path.isdir(suffix_path):
|
||||
|
@ -181,7 +192,9 @@ class AccountReaper(Daemon):
|
|||
AccountBroker(os.path.join(hsh_path, fname))
|
||||
if broker.is_status_deleted() and \
|
||||
not broker.empty():
|
||||
self.reap_account(broker, partition, nodes)
|
||||
self.reap_account(
|
||||
broker, partition, nodes,
|
||||
container_shard=container_shard)
|
||||
|
||||
def reset_stats(self):
|
||||
self.stats_return_codes = {}
|
||||
|
@ -192,7 +205,7 @@ class AccountReaper(Daemon):
|
|||
self.stats_containers_possibly_remaining = 0
|
||||
self.stats_objects_possibly_remaining = 0
|
||||
|
||||
def reap_account(self, broker, partition, nodes):
|
||||
def reap_account(self, broker, partition, nodes, container_shard=None):
|
||||
"""
|
||||
Called once per pass for each account this server is the primary for
|
||||
and attempts to delete the data for the given account. The reaper will
|
||||
|
@ -219,6 +232,8 @@ class AccountReaper(Daemon):
|
|||
:param broker: The AccountBroker for the account to delete.
|
||||
:param partition: The partition in the account ring the account is on.
|
||||
:param nodes: The primary node dicts for the account to delete.
|
||||
:param container_shard: int used to shard containers reaped. If None,
|
||||
will reap all containers.
|
||||
|
||||
.. seealso::
|
||||
|
||||
|
@ -237,16 +252,24 @@ class AccountReaper(Daemon):
|
|||
account = info['account']
|
||||
self.logger.info(_('Beginning pass on account %s'), account)
|
||||
self.reset_stats()
|
||||
container_limit = 1000
|
||||
if container_shard is not None:
|
||||
container_limit *= len(nodes)
|
||||
try:
|
||||
marker = ''
|
||||
while True:
|
||||
containers = \
|
||||
list(broker.list_containers_iter(1000, marker, None, None,
|
||||
None))
|
||||
list(broker.list_containers_iter(container_limit, marker,
|
||||
None, None, None))
|
||||
if not containers:
|
||||
break
|
||||
try:
|
||||
for (container, _junk, _junk, _junk) in containers:
|
||||
this_shard = int(md5(container).hexdigest(), 16) % \
|
||||
len(nodes)
|
||||
if container_shard not in (this_shard, None):
|
||||
continue
|
||||
|
||||
self.container_pool.spawn(self.reap_container, account,
|
||||
partition, nodes, container)
|
||||
self.container_pool.waitall()
|
||||
|
@ -347,10 +370,14 @@ class AccountReaper(Daemon):
|
|||
if self.logger.getEffectiveLevel() <= DEBUG:
|
||||
self.logger.exception(
|
||||
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
|
||||
self.stats_return_codes[err.http_status / 100] = \
|
||||
self.stats_return_codes.get(err.http_status / 100, 0) + 1
|
||||
self.stats_return_codes[err.http_status // 100] = \
|
||||
self.stats_return_codes.get(err.http_status // 100, 0) + 1
|
||||
self.logger.increment(
|
||||
'return_codes.%d' % (err.http_status / 100,))
|
||||
'return_codes.%d' % (err.http_status // 100,))
|
||||
except (Timeout, socket.error) as err:
|
||||
self.logger.error(
|
||||
_('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
|
||||
node)
|
||||
if not objects:
|
||||
break
|
||||
try:
|
||||
|
@ -399,10 +426,16 @@ class AccountReaper(Daemon):
|
|||
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
|
||||
failures += 1
|
||||
self.logger.increment('containers_failures')
|
||||
self.stats_return_codes[err.http_status / 100] = \
|
||||
self.stats_return_codes.get(err.http_status / 100, 0) + 1
|
||||
self.stats_return_codes[err.http_status // 100] = \
|
||||
self.stats_return_codes.get(err.http_status // 100, 0) + 1
|
||||
self.logger.increment(
|
||||
'return_codes.%d' % (err.http_status / 100,))
|
||||
'return_codes.%d' % (err.http_status // 100,))
|
||||
except (Timeout, socket.error) as err:
|
||||
self.logger.error(
|
||||
_('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
|
||||
node)
|
||||
failures += 1
|
||||
self.logger.increment('containers_failures')
|
||||
if successes > failures:
|
||||
self.stats_containers_deleted += 1
|
||||
self.logger.increment('containers_deleted')
|
||||
|
@ -469,10 +502,16 @@ class AccountReaper(Daemon):
|
|||
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
|
||||
failures += 1
|
||||
self.logger.increment('objects_failures')
|
||||
self.stats_return_codes[err.http_status / 100] = \
|
||||
self.stats_return_codes.get(err.http_status / 100, 0) + 1
|
||||
self.stats_return_codes[err.http_status // 100] = \
|
||||
self.stats_return_codes.get(err.http_status // 100, 0) + 1
|
||||
self.logger.increment(
|
||||
'return_codes.%d' % (err.http_status / 100,))
|
||||
'return_codes.%d' % (err.http_status // 100,))
|
||||
except (Timeout, socket.error) as err:
|
||||
failures += 1
|
||||
self.logger.increment('objects_failures')
|
||||
self.logger.error(
|
||||
_('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
|
||||
node)
|
||||
if successes > failures:
|
||||
self.stats_objects_deleted += 1
|
||||
self.logger.increment('objects_deleted')
|
||||
|
|
|
@ -15,6 +15,7 @@
|
|||
"""
|
||||
Script for generating a form signature for use with FormPost middleware.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
import hmac
|
||||
from hashlib import sha1
|
||||
from os.path import basename
|
||||
|
@ -24,41 +25,41 @@ from time import time
|
|||
def main(argv):
|
||||
if len(argv) != 7:
|
||||
prog = basename(argv[0])
|
||||
print 'Syntax: %s <path> <redirect> <max_file_size> ' \
|
||||
'<max_file_count> <seconds> <key>' % prog
|
||||
print
|
||||
print 'Where:'
|
||||
print ' <path> The prefix to use for form uploaded'
|
||||
print ' objects. For example:'
|
||||
print ' /v1/account/container/object_prefix_ would'
|
||||
print ' ensure all form uploads have that path'
|
||||
print ' prepended to the browser-given file name.'
|
||||
print ' <redirect> The URL to redirect the browser to after'
|
||||
print ' the uploads have completed.'
|
||||
print ' <max_file_size> The maximum file size per file uploaded.'
|
||||
print ' <max_file_count> The maximum number of uploaded files'
|
||||
print ' allowed.'
|
||||
print ' <seconds> The number of seconds from now to allow'
|
||||
print ' the form post to begin.'
|
||||
print ' <key> The X-Account-Meta-Temp-URL-Key for the'
|
||||
print ' account.'
|
||||
print
|
||||
print 'Example output:'
|
||||
print ' Expires: 1323842228'
|
||||
print ' Signature: 18de97e47345a82c4dbfb3b06a640dbb'
|
||||
print
|
||||
print 'Sample form:'
|
||||
print
|
||||
print('Syntax: %s <path> <redirect> <max_file_size> '
|
||||
'<max_file_count> <seconds> <key>' % prog)
|
||||
print()
|
||||
print('Where:')
|
||||
print(' <path> The prefix to use for form uploaded')
|
||||
print(' objects. For example:')
|
||||
print(' /v1/account/container/object_prefix_ would')
|
||||
print(' ensure all form uploads have that path')
|
||||
print(' prepended to the browser-given file name.')
|
||||
print(' <redirect> The URL to redirect the browser to after')
|
||||
print(' the uploads have completed.')
|
||||
print(' <max_file_size> The maximum file size per file uploaded.')
|
||||
print(' <max_file_count> The maximum number of uploaded files')
|
||||
print(' allowed.')
|
||||
print(' <seconds> The number of seconds from now to allow')
|
||||
print(' the form post to begin.')
|
||||
print(' <key> The X-Account-Meta-Temp-URL-Key for the')
|
||||
print(' account.')
|
||||
print()
|
||||
print('Example output:')
|
||||
print(' Expires: 1323842228')
|
||||
print(' Signature: 18de97e47345a82c4dbfb3b06a640dbb')
|
||||
print()
|
||||
print('Sample form:')
|
||||
print()
|
||||
print('NOTE: the <form> tag\'s "action" attribute does not contain '
|
||||
'the Swift cluster\'s hostname.')
|
||||
print 'You should manually add it before using the form.'
|
||||
print
|
||||
print('You should manually add it before using the form.')
|
||||
print()
|
||||
print('<form action="/v1/a/c/o" method="POST" '
|
||||
'enctype="multipart/form-data">')
|
||||
print ' <input type="hidden" name="max_file_size" value="123" />'
|
||||
print ' ... more HTML ...'
|
||||
print ' <input type="submit" />'
|
||||
print '</form>'
|
||||
print(' <input type="hidden" name="max_file_size" value="123" />')
|
||||
print(' ... more HTML ...')
|
||||
print(' <input type="submit" />')
|
||||
print('</form>')
|
||||
return 1
|
||||
path, redirect, max_file_size, max_file_count, seconds, key = argv[1:]
|
||||
try:
|
||||
|
@ -66,37 +67,37 @@ def main(argv):
|
|||
except ValueError:
|
||||
max_file_size = -1
|
||||
if max_file_size < 0:
|
||||
print 'Please use a <max_file_size> value greater than or equal to 0.'
|
||||
print('Please use a <max_file_size> value greater than or equal to 0.')
|
||||
return 1
|
||||
try:
|
||||
max_file_count = int(max_file_count)
|
||||
except ValueError:
|
||||
max_file_count = 0
|
||||
if max_file_count < 1:
|
||||
print 'Please use a positive <max_file_count> value.'
|
||||
print('Please use a positive <max_file_count> value.')
|
||||
return 1
|
||||
try:
|
||||
expires = int(time() + int(seconds))
|
||||
except ValueError:
|
||||
expires = 0
|
||||
if expires < 1:
|
||||
print 'Please use a positive <seconds> value.'
|
||||
print('Please use a positive <seconds> value.')
|
||||
return 1
|
||||
parts = path.split('/', 4)
|
||||
# Must be four parts, ['', 'v1', 'a', 'c'], must be a v1 request, have
|
||||
# account and container values, and optionally have an object prefix.
|
||||
if len(parts) < 4 or parts[0] or parts[1] != 'v1' or not parts[2] or \
|
||||
not parts[3]:
|
||||
print '<path> must point to a container at least.'
|
||||
print 'For example: /v1/account/container'
|
||||
print ' Or: /v1/account/container/object_prefix'
|
||||
print('<path> must point to a container at least.')
|
||||
print('For example: /v1/account/container')
|
||||
print(' Or: /v1/account/container/object_prefix')
|
||||
return 1
|
||||
sig = hmac.new(key, '%s\n%s\n%s\n%s\n%s' % (path, redirect, max_file_size,
|
||||
max_file_count, expires),
|
||||
sha1).hexdigest()
|
||||
print ' Expires:', expires
|
||||
print 'Signature:', sig
|
||||
print ''
|
||||
print(' Expires:', expires)
|
||||
print('Signature:', sig)
|
||||
print('')
|
||||
|
||||
print('Sample form:\n')
|
||||
|
||||
|
|
|
@ -10,6 +10,7 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
import itertools
|
||||
import os
|
||||
import sqlite3
|
||||
|
@ -84,17 +85,17 @@ def print_ring_locations(ring, datadir, account, container=None, obj=None,
|
|||
path_hash = hash_path(account, container, obj)
|
||||
else:
|
||||
path_hash = None
|
||||
print 'Partition\t%s' % part
|
||||
print 'Hash \t%s\n' % path_hash
|
||||
print('Partition\t%s' % part)
|
||||
print('Hash \t%s\n' % path_hash)
|
||||
|
||||
for node in primary_nodes:
|
||||
print 'Server:Port Device\t%s:%s %s' % (node['ip'], node['port'],
|
||||
node['device'])
|
||||
print('Server:Port Device\t%s:%s %s' % (node['ip'], node['port'],
|
||||
node['device']))
|
||||
for node in handoff_nodes:
|
||||
print 'Server:Port Device\t%s:%s %s\t [Handoff]' % (
|
||||
node['ip'], node['port'], node['device'])
|
||||
print('Server:Port Device\t%s:%s %s\t [Handoff]' % (
|
||||
node['ip'], node['port'], node['device']))
|
||||
|
||||
print "\n"
|
||||
print("\n")
|
||||
|
||||
for node in primary_nodes:
|
||||
cmd = 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' \
|
||||
|
@ -103,7 +104,7 @@ def print_ring_locations(ring, datadir, account, container=None, obj=None,
|
|||
if policy_index is not None:
|
||||
cmd += ' -H "%s: %s"' % ('X-Backend-Storage-Policy-Index',
|
||||
policy_index)
|
||||
print cmd
|
||||
print(cmd)
|
||||
for node in handoff_nodes:
|
||||
cmd = 'curl -I -XHEAD "http://%s:%s/%s/%s/%s"' \
|
||||
% (node['ip'], node['port'], node['device'], part,
|
||||
|
@ -112,30 +113,30 @@ def print_ring_locations(ring, datadir, account, container=None, obj=None,
|
|||
cmd += ' -H "%s: %s"' % ('X-Backend-Storage-Policy-Index',
|
||||
policy_index)
|
||||
cmd += ' # [Handoff]'
|
||||
print cmd
|
||||
print(cmd)
|
||||
|
||||
print "\n\nUse your own device location of servers:"
|
||||
print "such as \"export DEVICE=/srv/node\""
|
||||
print("\n\nUse your own device location of servers:")
|
||||
print("such as \"export DEVICE=/srv/node\"")
|
||||
if path_hash:
|
||||
for node in primary_nodes:
|
||||
print ('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s"' %
|
||||
(node['ip'], node['device'],
|
||||
storage_directory(datadir, part, path_hash)))
|
||||
print('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s"' %
|
||||
(node['ip'], node['device'],
|
||||
storage_directory(datadir, part, path_hash)))
|
||||
for node in handoff_nodes:
|
||||
print ('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s" # [Handoff]' %
|
||||
(node['ip'], node['device'],
|
||||
storage_directory(datadir, part, path_hash)))
|
||||
print('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s" # [Handoff]' %
|
||||
(node['ip'], node['device'],
|
||||
storage_directory(datadir, part, path_hash)))
|
||||
else:
|
||||
for node in primary_nodes:
|
||||
print ('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s/%d"' %
|
||||
(node['ip'], node['device'], datadir, part))
|
||||
print('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s/%d"' %
|
||||
(node['ip'], node['device'], datadir, part))
|
||||
for node in handoff_nodes:
|
||||
print ('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s/%d"'
|
||||
' # [Handoff]' %
|
||||
(node['ip'], node['device'], datadir, part))
|
||||
print('ssh %s "ls -lah ${DEVICE:-/srv/node*}/%s/%s/%d"'
|
||||
' # [Handoff]' %
|
||||
(node['ip'], node['device'], datadir, part))
|
||||
|
||||
print '\nnote: `/srv/node*` is used as default value of `devices`, the ' \
|
||||
'real value is set in the config file on each storage node.'
|
||||
print('\nnote: `/srv/node*` is used as default value of `devices`, the '
|
||||
'real value is set in the config file on each storage node.')
|
||||
|
||||
|
||||
def print_db_info_metadata(db_type, info, metadata):
|
||||
|
@ -162,52 +163,53 @@ def print_db_info_metadata(db_type, info, metadata):
|
|||
else:
|
||||
path = '/%s' % account
|
||||
|
||||
print 'Path: %s' % path
|
||||
print ' Account: %s' % account
|
||||
print('Path: %s' % path)
|
||||
print(' Account: %s' % account)
|
||||
|
||||
if db_type == 'container':
|
||||
print ' Container: %s' % container
|
||||
print(' Container: %s' % container)
|
||||
|
||||
path_hash = hash_path(account, container)
|
||||
if db_type == 'container':
|
||||
print ' Container Hash: %s' % path_hash
|
||||
print(' Container Hash: %s' % path_hash)
|
||||
else:
|
||||
print ' Account Hash: %s' % path_hash
|
||||
print(' Account Hash: %s' % path_hash)
|
||||
|
||||
print 'Metadata:'
|
||||
print (' Created at: %s (%s)' %
|
||||
(Timestamp(info['created_at']).isoformat,
|
||||
info['created_at']))
|
||||
print (' Put Timestamp: %s (%s)' %
|
||||
(Timestamp(info['put_timestamp']).isoformat,
|
||||
info['put_timestamp']))
|
||||
print (' Delete Timestamp: %s (%s)' %
|
||||
(Timestamp(info['delete_timestamp']).isoformat,
|
||||
info['delete_timestamp']))
|
||||
print (' Status Timestamp: %s (%s)' %
|
||||
(Timestamp(info['status_changed_at']).isoformat,
|
||||
info['status_changed_at']))
|
||||
print('Metadata:')
|
||||
print(' Created at: %s (%s)' %
|
||||
(Timestamp(info['created_at']).isoformat,
|
||||
info['created_at']))
|
||||
print(' Put Timestamp: %s (%s)' %
|
||||
(Timestamp(info['put_timestamp']).isoformat,
|
||||
info['put_timestamp']))
|
||||
print(' Delete Timestamp: %s (%s)' %
|
||||
(Timestamp(info['delete_timestamp']).isoformat,
|
||||
info['delete_timestamp']))
|
||||
print(' Status Timestamp: %s (%s)' %
|
||||
(Timestamp(info['status_changed_at']).isoformat,
|
||||
info['status_changed_at']))
|
||||
if db_type == 'account':
|
||||
print ' Container Count: %s' % info['container_count']
|
||||
print ' Object Count: %s' % info['object_count']
|
||||
print ' Bytes Used: %s' % info['bytes_used']
|
||||
print(' Container Count: %s' % info['container_count'])
|
||||
print(' Object Count: %s' % info['object_count'])
|
||||
print(' Bytes Used: %s' % info['bytes_used'])
|
||||
if db_type == 'container':
|
||||
try:
|
||||
policy_name = POLICIES[info['storage_policy_index']].name
|
||||
except KeyError:
|
||||
policy_name = 'Unknown'
|
||||
print (' Storage Policy: %s (%s)' % (
|
||||
print(' Storage Policy: %s (%s)' % (
|
||||
policy_name, info['storage_policy_index']))
|
||||
print (' Reported Put Timestamp: %s (%s)' %
|
||||
(Timestamp(info['reported_put_timestamp']).isoformat,
|
||||
info['reported_put_timestamp']))
|
||||
print (' Reported Delete Timestamp: %s (%s)' %
|
||||
(Timestamp(info['reported_delete_timestamp']).isoformat,
|
||||
info['reported_delete_timestamp']))
|
||||
print ' Reported Object Count: %s' % info['reported_object_count']
|
||||
print ' Reported Bytes Used: %s' % info['reported_bytes_used']
|
||||
print ' Chexor: %s' % info['hash']
|
||||
print ' UUID: %s' % info['id']
|
||||
print(' Reported Put Timestamp: %s (%s)' %
|
||||
(Timestamp(info['reported_put_timestamp']).isoformat,
|
||||
info['reported_put_timestamp']))
|
||||
print(' Reported Delete Timestamp: %s (%s)' %
|
||||
(Timestamp(info['reported_delete_timestamp']).isoformat,
|
||||
info['reported_delete_timestamp']))
|
||||
print(' Reported Object Count: %s' %
|
||||
info['reported_object_count'])
|
||||
print(' Reported Bytes Used: %s' % info['reported_bytes_used'])
|
||||
print(' Chexor: %s' % info['hash'])
|
||||
print(' UUID: %s' % info['id'])
|
||||
except KeyError as e:
|
||||
raise ValueError('Info is incomplete: %s' % e)
|
||||
|
||||
|
@ -215,7 +217,7 @@ def print_db_info_metadata(db_type, info, metadata):
|
|||
for key, value in info.items():
|
||||
if key.lower().startswith(meta_prefix):
|
||||
title = key.replace('_', '-').title()
|
||||
print ' %s: %s' % (title, value)
|
||||
print(' %s: %s' % (title, value))
|
||||
user_metadata = {}
|
||||
sys_metadata = {}
|
||||
for key, (value, timestamp) in metadata.items():
|
||||
|
@ -225,16 +227,16 @@ def print_db_info_metadata(db_type, info, metadata):
|
|||
sys_metadata[strip_sys_meta_prefix(db_type, key)] = value
|
||||
else:
|
||||
title = key.replace('_', '-').title()
|
||||
print ' %s: %s' % (title, value)
|
||||
print(' %s: %s' % (title, value))
|
||||
if sys_metadata:
|
||||
print ' System Metadata: %s' % sys_metadata
|
||||
print(' System Metadata: %s' % sys_metadata)
|
||||
else:
|
||||
print 'No system metadata found in db file'
|
||||
print('No system metadata found in db file')
|
||||
|
||||
if user_metadata:
|
||||
print ' User Metadata: %s' % user_metadata
|
||||
print(' User Metadata: %s' % user_metadata)
|
||||
else:
|
||||
print 'No user metadata found in db file'
|
||||
print('No user metadata found in db file')
|
||||
|
||||
|
||||
def print_obj_metadata(metadata):
|
||||
|
@ -268,21 +270,21 @@ def print_obj_metadata(metadata):
|
|||
raise ValueError('Path is invalid for object %r' % path)
|
||||
else:
|
||||
obj_hash = hash_path(account, container, obj)
|
||||
print 'Path: %s' % path
|
||||
print ' Account: %s' % account
|
||||
print ' Container: %s' % container
|
||||
print ' Object: %s' % obj
|
||||
print ' Object hash: %s' % obj_hash
|
||||
print('Path: %s' % path)
|
||||
print(' Account: %s' % account)
|
||||
print(' Container: %s' % container)
|
||||
print(' Object: %s' % obj)
|
||||
print(' Object hash: %s' % obj_hash)
|
||||
else:
|
||||
print 'Path: Not found in metadata'
|
||||
print('Path: Not found in metadata')
|
||||
if content_type:
|
||||
print 'Content-Type: %s' % content_type
|
||||
print('Content-Type: %s' % content_type)
|
||||
else:
|
||||
print 'Content-Type: Not found in metadata'
|
||||
print('Content-Type: Not found in metadata')
|
||||
if ts:
|
||||
print ('Timestamp: %s (%s)' % (ts.isoformat, ts.internal))
|
||||
print('Timestamp: %s (%s)' % (ts.isoformat, ts.internal))
|
||||
else:
|
||||
print 'Timestamp: Not found in metadata'
|
||||
print('Timestamp: Not found in metadata')
|
||||
|
||||
for key, value in metadata.items():
|
||||
if is_user_meta('Object', key):
|
||||
|
@ -293,12 +295,12 @@ def print_obj_metadata(metadata):
|
|||
other_metadata[key] = value
|
||||
|
||||
def print_metadata(title, items):
|
||||
print title
|
||||
print(title)
|
||||
if items:
|
||||
for meta_key in sorted(items):
|
||||
print ' %s: %s' % (meta_key, items[meta_key])
|
||||
print(' %s: %s' % (meta_key, items[meta_key]))
|
||||
else:
|
||||
print ' No metadata found'
|
||||
print(' No metadata found')
|
||||
|
||||
print_metadata('System Metadata:', sys_metadata)
|
||||
print_metadata('User Metadata:', user_metadata)
|
||||
|
@ -307,10 +309,10 @@ def print_obj_metadata(metadata):
|
|||
|
||||
def print_info(db_type, db_file, swift_dir='/etc/swift'):
|
||||
if db_type not in ('account', 'container'):
|
||||
print "Unrecognized DB type: internal error"
|
||||
print("Unrecognized DB type: internal error")
|
||||
raise InfoSystemExit()
|
||||
if not os.path.exists(db_file) or not db_file.endswith('.db'):
|
||||
print "DB file doesn't exist"
|
||||
print("DB file doesn't exist")
|
||||
raise InfoSystemExit()
|
||||
if not db_file.startswith(('/', './')):
|
||||
db_file = './' + db_file # don't break if the bare db file is given
|
||||
|
@ -324,8 +326,8 @@ def print_info(db_type, db_file, swift_dir='/etc/swift'):
|
|||
info = broker.get_info()
|
||||
except sqlite3.OperationalError as err:
|
||||
if 'no such table' in str(err):
|
||||
print "Does not appear to be a DB of type \"%s\": %s" % (
|
||||
db_type, db_file)
|
||||
print("Does not appear to be a DB of type \"%s\": %s"
|
||||
% (db_type, db_file))
|
||||
raise InfoSystemExit()
|
||||
raise
|
||||
account = info['account']
|
||||
|
@ -353,7 +355,7 @@ def print_obj(datafile, check_etag=True, swift_dir='/etc/swift',
|
|||
:param policy_name: optionally the name to use when finding the ring
|
||||
"""
|
||||
if not os.path.exists(datafile):
|
||||
print "Data file doesn't exist"
|
||||
print("Data file doesn't exist")
|
||||
raise InfoSystemExit()
|
||||
if not datafile.startswith(('/', './')):
|
||||
datafile = './' + datafile
|
||||
|
@ -363,7 +365,8 @@ def print_obj(datafile, check_etag=True, swift_dir='/etc/swift',
|
|||
datadir = DATADIR_BASE
|
||||
|
||||
# try to extract policy index from datafile disk path
|
||||
policy_index = int(extract_policy(datafile) or POLICIES.legacy)
|
||||
fullpath = os.path.abspath(datafile)
|
||||
policy_index = int(extract_policy(fullpath) or POLICIES.legacy)
|
||||
|
||||
try:
|
||||
if policy_index:
|
||||
|
@ -382,8 +385,8 @@ def print_obj(datafile, check_etag=True, swift_dir='/etc/swift',
|
|||
if (policy_index is not None and
|
||||
policy_index_for_name is not None and
|
||||
policy_index != policy_index_for_name):
|
||||
print 'Warning: Ring does not match policy!'
|
||||
print 'Double check your policy name!'
|
||||
print('Warning: Ring does not match policy!')
|
||||
print('Double check your policy name!')
|
||||
if not ring and policy_index_for_name:
|
||||
ring = POLICIES.get_object_ring(policy_index_for_name,
|
||||
swift_dir)
|
||||
|
@ -393,7 +396,7 @@ def print_obj(datafile, check_etag=True, swift_dir='/etc/swift',
|
|||
try:
|
||||
metadata = read_metadata(fp)
|
||||
except EOFError:
|
||||
print "Invalid metadata"
|
||||
print("Invalid metadata")
|
||||
raise InfoSystemExit()
|
||||
|
||||
etag = metadata.pop('ETag', '')
|
||||
|
@ -415,24 +418,24 @@ def print_obj(datafile, check_etag=True, swift_dir='/etc/swift',
|
|||
h = h.hexdigest()
|
||||
if etag:
|
||||
if h == etag:
|
||||
print 'ETag: %s (valid)' % etag
|
||||
print('ETag: %s (valid)' % etag)
|
||||
else:
|
||||
print ("ETag: %s doesn't match file hash of %s!" %
|
||||
(etag, h))
|
||||
print("ETag: %s doesn't match file hash of %s!" %
|
||||
(etag, h))
|
||||
else:
|
||||
print 'ETag: Not found in metadata'
|
||||
print('ETag: Not found in metadata')
|
||||
else:
|
||||
print 'ETag: %s (not checked)' % etag
|
||||
print('ETag: %s (not checked)' % etag)
|
||||
file_len = os.fstat(fp.fileno()).st_size
|
||||
|
||||
if length:
|
||||
if file_len == int(length):
|
||||
print 'Content-Length: %s (valid)' % length
|
||||
print('Content-Length: %s (valid)' % length)
|
||||
else:
|
||||
print ("Content-Length: %s doesn't match file length of %s"
|
||||
% (length, file_len))
|
||||
print("Content-Length: %s doesn't match file length of %s"
|
||||
% (length, file_len))
|
||||
else:
|
||||
print 'Content-Length: Not found in metadata'
|
||||
print('Content-Length: Not found in metadata')
|
||||
|
||||
account, container, obj = path.split('/', 3)[1:]
|
||||
if ring:
|
||||
|
@ -472,33 +475,33 @@ def print_item_locations(ring, ring_name=None, account=None, container=None,
|
|||
policy = POLICIES.get_by_name(policy_name)
|
||||
if policy:
|
||||
if ring_name != policy.ring_name:
|
||||
print 'Warning: mismatch between ring and policy name!'
|
||||
print('Warning: mismatch between ring and policy name!')
|
||||
else:
|
||||
print 'Warning: Policy %s is not valid' % policy_name
|
||||
print('Warning: Policy %s is not valid' % policy_name)
|
||||
|
||||
policy_index = None
|
||||
if ring is None and (obj or part):
|
||||
if not policy_name:
|
||||
print 'Need a ring or policy'
|
||||
print('Need a ring or policy')
|
||||
raise InfoSystemExit()
|
||||
policy = POLICIES.get_by_name(policy_name)
|
||||
if not policy:
|
||||
print 'No policy named %r' % policy_name
|
||||
print('No policy named %r' % policy_name)
|
||||
raise InfoSystemExit()
|
||||
policy_index = int(policy)
|
||||
ring = POLICIES.get_object_ring(policy_index, swift_dir)
|
||||
ring_name = (POLICIES.get_by_name(policy_name)).ring_name
|
||||
|
||||
if account is None and (container is not None or obj is not None):
|
||||
print 'No account specified'
|
||||
print('No account specified')
|
||||
raise InfoSystemExit()
|
||||
|
||||
if container is None and obj is not None:
|
||||
print 'No container specified'
|
||||
print('No container specified')
|
||||
raise InfoSystemExit()
|
||||
|
||||
if account is None and part is None:
|
||||
print 'No target specified'
|
||||
print('No target specified')
|
||||
raise InfoSystemExit()
|
||||
|
||||
loc = '<type>'
|
||||
|
@ -518,19 +521,19 @@ def print_item_locations(ring, ring_name=None, account=None, container=None,
|
|||
ring = Ring(swift_dir, ring_name='container')
|
||||
else:
|
||||
if ring_name != 'container':
|
||||
print 'Warning: account/container specified ' + \
|
||||
'but ring not named "container"'
|
||||
print('Warning: account/container specified ' +
|
||||
'but ring not named "container"')
|
||||
if account and not container and not obj:
|
||||
loc = 'accounts'
|
||||
if not any([ring, ring_name]):
|
||||
ring = Ring(swift_dir, ring_name='account')
|
||||
else:
|
||||
if ring_name != 'account':
|
||||
print 'Warning: account specified ' + \
|
||||
'but ring not named "account"'
|
||||
print('Warning: account specified ' +
|
||||
'but ring not named "account"')
|
||||
|
||||
print '\nAccount \t%s' % account
|
||||
print 'Container\t%s' % container
|
||||
print 'Object \t%s\n\n' % obj
|
||||
print('\nAccount \t%s' % account)
|
||||
print('Container\t%s' % container)
|
||||
print('Object \t%s\n\n' % obj)
|
||||
print_ring_locations(ring, loc, account, container, obj, part, all_nodes,
|
||||
policy_index=policy_index)
|
||||
|
|
|
@ -51,7 +51,7 @@ def size_suffix(size):
|
|||
for suffix in suffixes:
|
||||
if size < 1000:
|
||||
return "%s %s" % (size, suffix)
|
||||
size = size / 1000
|
||||
size = size // 1000
|
||||
return "%s %s" % (size, suffix)
|
||||
|
||||
|
||||
|
@ -100,11 +100,14 @@ class Scout(object):
|
|||
Obtain telemetry from a host running the swift recon middleware.
|
||||
|
||||
:param host: host to check
|
||||
:returns: tuple of (recon url used, response body, and status)
|
||||
:returns: tuple of (recon url used, response body, status, time start
|
||||
and time end)
|
||||
"""
|
||||
base_url = "http://%s:%s/recon/" % (host[0], host[1])
|
||||
ts_start = time.time()
|
||||
url, content, status = self.scout_host(base_url, self.recon_type)
|
||||
return url, content, status
|
||||
ts_end = time.time()
|
||||
return url, content, status, ts_start, ts_end
|
||||
|
||||
def scout_server_type(self, host):
|
||||
"""
|
||||
|
@ -253,7 +256,8 @@ class SwiftRecon(object):
|
|||
if self.verbose:
|
||||
for ring_file, ring_sum in rings.items():
|
||||
print("-> On disk %s md5sum: %s" % (ring_file, ring_sum))
|
||||
for url, response, status in self.pool.imap(recon.scout, hosts):
|
||||
for url, response, status, ts_start, ts_end in self.pool.imap(
|
||||
recon.scout, hosts):
|
||||
if status != 200:
|
||||
errors = errors + 1
|
||||
continue
|
||||
|
@ -291,7 +295,8 @@ class SwiftRecon(object):
|
|||
printfn("[%s] Checking swift.conf md5sum" % self._ptime())
|
||||
if self.verbose:
|
||||
printfn("-> On disk swift.conf md5sum: %s" % (conf_sum,))
|
||||
for url, response, status in self.pool.imap(recon.scout, hosts):
|
||||
for url, response, status, ts_start, ts_end in self.pool.imap(
|
||||
recon.scout, hosts):
|
||||
if status == 200:
|
||||
if response[SWIFT_CONF_FILE] != conf_sum:
|
||||
printfn("!! %s (%s) doesn't match on disk md5sum" %
|
||||
|
@ -317,7 +322,8 @@ class SwiftRecon(object):
|
|||
recon = Scout("async", self.verbose, self.suppress_errors,
|
||||
self.timeout)
|
||||
print("[%s] Checking async pendings" % self._ptime())
|
||||
for url, response, status in self.pool.imap(recon.scout, hosts):
|
||||
for url, response, status, ts_start, ts_end in self.pool.imap(
|
||||
recon.scout, hosts):
|
||||
if status == 200:
|
||||
scan[url] = response['async_pending']
|
||||
stats = self._gen_stats(scan.values(), 'async_pending')
|
||||
|
@ -338,7 +344,8 @@ class SwiftRecon(object):
|
|||
recon = Scout("driveaudit", self.verbose, self.suppress_errors,
|
||||
self.timeout)
|
||||
print("[%s] Checking drive-audit errors" % self._ptime())
|
||||
for url, response, status in self.pool.imap(recon.scout, hosts):
|
||||
for url, response, status, ts_start, ts_end in self.pool.imap(
|
||||
recon.scout, hosts):
|
||||
if status == 200:
|
||||
scan[url] = response['drive_audit_errors']
|
||||
stats = self._gen_stats(scan.values(), 'drive_audit_errors')
|
||||
|
@ -361,7 +368,8 @@ class SwiftRecon(object):
|
|||
self.timeout)
|
||||
print("[%s] Getting unmounted drives from %s hosts..." %
|
||||
(self._ptime(), len(hosts)))
|
||||
for url, response, status in self.pool.imap(recon.scout, hosts):
|
||||
for url, response, status, ts_start, ts_end in self.pool.imap(
|
||||
recon.scout, hosts):
|
||||
if status == 200:
|
||||
unmounted[url] = []
|
||||
errors[url] = []
|
||||
|
@ -414,7 +422,8 @@ class SwiftRecon(object):
|
|||
recon = Scout("expirer/%s" % self.server_type, self.verbose,
|
||||
self.suppress_errors, self.timeout)
|
||||
print("[%s] Checking on expirers" % self._ptime())
|
||||
for url, response, status in self.pool.imap(recon.scout, hosts):
|
||||
for url, response, status, ts_start, ts_end in self.pool.imap(
|
||||
recon.scout, hosts):
|
||||
if status == 200:
|
||||
stats['object_expiration_pass'].append(
|
||||
response.get('object_expiration_pass'))
|
||||
|
@ -447,15 +456,18 @@ class SwiftRecon(object):
|
|||
least_recent_url = None
|
||||
most_recent_time = 0
|
||||
most_recent_url = None
|
||||
for url, response, status in self.pool.imap(recon.scout, hosts):
|
||||
for url, response, status, ts_start, ts_end in self.pool.imap(
|
||||
recon.scout, hosts):
|
||||
if status == 200:
|
||||
stats['replication_time'].append(
|
||||
response.get('replication_time'))
|
||||
repl_stats = response['replication_stats']
|
||||
response.get('replication_time',
|
||||
response.get('object_replication_time', 0)))
|
||||
repl_stats = response.get('replication_stats')
|
||||
if repl_stats:
|
||||
for stat_key in ['attempted', 'failure', 'success']:
|
||||
stats[stat_key].append(repl_stats.get(stat_key))
|
||||
last = response.get('replication_last', 0)
|
||||
last = response.get('replication_last',
|
||||
response.get('object_replication_last', 0))
|
||||
if last < least_recent_time:
|
||||
least_recent_time = last
|
||||
least_recent_url = url
|
||||
|
@ -496,61 +508,6 @@ class SwiftRecon(object):
|
|||
elapsed, elapsed_unit, host))
|
||||
print("=" * 79)
|
||||
|
||||
def object_replication_check(self, hosts):
|
||||
"""
|
||||
Obtain and print replication statistics from object servers
|
||||
|
||||
:param hosts: set of hosts to check. in the format of:
|
||||
set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
|
||||
"""
|
||||
stats = {}
|
||||
recon = Scout("replication", self.verbose, self.suppress_errors,
|
||||
self.timeout)
|
||||
print("[%s] Checking on replication" % self._ptime())
|
||||
least_recent_time = 9999999999
|
||||
least_recent_url = None
|
||||
most_recent_time = 0
|
||||
most_recent_url = None
|
||||
for url, response, status in self.pool.imap(recon.scout, hosts):
|
||||
if status == 200:
|
||||
stats[url] = response['object_replication_time']
|
||||
last = response.get('object_replication_last', 0)
|
||||
if last < least_recent_time:
|
||||
least_recent_time = last
|
||||
least_recent_url = url
|
||||
if last > most_recent_time:
|
||||
most_recent_time = last
|
||||
most_recent_url = url
|
||||
times = [x for x in stats.values() if x is not None]
|
||||
if len(stats) > 0 and len(times) > 0:
|
||||
computed = self._gen_stats(times, 'replication_time')
|
||||
if computed['reported'] > 0:
|
||||
self._print_stats(computed)
|
||||
else:
|
||||
print("[replication_time] - No hosts returned valid data.")
|
||||
else:
|
||||
print("[replication_time] - No hosts returned valid data.")
|
||||
if least_recent_url is not None:
|
||||
host = urlparse(least_recent_url).netloc
|
||||
if not least_recent_time:
|
||||
print('Oldest completion was NEVER by %s.' % host)
|
||||
else:
|
||||
elapsed = time.time() - least_recent_time
|
||||
elapsed, elapsed_unit = seconds2timeunit(elapsed)
|
||||
print('Oldest completion was %s (%d %s ago) by %s.' % (
|
||||
time.strftime('%Y-%m-%d %H:%M:%S',
|
||||
time.gmtime(least_recent_time)),
|
||||
elapsed, elapsed_unit, host))
|
||||
if most_recent_url is not None:
|
||||
host = urlparse(most_recent_url).netloc
|
||||
elapsed = time.time() - most_recent_time
|
||||
elapsed, elapsed_unit = seconds2timeunit(elapsed)
|
||||
print('Most recent completion was %s (%d %s ago) by %s.' % (
|
||||
time.strftime('%Y-%m-%d %H:%M:%S',
|
||||
time.gmtime(most_recent_time)),
|
||||
elapsed, elapsed_unit, host))
|
||||
print("=" * 79)
|
||||
|
||||
def updater_check(self, hosts):
|
||||
"""
|
||||
Obtain and print updater statistics
|
||||
|
@ -562,7 +519,8 @@ class SwiftRecon(object):
|
|||
recon = Scout("updater/%s" % self.server_type, self.verbose,
|
||||
self.suppress_errors, self.timeout)
|
||||
print("[%s] Checking updater times" % self._ptime())
|
||||
for url, response, status in self.pool.imap(recon.scout, hosts):
|
||||
for url, response, status, ts_start, ts_end in self.pool.imap(
|
||||
recon.scout, hosts):
|
||||
if status == 200:
|
||||
if response['%s_updater_sweep' % self.server_type]:
|
||||
stats.append(response['%s_updater_sweep' %
|
||||
|
@ -592,7 +550,8 @@ class SwiftRecon(object):
|
|||
recon = Scout("auditor/%s" % self.server_type, self.verbose,
|
||||
self.suppress_errors, self.timeout)
|
||||
print("[%s] Checking auditor stats" % self._ptime())
|
||||
for url, response, status in self.pool.imap(recon.scout, hosts):
|
||||
for url, response, status, ts_start, ts_end in self.pool.imap(
|
||||
recon.scout, hosts):
|
||||
if status == 200:
|
||||
scan[url] = response
|
||||
if len(scan) < 1:
|
||||
|
@ -665,7 +624,8 @@ class SwiftRecon(object):
|
|||
recon = Scout("auditor/object", self.verbose, self.suppress_errors,
|
||||
self.timeout)
|
||||
print("[%s] Checking auditor stats " % self._ptime())
|
||||
for url, response, status in self.pool.imap(recon.scout, hosts):
|
||||
for url, response, status, ts_start, ts_end in self.pool.imap(
|
||||
recon.scout, hosts):
|
||||
if status == 200:
|
||||
if response['object_auditor_stats_ALL']:
|
||||
all_scan[url] = response['object_auditor_stats_ALL']
|
||||
|
@ -736,7 +696,8 @@ class SwiftRecon(object):
|
|||
recon = Scout("load", self.verbose, self.suppress_errors,
|
||||
self.timeout)
|
||||
print("[%s] Checking load averages" % self._ptime())
|
||||
for url, response, status in self.pool.imap(recon.scout, hosts):
|
||||
for url, response, status, ts_start, ts_end in self.pool.imap(
|
||||
recon.scout, hosts):
|
||||
if status == 200:
|
||||
load1[url] = response['1m']
|
||||
load5[url] = response['5m']
|
||||
|
@ -765,7 +726,8 @@ class SwiftRecon(object):
|
|||
recon = Scout("quarantined", self.verbose, self.suppress_errors,
|
||||
self.timeout)
|
||||
print("[%s] Checking quarantine" % self._ptime())
|
||||
for url, response, status in self.pool.imap(recon.scout, hosts):
|
||||
for url, response, status, ts_start, ts_end in self.pool.imap(
|
||||
recon.scout, hosts):
|
||||
if status == 200:
|
||||
objq[url] = response['objects']
|
||||
conq[url] = response['containers']
|
||||
|
@ -799,7 +761,8 @@ class SwiftRecon(object):
|
|||
recon = Scout("sockstat", self.verbose, self.suppress_errors,
|
||||
self.timeout)
|
||||
print("[%s] Checking socket usage" % self._ptime())
|
||||
for url, response, status in self.pool.imap(recon.scout, hosts):
|
||||
for url, response, status, ts_start, ts_end in self.pool.imap(
|
||||
recon.scout, hosts):
|
||||
if status == 200:
|
||||
inuse4[url] = response['tcp_in_use']
|
||||
mem[url] = response['tcp_mem_allocated_bytes']
|
||||
|
@ -835,7 +798,8 @@ class SwiftRecon(object):
|
|||
recon = Scout("diskusage", self.verbose, self.suppress_errors,
|
||||
self.timeout)
|
||||
print("[%s] Checking disk usage now" % self._ptime())
|
||||
for url, response, status in self.pool.imap(recon.scout, hosts):
|
||||
for url, response, status, ts_start, ts_end in self.pool.imap(
|
||||
recon.scout, hosts):
|
||||
if status == 200:
|
||||
hostusage = []
|
||||
for entry in response:
|
||||
|
@ -915,6 +879,47 @@ class SwiftRecon(object):
|
|||
host = urlparse(url).netloc.split(':')[0]
|
||||
print('%.02f%% %s' % (used, '%-15s %s' % (host, device)))
|
||||
|
||||
def time_check(self, hosts):
|
||||
"""
|
||||
Check a time synchronization of hosts with current time
|
||||
|
||||
:param hosts: set of hosts to check. in the format of:
|
||||
set([('127.0.0.1', 6020), ('127.0.0.2', 6030)])
|
||||
"""
|
||||
|
||||
matches = 0
|
||||
errors = 0
|
||||
recon = Scout("time", self.verbose, self.suppress_errors,
|
||||
self.timeout)
|
||||
print("[%s] Checking time-sync" % self._ptime())
|
||||
for url, ts_remote, status, ts_start, ts_end in self.pool.imap(
|
||||
recon.scout, hosts):
|
||||
if status != 200:
|
||||
errors = errors + 1
|
||||
continue
|
||||
if (ts_remote < ts_start or ts_remote > ts_end):
|
||||
diff = abs(ts_end - ts_remote)
|
||||
ts_end_f = time.strftime(
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
time.localtime(ts_end))
|
||||
ts_remote_f = time.strftime(
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
time.localtime(ts_remote))
|
||||
|
||||
print("!! %s current time is %s, but remote is %s, "
|
||||
"differs by %.2f sec" % (
|
||||
url,
|
||||
ts_end_f,
|
||||
ts_remote_f,
|
||||
diff))
|
||||
continue
|
||||
matches += 1
|
||||
if self.verbose:
|
||||
print("-> %s matches." % url)
|
||||
print("%s/%s hosts matched, %s error[s] while checking hosts." % (
|
||||
matches, len(hosts), errors))
|
||||
print("=" * 79)
|
||||
|
||||
def main(self):
|
||||
"""
|
||||
Retrieve and report cluster info from hosts running recon middleware.
|
||||
|
@ -922,7 +927,7 @@ class SwiftRecon(object):
|
|||
print("=" * 79)
|
||||
usage = '''
|
||||
usage: %prog <server_type> [-v] [--suppress] [-a] [-r] [-u] [-d]
|
||||
[-l] [--md5] [--auditor] [--updater] [--expirer] [--sockstat]
|
||||
[-l] [-T] [--md5] [--auditor] [--updater] [--expirer] [--sockstat]
|
||||
[--human-readable]
|
||||
|
||||
<server_type>\taccount|container|object
|
||||
|
@ -964,13 +969,15 @@ class SwiftRecon(object):
|
|||
help="Get cluster socket usage stats")
|
||||
args.add_option('--driveaudit', action="store_true",
|
||||
help="Get drive audit error stats")
|
||||
args.add_option('--time', '-T', action="store_true",
|
||||
help="Check time synchronization")
|
||||
args.add_option('--top', type='int', metavar='COUNT', default=0,
|
||||
help='Also show the top COUNT entries in rank order.')
|
||||
args.add_option('--lowest', type='int', metavar='COUNT', default=0,
|
||||
help='Also show the lowest COUNT entries in rank \
|
||||
order.')
|
||||
args.add_option('--all', action="store_true",
|
||||
help="Perform all checks. Equal to \t\t\t-arudlq "
|
||||
help="Perform all checks. Equal to \t\t\t-arudlqT "
|
||||
"--md5 --sockstat --auditor --updater --expirer")
|
||||
args.add_option('--region', type="int",
|
||||
help="Only query servers in specified region")
|
||||
|
@ -1011,7 +1018,7 @@ class SwiftRecon(object):
|
|||
if options.all:
|
||||
if self.server_type == 'object':
|
||||
self.async_check(hosts)
|
||||
self.object_replication_check(hosts)
|
||||
self.replication_check(hosts)
|
||||
self.object_auditor_check(hosts)
|
||||
self.updater_check(hosts)
|
||||
self.expirer_check(hosts)
|
||||
|
@ -1031,6 +1038,7 @@ class SwiftRecon(object):
|
|||
self.socket_usage(hosts)
|
||||
self.server_type_check(hosts)
|
||||
self.driveaudit_check(hosts)
|
||||
self.time_check(hosts)
|
||||
else:
|
||||
if options.async:
|
||||
if self.server_type == 'object':
|
||||
|
@ -1040,10 +1048,7 @@ class SwiftRecon(object):
|
|||
if options.unmounted:
|
||||
self.umount_check(hosts)
|
||||
if options.replication:
|
||||
if self.server_type == 'object':
|
||||
self.object_replication_check(hosts)
|
||||
else:
|
||||
self.replication_check(hosts)
|
||||
self.replication_check(hosts)
|
||||
if options.auditor:
|
||||
if self.server_type == 'object':
|
||||
self.object_auditor_check(hosts)
|
||||
|
@ -1075,6 +1080,8 @@ class SwiftRecon(object):
|
|||
self.socket_usage(hosts)
|
||||
if options.driveaudit:
|
||||
self.driveaudit_check(hosts)
|
||||
if options.time:
|
||||
self.time_check(hosts)
|
||||
|
||||
|
||||
def main():
|
||||
|
|
|
@ -96,26 +96,30 @@ ARG_PARSER.add_argument(
|
|||
help="Path to the scenario file")
|
||||
|
||||
|
||||
class ParseCommandError(ValueError):
|
||||
|
||||
def __init__(self, name, round_index, command_index, msg):
|
||||
msg = "Invalid %s (round %s, command %s): %s" % (
|
||||
name, round_index, command_index, msg)
|
||||
super(ParseCommandError, self).__init__(msg)
|
||||
|
||||
|
||||
def _parse_weight(round_index, command_index, weight_str):
|
||||
try:
|
||||
weight = float(weight_str)
|
||||
except ValueError as err:
|
||||
raise ValueError(
|
||||
"Invalid weight %r (round %d, command %d): %s"
|
||||
% (weight_str, round_index, command_index, err))
|
||||
raise ParseCommandError('weight', round_index, command_index, err)
|
||||
if weight < 0:
|
||||
raise ValueError(
|
||||
"Negative weight (round %d, command %d)"
|
||||
% (round_index, command_index))
|
||||
raise ParseCommandError('weight', round_index, command_index,
|
||||
'cannot be negative')
|
||||
return weight
|
||||
|
||||
|
||||
def _parse_add_command(round_index, command_index, command):
|
||||
if len(command) != 3:
|
||||
raise ValueError(
|
||||
"Invalid add command (round %d, command %d): expected array of "
|
||||
"length 3, but got %d"
|
||||
% (round_index, command_index, len(command)))
|
||||
raise ParseCommandError(
|
||||
'add command', round_index, command_index,
|
||||
'expected array of length 3, but got %r' % command)
|
||||
|
||||
dev_str = command[1]
|
||||
weight_str = command[2]
|
||||
|
@ -123,43 +127,47 @@ def _parse_add_command(round_index, command_index, command):
|
|||
try:
|
||||
dev = parse_add_value(dev_str)
|
||||
except ValueError as err:
|
||||
raise ValueError(
|
||||
"Invalid device specifier '%s' in add (round %d, command %d): %s"
|
||||
% (dev_str, round_index, command_index, err))
|
||||
raise ParseCommandError('device specifier', round_index,
|
||||
command_index, err)
|
||||
|
||||
dev['weight'] = _parse_weight(round_index, command_index, weight_str)
|
||||
|
||||
if dev['region'] is None:
|
||||
dev['region'] = 1
|
||||
|
||||
default_key_map = {
|
||||
'replication_ip': 'ip',
|
||||
'replication_port': 'port',
|
||||
}
|
||||
for empty_key, default_key in default_key_map.items():
|
||||
if dev[empty_key] is None:
|
||||
dev[empty_key] = dev[default_key]
|
||||
|
||||
return ['add', dev]
|
||||
|
||||
|
||||
def _parse_remove_command(round_index, command_index, command):
|
||||
if len(command) != 2:
|
||||
raise ValueError(
|
||||
"Invalid remove command (round %d, command %d): expected array of "
|
||||
"length 2, but got %d"
|
||||
% (round_index, command_index, len(command)))
|
||||
raise ParseCommandError('remove commnd', round_index, command_index,
|
||||
"expected array of length 2, but got %r" %
|
||||
(command,))
|
||||
|
||||
dev_str = command[1]
|
||||
|
||||
try:
|
||||
dev_id = int(dev_str)
|
||||
except ValueError as err:
|
||||
raise ValueError(
|
||||
"Invalid device ID '%s' in remove (round %d, command %d): %s"
|
||||
% (dev_str, round_index, command_index, err))
|
||||
raise ParseCommandError('device ID in remove',
|
||||
round_index, command_index, err)
|
||||
|
||||
return ['remove', dev_id]
|
||||
|
||||
|
||||
def _parse_set_weight_command(round_index, command_index, command):
|
||||
if len(command) != 3:
|
||||
raise ValueError(
|
||||
"Invalid remove command (round %d, command %d): expected array of "
|
||||
"length 3, but got %d"
|
||||
% (round_index, command_index, len(command)))
|
||||
raise ParseCommandError('remove command', round_index, command_index,
|
||||
"expected array of length 3, but got %r" %
|
||||
(command,))
|
||||
|
||||
dev_str = command[1]
|
||||
weight_str = command[2]
|
||||
|
@ -167,14 +175,21 @@ def _parse_set_weight_command(round_index, command_index, command):
|
|||
try:
|
||||
dev_id = int(dev_str)
|
||||
except ValueError as err:
|
||||
raise ValueError(
|
||||
"Invalid device ID '%s' in set_weight (round %d, command %d): %s"
|
||||
% (dev_str, round_index, command_index, err))
|
||||
raise ParseCommandError('device ID in set_weight',
|
||||
round_index, command_index, err)
|
||||
|
||||
weight = _parse_weight(round_index, command_index, weight_str)
|
||||
return ['set_weight', dev_id, weight]
|
||||
|
||||
|
||||
def _parse_save_command(round_index, command_index, command):
|
||||
if len(command) != 2:
|
||||
raise ParseCommandError(
|
||||
command, round_index, command_index,
|
||||
"expected array of length 2 but got %r" % (command,))
|
||||
return ['save', command[1]]
|
||||
|
||||
|
||||
def parse_scenario(scenario_data):
|
||||
"""
|
||||
Takes a serialized scenario and turns it into a data structure suitable
|
||||
|
@ -236,9 +251,12 @@ def parse_scenario(scenario_data):
|
|||
if not isinstance(raw_scenario['rounds'], list):
|
||||
raise ValueError("rounds must be an array")
|
||||
|
||||
parser_for_command = {'add': _parse_add_command,
|
||||
'remove': _parse_remove_command,
|
||||
'set_weight': _parse_set_weight_command}
|
||||
parser_for_command = {
|
||||
'add': _parse_add_command,
|
||||
'remove': _parse_remove_command,
|
||||
'set_weight': _parse_set_weight_command,
|
||||
'save': _parse_save_command,
|
||||
}
|
||||
|
||||
parsed_scenario['rounds'] = []
|
||||
for round_index, raw_round in enumerate(raw_scenario['rounds']):
|
||||
|
@ -268,18 +286,24 @@ def run_scenario(scenario):
|
|||
|
||||
rb = builder.RingBuilder(scenario['part_power'], scenario['replicas'], 1)
|
||||
rb.set_overload(scenario['overload'])
|
||||
|
||||
command_map = {
|
||||
'add': rb.add_dev,
|
||||
'remove': rb.remove_dev,
|
||||
'set_weight': rb.set_dev_weight,
|
||||
'save': rb.save,
|
||||
}
|
||||
|
||||
for round_index, commands in enumerate(scenario['rounds']):
|
||||
print "Round %d" % (round_index + 1)
|
||||
|
||||
for command in commands:
|
||||
if command[0] == 'add':
|
||||
rb.add_dev(command[1])
|
||||
elif command[0] == 'remove':
|
||||
rb.remove_dev(command[1])
|
||||
elif command[0] == 'set_weight':
|
||||
rb.set_dev_weight(command[1], command[2])
|
||||
else:
|
||||
raise ValueError("unknown command %r" % (command[0],))
|
||||
key = command.pop(0)
|
||||
try:
|
||||
command_f = command_map[key]
|
||||
except KeyError:
|
||||
raise ValueError("unknown command %r" % key)
|
||||
command_f(*command)
|
||||
|
||||
rebalance_number = 1
|
||||
parts_moved, old_balance = rb.rebalance(seed=seed)
|
||||
|
|
|
@ -14,6 +14,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
import logging
|
||||
|
||||
from errno import EEXIST
|
||||
|
@ -71,14 +72,14 @@ def _parse_search_values(argvish):
|
|||
search_values = {}
|
||||
if len(args) > 0:
|
||||
if new_cmd_format or len(args) != 1:
|
||||
print Commands.search.__doc__.strip()
|
||||
print(Commands.search.__doc__.strip())
|
||||
exit(EXIT_ERROR)
|
||||
search_values = parse_search_value(args[0])
|
||||
else:
|
||||
search_values = parse_search_values_from_opts(opts)
|
||||
return search_values
|
||||
except ValueError as e:
|
||||
print e
|
||||
print(e)
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
|
||||
|
@ -113,7 +114,7 @@ def _parse_list_parts_values(argvish):
|
|||
devs = []
|
||||
if len(args) > 0:
|
||||
if new_cmd_format:
|
||||
print Commands.list_parts.__doc__.strip()
|
||||
print(Commands.list_parts.__doc__.strip())
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
for arg in args:
|
||||
|
@ -125,7 +126,7 @@ def _parse_list_parts_values(argvish):
|
|||
|
||||
return devs
|
||||
except ValueError as e:
|
||||
print e
|
||||
print(e)
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
|
||||
|
@ -145,7 +146,7 @@ def _parse_add_values(argvish):
|
|||
parsed_devs = []
|
||||
if len(args) > 0:
|
||||
if new_cmd_format or len(args) % 2 != 0:
|
||||
print Commands.add.__doc__.strip()
|
||||
print(Commands.add.__doc__.strip())
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
devs_and_weights = izip(islice(args, 0, len(args), 2),
|
||||
|
@ -184,18 +185,18 @@ def _set_weight_values(devs, weight):
|
|||
exit(EXIT_ERROR)
|
||||
|
||||
if len(devs) > 1:
|
||||
print 'Matched more than one device:'
|
||||
print('Matched more than one device:')
|
||||
for dev in devs:
|
||||
print ' %s' % format_device(dev)
|
||||
print(' %s' % format_device(dev))
|
||||
if raw_input('Are you sure you want to update the weight for '
|
||||
'these %s devices? (y/N) ' % len(devs)) != 'y':
|
||||
print 'Aborting device modifications'
|
||||
print('Aborting device modifications')
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
for dev in devs:
|
||||
builder.set_dev_weight(dev['id'], weight)
|
||||
print '%s weight set to %s' % (format_device(dev),
|
||||
dev['weight'])
|
||||
print('%s weight set to %s' % (format_device(dev),
|
||||
dev['weight']))
|
||||
|
||||
|
||||
def _parse_set_weight_values(argvish):
|
||||
|
@ -209,7 +210,7 @@ def _parse_set_weight_values(argvish):
|
|||
devs = []
|
||||
if not new_cmd_format:
|
||||
if len(args) % 2 != 0:
|
||||
print Commands.set_weight.__doc__.strip()
|
||||
print(Commands.set_weight.__doc__.strip())
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
devs_and_weights = izip(islice(argvish, 0, len(argvish), 2),
|
||||
|
@ -221,7 +222,7 @@ def _parse_set_weight_values(argvish):
|
|||
_set_weight_values(devs, weight)
|
||||
else:
|
||||
if len(args) != 1:
|
||||
print Commands.set_weight.__doc__.strip()
|
||||
print(Commands.set_weight.__doc__.strip())
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
devs.extend(builder.search_devs(
|
||||
|
@ -229,7 +230,7 @@ def _parse_set_weight_values(argvish):
|
|||
weight = float(args[0])
|
||||
_set_weight_values(devs, weight)
|
||||
except ValueError as e:
|
||||
print e
|
||||
print(e)
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
|
||||
|
@ -241,12 +242,12 @@ def _set_info_values(devs, change):
|
|||
exit(EXIT_ERROR)
|
||||
|
||||
if len(devs) > 1:
|
||||
print 'Matched more than one device:'
|
||||
print('Matched more than one device:')
|
||||
for dev in devs:
|
||||
print ' %s' % format_device(dev)
|
||||
print(' %s' % format_device(dev))
|
||||
if raw_input('Are you sure you want to update the info for '
|
||||
'these %s devices? (y/N) ' % len(devs)) != 'y':
|
||||
print 'Aborting device modifications'
|
||||
print('Aborting device modifications')
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
for dev in devs:
|
||||
|
@ -260,14 +261,14 @@ def _set_info_values(devs, change):
|
|||
if check_dev['ip'] == test_dev['ip'] and \
|
||||
check_dev['port'] == test_dev['port'] and \
|
||||
check_dev['device'] == test_dev['device']:
|
||||
print 'Device %d already uses %s:%d/%s.' % \
|
||||
print('Device %d already uses %s:%d/%s.' %
|
||||
(check_dev['id'], check_dev['ip'],
|
||||
check_dev['port'], check_dev['device'])
|
||||
check_dev['port'], check_dev['device']))
|
||||
exit(EXIT_ERROR)
|
||||
for key in change:
|
||||
dev[key] = change[key]
|
||||
print 'Device %s is now %s' % (orig_dev_string,
|
||||
format_device(dev))
|
||||
print('Device %s is now %s' % (orig_dev_string,
|
||||
format_device(dev)))
|
||||
|
||||
|
||||
def _parse_set_info_values(argvish):
|
||||
|
@ -279,7 +280,7 @@ def _parse_set_info_values(argvish):
|
|||
# but not both. If both are specified, raise an error.
|
||||
if not new_cmd_format:
|
||||
if len(args) % 2 != 0:
|
||||
print Commands.search.__doc__.strip()
|
||||
print(Commands.search.__doc__.strip())
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
searches_and_changes = izip(islice(argvish, 0, len(argvish), 2),
|
||||
|
@ -368,7 +369,7 @@ def _parse_remove_values(argvish):
|
|||
devs = []
|
||||
if len(args) > 0:
|
||||
if new_cmd_format:
|
||||
print Commands.remove.__doc__.strip()
|
||||
print(Commands.remove.__doc__.strip())
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
for arg in args:
|
||||
|
@ -380,14 +381,14 @@ def _parse_remove_values(argvish):
|
|||
|
||||
return devs
|
||||
except ValueError as e:
|
||||
print e
|
||||
print(e)
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
|
||||
class Commands(object):
|
||||
|
||||
def unknown():
|
||||
print 'Unknown command: %s' % argv[2]
|
||||
print('Unknown command: %s' % argv[2])
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
def create():
|
||||
|
@ -399,17 +400,18 @@ swift-ring-builder <builder_file> create <part_power> <replicas>
|
|||
than once.
|
||||
"""
|
||||
if len(argv) < 6:
|
||||
print Commands.create.__doc__.strip()
|
||||
print(Commands.create.__doc__.strip())
|
||||
exit(EXIT_ERROR)
|
||||
builder = RingBuilder(int(argv[3]), float(argv[4]), int(argv[5]))
|
||||
backup_dir = pathjoin(dirname(argv[1]), 'backups')
|
||||
backup_dir = pathjoin(dirname(builder_file), 'backups')
|
||||
try:
|
||||
mkdir(backup_dir)
|
||||
except OSError as err:
|
||||
if err.errno != EEXIST:
|
||||
raise
|
||||
builder.save(pathjoin(backup_dir, '%d.' % time() + basename(argv[1])))
|
||||
builder.save(argv[1])
|
||||
builder.save(pathjoin(backup_dir,
|
||||
'%d.' % time() + basename(builder_file)))
|
||||
builder.save(builder_file)
|
||||
exit(EXIT_SUCCESS)
|
||||
|
||||
def default():
|
||||
|
@ -417,7 +419,7 @@ swift-ring-builder <builder_file> create <part_power> <replicas>
|
|||
swift-ring-builder <builder_file>
|
||||
Shows information about the ring and the devices within.
|
||||
"""
|
||||
print '%s, build version %d' % (argv[1], builder.version)
|
||||
print('%s, build version %d' % (builder_file, builder.version))
|
||||
regions = 0
|
||||
zones = 0
|
||||
balance = 0
|
||||
|
@ -432,18 +434,18 @@ swift-ring-builder <builder_file>
|
|||
balance = builder.get_balance()
|
||||
dispersion_trailer = '' if builder.dispersion is None else (
|
||||
', %.02f dispersion' % (builder.dispersion))
|
||||
print '%d partitions, %.6f replicas, %d regions, %d zones, ' \
|
||||
print('%d partitions, %.6f replicas, %d regions, %d zones, '
|
||||
'%d devices, %.02f balance%s' % (
|
||||
builder.parts, builder.replicas, regions, zones, dev_count,
|
||||
balance, dispersion_trailer)
|
||||
print 'The minimum number of hours before a partition can be ' \
|
||||
'reassigned is %s' % builder.min_part_hours
|
||||
print 'The overload factor is %0.2f%% (%.6f)' % (
|
||||
builder.overload * 100, builder.overload)
|
||||
balance, dispersion_trailer))
|
||||
print('The minimum number of hours before a partition can be '
|
||||
'reassigned is %s' % builder.min_part_hours)
|
||||
print('The overload factor is %0.2f%% (%.6f)' % (
|
||||
builder.overload * 100, builder.overload))
|
||||
if builder.devs:
|
||||
print 'Devices: id region zone ip address port ' \
|
||||
'replication ip replication port name ' \
|
||||
'weight partitions balance meta'
|
||||
print('Devices: id region zone ip address port '
|
||||
'replication ip replication port name '
|
||||
'weight partitions balance meta')
|
||||
weighted_parts = builder.parts * builder.replicas / \
|
||||
sum(d['weight'] for d in builder.devs if d is not None)
|
||||
for dev in builder.devs:
|
||||
|
@ -483,19 +485,19 @@ swift-ring-builder <builder_file> search
|
|||
Shows information about matching devices.
|
||||
"""
|
||||
if len(argv) < 4:
|
||||
print Commands.search.__doc__.strip()
|
||||
print
|
||||
print parse_search_value.__doc__.strip()
|
||||
print(Commands.search.__doc__.strip())
|
||||
print()
|
||||
print(parse_search_value.__doc__.strip())
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
devs = builder.search_devs(_parse_search_values(argv[3:]))
|
||||
|
||||
if not devs:
|
||||
print 'No matching devices found'
|
||||
print('No matching devices found')
|
||||
exit(EXIT_ERROR)
|
||||
print 'Devices: id region zone ip address port ' \
|
||||
'replication ip replication port name weight partitions ' \
|
||||
'balance meta'
|
||||
print('Devices: id region zone ip address port '
|
||||
'replication ip replication port name weight partitions '
|
||||
'balance meta')
|
||||
weighted_parts = builder.parts * builder.replicas / \
|
||||
sum(d['weight'] for d in builder.devs if d is not None)
|
||||
for dev in devs:
|
||||
|
@ -538,30 +540,30 @@ swift-ring-builder <builder_file> list_parts
|
|||
could take a while to run.
|
||||
"""
|
||||
if len(argv) < 4:
|
||||
print Commands.list_parts.__doc__.strip()
|
||||
print
|
||||
print parse_search_value.__doc__.strip()
|
||||
print(Commands.list_parts.__doc__.strip())
|
||||
print()
|
||||
print(parse_search_value.__doc__.strip())
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
if not builder._replica2part2dev:
|
||||
print('Specified builder file \"%s\" is not rebalanced yet. '
|
||||
'Please rebalance first.' % argv[1])
|
||||
'Please rebalance first.' % builder_file)
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
devs = _parse_list_parts_values(argv[3:])
|
||||
if not devs:
|
||||
print 'No matching devices found'
|
||||
print('No matching devices found')
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
sorted_partition_count = _find_parts(devs)
|
||||
|
||||
if not sorted_partition_count:
|
||||
print 'No matching devices found'
|
||||
print('No matching devices found')
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
print 'Partition Matches'
|
||||
print('Partition Matches')
|
||||
for partition, count in sorted_partition_count:
|
||||
print '%9d %7d' % (partition, count)
|
||||
print('%9d %7d' % (partition, count))
|
||||
exit(EXIT_SUCCESS)
|
||||
|
||||
def add():
|
||||
|
@ -587,7 +589,7 @@ swift-ring-builder <builder_file> add
|
|||
can make multiple device changes and rebalance them all just once.
|
||||
"""
|
||||
if len(argv) < 5:
|
||||
print Commands.add.__doc__.strip()
|
||||
print(Commands.add.__doc__.strip())
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
try:
|
||||
|
@ -598,20 +600,20 @@ swift-ring-builder <builder_file> add
|
|||
if dev['ip'] == new_dev['ip'] and \
|
||||
dev['port'] == new_dev['port'] and \
|
||||
dev['device'] == new_dev['device']:
|
||||
print 'Device %d already uses %s:%d/%s.' % \
|
||||
print('Device %d already uses %s:%d/%s.' %
|
||||
(dev['id'], dev['ip'],
|
||||
dev['port'], dev['device'])
|
||||
print "The on-disk ring builder is unchanged.\n"
|
||||
dev['port'], dev['device']))
|
||||
print("The on-disk ring builder is unchanged.\n")
|
||||
exit(EXIT_ERROR)
|
||||
dev_id = builder.add_dev(new_dev)
|
||||
print('Device %s with %s weight got id %s' %
|
||||
(format_device(new_dev), new_dev['weight'], dev_id))
|
||||
except ValueError as err:
|
||||
print err
|
||||
print 'The on-disk ring builder is unchanged.'
|
||||
print(err)
|
||||
print('The on-disk ring builder is unchanged.')
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
builder.save(argv[1])
|
||||
builder.save(builder_file)
|
||||
exit(EXIT_SUCCESS)
|
||||
|
||||
def set_weight():
|
||||
|
@ -636,14 +638,14 @@ swift-ring-builder <builder_file> set_weight
|
|||
"""
|
||||
# if len(argv) < 5 or len(argv) % 2 != 1:
|
||||
if len(argv) < 5:
|
||||
print Commands.set_weight.__doc__.strip()
|
||||
print
|
||||
print parse_search_value.__doc__.strip()
|
||||
print(Commands.set_weight.__doc__.strip())
|
||||
print()
|
||||
print(parse_search_value.__doc__.strip())
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
_parse_set_weight_values(argv[3:])
|
||||
|
||||
builder.save(argv[1])
|
||||
builder.save(builder_file)
|
||||
exit(EXIT_SUCCESS)
|
||||
|
||||
def set_info():
|
||||
|
@ -677,18 +679,18 @@ swift-ring-builder <builder_file> set_info
|
|||
just update the meta data for device id 74.
|
||||
"""
|
||||
if len(argv) < 5:
|
||||
print Commands.set_info.__doc__.strip()
|
||||
print
|
||||
print parse_search_value.__doc__.strip()
|
||||
print(Commands.set_info.__doc__.strip())
|
||||
print()
|
||||
print(parse_search_value.__doc__.strip())
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
try:
|
||||
_parse_set_info_values(argv[3:])
|
||||
except ValueError as err:
|
||||
print err
|
||||
print(err)
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
builder.save(argv[1])
|
||||
builder.save(builder_file)
|
||||
exit(EXIT_SUCCESS)
|
||||
|
||||
def remove():
|
||||
|
@ -714,9 +716,9 @@ swift-ring-builder <builder_file> search
|
|||
once.
|
||||
"""
|
||||
if len(argv) < 4:
|
||||
print Commands.remove.__doc__.strip()
|
||||
print
|
||||
print parse_search_value.__doc__.strip()
|
||||
print(Commands.remove.__doc__.strip())
|
||||
print()
|
||||
print(parse_search_value.__doc__.strip())
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
devs = _parse_remove_values(argv[3:])
|
||||
|
@ -727,19 +729,19 @@ swift-ring-builder <builder_file> search
|
|||
exit(EXIT_ERROR)
|
||||
|
||||
if len(devs) > 1:
|
||||
print 'Matched more than one device:'
|
||||
print('Matched more than one device:')
|
||||
for dev in devs:
|
||||
print ' %s' % format_device(dev)
|
||||
print(' %s' % format_device(dev))
|
||||
if raw_input('Are you sure you want to remove these %s '
|
||||
'devices? (y/N) ' % len(devs)) != 'y':
|
||||
print 'Aborting device removals'
|
||||
print('Aborting device removals')
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
for dev in devs:
|
||||
try:
|
||||
builder.remove_dev(dev['id'])
|
||||
except exceptions.RingBuilderError as e:
|
||||
print '-' * 79
|
||||
print('-' * 79)
|
||||
print(
|
||||
'An error occurred while removing device with id %d\n'
|
||||
'This usually means that you attempted to remove\n'
|
||||
|
@ -748,12 +750,12 @@ swift-ring-builder <builder_file> search
|
|||
'The on-disk ring builder is unchanged.\n'
|
||||
'Original exception message: %s' %
|
||||
(dev['id'], e))
|
||||
print '-' * 79
|
||||
print('-' * 79)
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
print '%s marked for removal and will ' \
|
||||
'be removed next rebalance.' % format_device(dev)
|
||||
builder.save(argv[1])
|
||||
print('%s marked for removal and will '
|
||||
'be removed next rebalance.' % format_device(dev))
|
||||
builder.save(builder_file)
|
||||
exit(EXIT_SUCCESS)
|
||||
|
||||
def rebalance():
|
||||
|
@ -793,18 +795,18 @@ swift-ring-builder <builder_file> rebalance [options]
|
|||
last_balance = builder.get_balance()
|
||||
parts, balance = builder.rebalance(seed=get_seed(3))
|
||||
except exceptions.RingBuilderError as e:
|
||||
print '-' * 79
|
||||
print('-' * 79)
|
||||
print("An error has occurred during ring validation. Common\n"
|
||||
"causes of failure are rings that are empty or do not\n"
|
||||
"have enough devices to accommodate the replica count.\n"
|
||||
"Original exception message:\n %s" %
|
||||
(e,))
|
||||
print '-' * 79
|
||||
print('-' * 79)
|
||||
exit(EXIT_ERROR)
|
||||
if not (parts or options.force):
|
||||
print 'No partitions could be reassigned.'
|
||||
print 'Either none need to be or none can be due to ' \
|
||||
'min_part_hours [%s].' % builder.min_part_hours
|
||||
print('No partitions could be reassigned.')
|
||||
print('Either none need to be or none can be due to '
|
||||
'min_part_hours [%s].' % builder.min_part_hours)
|
||||
exit(EXIT_WARNING)
|
||||
# If we set device's weight to zero, currently balance will be set
|
||||
# special value(MAX_BALANCE) until zero weighted device return all
|
||||
|
@ -813,29 +815,29 @@ swift-ring-builder <builder_file> rebalance [options]
|
|||
if not options.force and \
|
||||
not devs_changed and abs(last_balance - balance) < 1 and \
|
||||
not (last_balance == MAX_BALANCE and balance == MAX_BALANCE):
|
||||
print 'Cowardly refusing to save rebalance as it did not change ' \
|
||||
'at least 1%.'
|
||||
print('Cowardly refusing to save rebalance as it did not change '
|
||||
'at least 1%.')
|
||||
exit(EXIT_WARNING)
|
||||
try:
|
||||
builder.validate()
|
||||
except exceptions.RingValidationError as e:
|
||||
print '-' * 79
|
||||
print('-' * 79)
|
||||
print("An error has occurred during ring validation. Common\n"
|
||||
"causes of failure are rings that are empty or do not\n"
|
||||
"have enough devices to accommodate the replica count.\n"
|
||||
"Original exception message:\n %s" %
|
||||
(e,))
|
||||
print '-' * 79
|
||||
print('-' * 79)
|
||||
exit(EXIT_ERROR)
|
||||
print ('Reassigned %d (%.02f%%) partitions. '
|
||||
'Balance is now %.02f. '
|
||||
'Dispersion is now %.02f' % (
|
||||
parts, 100.0 * parts / builder.parts,
|
||||
balance,
|
||||
builder.dispersion))
|
||||
print('Reassigned %d (%.02f%%) partitions. '
|
||||
'Balance is now %.02f. '
|
||||
'Dispersion is now %.02f' % (
|
||||
parts, 100.0 * parts / builder.parts,
|
||||
balance,
|
||||
builder.dispersion))
|
||||
status = EXIT_SUCCESS
|
||||
if builder.dispersion > 0:
|
||||
print '-' * 79
|
||||
print('-' * 79)
|
||||
print(
|
||||
'NOTE: Dispersion of %.06f indicates some parts are not\n'
|
||||
' optimally dispersed.\n\n'
|
||||
|
@ -843,21 +845,21 @@ swift-ring-builder <builder_file> rebalance [options]
|
|||
' the overload or review the dispersion report.' %
|
||||
builder.dispersion)
|
||||
status = EXIT_WARNING
|
||||
print '-' * 79
|
||||
print('-' * 79)
|
||||
elif balance > 5 and balance / 100.0 > builder.overload:
|
||||
print '-' * 79
|
||||
print 'NOTE: Balance of %.02f indicates you should push this ' % \
|
||||
balance
|
||||
print ' ring, wait at least %d hours, and rebalance/repush.' \
|
||||
% builder.min_part_hours
|
||||
print '-' * 79
|
||||
print('-' * 79)
|
||||
print('NOTE: Balance of %.02f indicates you should push this ' %
|
||||
balance)
|
||||
print(' ring, wait at least %d hours, and rebalance/repush.'
|
||||
% builder.min_part_hours)
|
||||
print('-' * 79)
|
||||
status = EXIT_WARNING
|
||||
ts = time()
|
||||
builder.get_ring().save(
|
||||
pathjoin(backup_dir, '%d.' % ts + basename(ring_file)))
|
||||
builder.save(pathjoin(backup_dir, '%d.' % ts + basename(argv[1])))
|
||||
builder.save(pathjoin(backup_dir, '%d.' % ts + basename(builder_file)))
|
||||
builder.get_ring().save(ring_file)
|
||||
builder.save(argv[1])
|
||||
builder.save(builder_file)
|
||||
exit(status)
|
||||
|
||||
def dispersion():
|
||||
|
@ -892,7 +894,7 @@ swift-ring-builder <builder_file> dispersion <search_filter> [options]
|
|||
status = EXIT_SUCCESS
|
||||
if not builder._replica2part2dev:
|
||||
print('Specified builder file \"%s\" is not rebalanced yet. '
|
||||
'Please rebalance first.' % argv[1])
|
||||
'Please rebalance first.' % builder_file)
|
||||
exit(EXIT_ERROR)
|
||||
usage = Commands.dispersion.__doc__.strip()
|
||||
parser = optparse.OptionParser(usage)
|
||||
|
@ -905,12 +907,12 @@ swift-ring-builder <builder_file> dispersion <search_filter> [options]
|
|||
search_filter = None
|
||||
report = dispersion_report(builder, search_filter=search_filter,
|
||||
verbose=options.verbose)
|
||||
print 'Dispersion is %.06f, Balance is %.06f, Overload is %0.2f%%' % (
|
||||
builder.dispersion, builder.get_balance(), builder.overload * 100)
|
||||
print('Dispersion is %.06f, Balance is %.06f, Overload is %0.2f%%' % (
|
||||
builder.dispersion, builder.get_balance(), builder.overload * 100))
|
||||
if report['worst_tier']:
|
||||
status = EXIT_WARNING
|
||||
print 'Worst tier is %.06f (%s)' % (report['max_dispersion'],
|
||||
report['worst_tier'])
|
||||
print('Worst tier is %.06f (%s)' % (report['max_dispersion'],
|
||||
report['worst_tier']))
|
||||
if report['graph']:
|
||||
replica_range = range(int(math.ceil(builder.replicas + 1)))
|
||||
part_count_width = '%%%ds' % max(len(str(builder.parts)), 5)
|
||||
|
@ -929,13 +931,19 @@ swift-ring-builder <builder_file> dispersion <search_filter> [options]
|
|||
for tier_name, dispersion in report['graph']:
|
||||
replica_counts_repr = replica_counts_tmpl % tuple(
|
||||
dispersion['replicas'])
|
||||
print ('%-' + str(tier_width) + 's ' + part_count_width +
|
||||
' %6.02f %6d %s') % (tier_name,
|
||||
dispersion['placed_parts'],
|
||||
dispersion['dispersion'],
|
||||
dispersion['max_replicas'],
|
||||
replica_counts_repr,
|
||||
)
|
||||
template = ''.join([
|
||||
'%-', str(tier_width), 's ',
|
||||
part_count_width,
|
||||
' %6.02f %6d %s',
|
||||
])
|
||||
args = (
|
||||
tier_name,
|
||||
dispersion['placed_parts'],
|
||||
dispersion['dispersion'],
|
||||
dispersion['max_replicas'],
|
||||
replica_counts_repr,
|
||||
)
|
||||
print(template % args)
|
||||
exit(status)
|
||||
|
||||
def validate():
|
||||
|
@ -957,11 +965,11 @@ swift-ring-builder <builder_file> write_ring
|
|||
ring_data = builder.get_ring()
|
||||
if not ring_data._replica2part2dev_id:
|
||||
if ring_data.devs:
|
||||
print 'Warning: Writing a ring with no partition ' \
|
||||
'assignments but with devices; did you forget to run ' \
|
||||
'"rebalance"?'
|
||||
print('Warning: Writing a ring with no partition '
|
||||
'assignments but with devices; did you forget to run '
|
||||
'"rebalance"?')
|
||||
else:
|
||||
print 'Warning: Writing an empty ring'
|
||||
print('Warning: Writing an empty ring')
|
||||
ring_data.save(
|
||||
pathjoin(backup_dir, '%d.' % time() + basename(ring_file)))
|
||||
ring_data.save(ring_file)
|
||||
|
@ -976,8 +984,8 @@ swift-ring-builder <ring_file> write_builder [min_part_hours]
|
|||
you can change it with set_min_part_hours.
|
||||
"""
|
||||
if exists(builder_file):
|
||||
print 'Cowardly refusing to overwrite existing ' \
|
||||
'Ring Builder file: %s' % builder_file
|
||||
print('Cowardly refusing to overwrite existing '
|
||||
'Ring Builder file: %s' % builder_file)
|
||||
exit(EXIT_ERROR)
|
||||
if len(argv) > 3:
|
||||
min_part_hours = int(argv[3])
|
||||
|
@ -1014,7 +1022,7 @@ swift-ring-builder <ring_file> write_builder [min_part_hours]
|
|||
|
||||
def pretend_min_part_hours_passed():
|
||||
builder.pretend_min_part_hours_passed()
|
||||
builder.save(argv[1])
|
||||
builder.save(builder_file)
|
||||
exit(EXIT_SUCCESS)
|
||||
|
||||
def set_min_part_hours():
|
||||
|
@ -1025,12 +1033,12 @@ swift-ring-builder <builder_file> set_min_part_hours <hours>
|
|||
to determine this more easily than scanning logs.
|
||||
"""
|
||||
if len(argv) < 4:
|
||||
print Commands.set_min_part_hours.__doc__.strip()
|
||||
print(Commands.set_min_part_hours.__doc__.strip())
|
||||
exit(EXIT_ERROR)
|
||||
builder.change_min_part_hours(int(argv[3]))
|
||||
print 'The minimum number of hours before a partition can be ' \
|
||||
'reassigned is now set to %s' % argv[3]
|
||||
builder.save(argv[1])
|
||||
print('The minimum number of hours before a partition can be '
|
||||
'reassigned is now set to %s' % argv[3])
|
||||
builder.save(builder_file)
|
||||
exit(EXIT_SUCCESS)
|
||||
|
||||
def set_replicas():
|
||||
|
@ -1044,25 +1052,25 @@ swift-ring-builder <builder_file> set_replicas <replicas>
|
|||
A rebalance is needed to make the change take effect.
|
||||
"""
|
||||
if len(argv) < 4:
|
||||
print Commands.set_replicas.__doc__.strip()
|
||||
print(Commands.set_replicas.__doc__.strip())
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
new_replicas = argv[3]
|
||||
try:
|
||||
new_replicas = float(new_replicas)
|
||||
except ValueError:
|
||||
print Commands.set_replicas.__doc__.strip()
|
||||
print "\"%s\" is not a valid number." % new_replicas
|
||||
print(Commands.set_replicas.__doc__.strip())
|
||||
print("\"%s\" is not a valid number." % new_replicas)
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
if new_replicas < 1:
|
||||
print "Replica count must be at least 1."
|
||||
print("Replica count must be at least 1.")
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
builder.set_replicas(new_replicas)
|
||||
print 'The replica count is now %.6f.' % builder.replicas
|
||||
print 'The change will take effect after the next rebalance.'
|
||||
builder.save(argv[1])
|
||||
print('The replica count is now %.6f.' % builder.replicas)
|
||||
print('The change will take effect after the next rebalance.')
|
||||
builder.save(builder_file)
|
||||
exit(EXIT_SUCCESS)
|
||||
|
||||
def set_overload():
|
||||
|
@ -1073,7 +1081,7 @@ swift-ring-builder <builder_file> set_overload <overload>[%]
|
|||
A rebalance is needed to make the change take effect.
|
||||
"""
|
||||
if len(argv) < 4:
|
||||
print Commands.set_overload.__doc__.strip()
|
||||
print(Commands.set_overload.__doc__.strip())
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
new_overload = argv[3]
|
||||
|
@ -1085,27 +1093,27 @@ swift-ring-builder <builder_file> set_overload <overload>[%]
|
|||
try:
|
||||
new_overload = float(new_overload)
|
||||
except ValueError:
|
||||
print Commands.set_overload.__doc__.strip()
|
||||
print "%r is not a valid number." % new_overload
|
||||
print(Commands.set_overload.__doc__.strip())
|
||||
print("%r is not a valid number." % new_overload)
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
if percent:
|
||||
new_overload *= 0.01
|
||||
if new_overload < 0:
|
||||
print "Overload must be non-negative."
|
||||
print("Overload must be non-negative.")
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
if new_overload > 1 and not percent:
|
||||
print "!?! Warning overload is greater than 100% !?!"
|
||||
print("!?! Warning overload is greater than 100% !?!")
|
||||
status = EXIT_WARNING
|
||||
else:
|
||||
status = EXIT_SUCCESS
|
||||
|
||||
builder.set_overload(new_overload)
|
||||
print 'The overload factor is now %0.2f%% (%.6f)' % (
|
||||
builder.overload * 100, builder.overload)
|
||||
print 'The change will take effect after the next rebalance.'
|
||||
builder.save(argv[1])
|
||||
print('The overload factor is now %0.2f%% (%.6f)' % (
|
||||
builder.overload * 100, builder.overload))
|
||||
print('The change will take effect after the next rebalance.')
|
||||
builder.save(builder_file)
|
||||
exit(status)
|
||||
|
||||
|
||||
|
@ -1117,43 +1125,46 @@ def main(arguments=None):
|
|||
argv = sys_argv
|
||||
|
||||
if len(argv) < 2:
|
||||
print "swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s\n" % \
|
||||
globals()
|
||||
print Commands.default.__doc__.strip()
|
||||
print
|
||||
print("swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s\n" %
|
||||
globals())
|
||||
print(Commands.default.__doc__.strip())
|
||||
print()
|
||||
cmds = [c for c, f in Commands.__dict__.items()
|
||||
if f.__doc__ and c[0] != '_' and c != 'default']
|
||||
cmds.sort()
|
||||
for cmd in cmds:
|
||||
print Commands.__dict__[cmd].__doc__.strip()
|
||||
print
|
||||
print parse_search_value.__doc__.strip()
|
||||
print
|
||||
print(Commands.__dict__[cmd].__doc__.strip())
|
||||
print()
|
||||
print(parse_search_value.__doc__.strip())
|
||||
print()
|
||||
for line in wrap(' '.join(cmds), 79, initial_indent='Quick list: ',
|
||||
subsequent_indent=' '):
|
||||
print line
|
||||
print(line)
|
||||
print('Exit codes: 0 = operation successful\n'
|
||||
' 1 = operation completed with warnings\n'
|
||||
' 2 = error')
|
||||
exit(EXIT_SUCCESS)
|
||||
|
||||
builder_file, ring_file = parse_builder_ring_filename_args(argv)
|
||||
if builder_file != argv[1]:
|
||||
print('Note: using %s instead of %s as builder file' % (
|
||||
builder_file, argv[1]))
|
||||
|
||||
try:
|
||||
builder = RingBuilder.load(builder_file)
|
||||
except exceptions.UnPicklingError as e:
|
||||
print e
|
||||
print(e)
|
||||
exit(EXIT_ERROR)
|
||||
except (exceptions.FileNotFoundError, exceptions.PermissionError) as e:
|
||||
if len(argv) < 3 or argv[2] not in('create', 'write_builder'):
|
||||
print e
|
||||
print(e)
|
||||
exit(EXIT_ERROR)
|
||||
except Exception as e:
|
||||
print('Problem occurred while reading builder file: %s. %s' %
|
||||
(argv[1], e))
|
||||
(builder_file, e))
|
||||
exit(EXIT_ERROR)
|
||||
|
||||
backup_dir = pathjoin(dirname(argv[1]), 'backups')
|
||||
backup_dir = pathjoin(dirname(builder_file), 'backups')
|
||||
try:
|
||||
mkdir(backup_dir)
|
||||
except OSError as err:
|
||||
|
@ -1166,10 +1177,10 @@ def main(arguments=None):
|
|||
command = argv[2]
|
||||
if argv[0].endswith('-safe'):
|
||||
try:
|
||||
with lock_parent_directory(abspath(argv[1]), 15):
|
||||
with lock_parent_directory(abspath(builder_file), 15):
|
||||
Commands.__dict__.get(command, Commands.unknown.im_func)()
|
||||
except exceptions.LockTimeout:
|
||||
print "Ring/builder dir currently locked."
|
||||
print("Ring/builder dir currently locked.")
|
||||
exit(2)
|
||||
else:
|
||||
Commands.__dict__.get(command, Commands.unknown.im_func)()
|
||||
|
|
|
@ -13,11 +13,13 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import functools
|
||||
import os
|
||||
import urllib
|
||||
import time
|
||||
from urllib import unquote
|
||||
from ConfigParser import ConfigParser, NoSectionError, NoOptionError
|
||||
|
||||
from six.moves.configparser import ConfigParser, NoSectionError, NoOptionError
|
||||
|
||||
from swift.common import utils, exceptions
|
||||
from swift.common.swob import HTTPBadRequest, HTTPLengthRequired, \
|
||||
|
@ -405,28 +407,33 @@ def check_destination_header(req):
|
|||
'<container name>/<object name>')
|
||||
|
||||
|
||||
def check_account_format(req, account):
|
||||
def check_name_format(req, name, target_type):
|
||||
"""
|
||||
Validate that the header contains valid account name.
|
||||
We assume the caller ensures that
|
||||
destination header is present in req.headers.
|
||||
Validate that the header contains valid account or container name.
|
||||
|
||||
:param req: HTTP request object
|
||||
:returns: A properly encoded account name
|
||||
:param name: header value to validate
|
||||
:param target_type: which header is being validated (Account or Container)
|
||||
:returns: A properly encoded account name or container name
|
||||
:raise: HTTPPreconditionFailed if account header
|
||||
is not well formatted.
|
||||
"""
|
||||
if not account:
|
||||
if not name:
|
||||
raise HTTPPreconditionFailed(
|
||||
request=req,
|
||||
body='Account name cannot be empty')
|
||||
if isinstance(account, unicode):
|
||||
account = account.encode('utf-8')
|
||||
if '/' in account:
|
||||
body='%s name cannot be empty' % target_type)
|
||||
if isinstance(name, unicode):
|
||||
name = name.encode('utf-8')
|
||||
if '/' in name:
|
||||
raise HTTPPreconditionFailed(
|
||||
request=req,
|
||||
body='Account name cannot contain slashes')
|
||||
return account
|
||||
body='%s name cannot contain slashes' % target_type)
|
||||
return name
|
||||
|
||||
check_account_format = functools.partial(check_name_format,
|
||||
target_type='Account')
|
||||
check_container_format = functools.partial(check_name_format,
|
||||
target_type='Container')
|
||||
|
||||
|
||||
def valid_api_version(version):
|
||||
|
|
|
@ -13,13 +13,14 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import ConfigParser
|
||||
import errno
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import time
|
||||
|
||||
from six.moves import configparser
|
||||
|
||||
from swift import gettext_ as _
|
||||
from swift.common.utils import get_valid_utf8_str
|
||||
|
||||
|
@ -61,9 +62,9 @@ class ContainerSyncRealms(object):
|
|||
if mtime != self.conf_path_mtime:
|
||||
self.conf_path_mtime = mtime
|
||||
try:
|
||||
conf = ConfigParser.SafeConfigParser()
|
||||
conf = configparser.SafeConfigParser()
|
||||
conf.read(self.conf_path)
|
||||
except ConfigParser.ParsingError as err:
|
||||
except configparser.ParsingError as err:
|
||||
self.logger.error(
|
||||
_('Could not load %r: %s'), self.conf_path, err)
|
||||
else:
|
||||
|
@ -72,11 +73,11 @@ class ContainerSyncRealms(object):
|
|||
'DEFAULT', 'mtime_check_interval')
|
||||
self.next_mtime_check = \
|
||||
now + self.mtime_check_interval
|
||||
except ConfigParser.NoOptionError:
|
||||
except configparser.NoOptionError:
|
||||
self.mtime_check_interval = 300
|
||||
self.next_mtime_check = \
|
||||
now + self.mtime_check_interval
|
||||
except (ConfigParser.ParsingError, ValueError) as err:
|
||||
except (configparser.ParsingError, ValueError) as err:
|
||||
self.logger.error(
|
||||
_('Error in %r with mtime_check_interval: %s'),
|
||||
self.conf_path, err)
|
||||
|
|
|
@ -23,7 +23,7 @@ from uuid import uuid4
|
|||
import sys
|
||||
import time
|
||||
import errno
|
||||
import cPickle as pickle
|
||||
import six.moves.cPickle as pickle
|
||||
from swift import gettext_ as _
|
||||
from tempfile import mkstemp
|
||||
|
||||
|
|
|
@ -187,7 +187,8 @@ class Replicator(Daemon):
|
|||
self.stats = {'attempted': 0, 'success': 0, 'failure': 0, 'ts_repl': 0,
|
||||
'no_change': 0, 'hashmatch': 0, 'rsync': 0, 'diff': 0,
|
||||
'remove': 0, 'empty': 0, 'remote_merge': 0,
|
||||
'start': time.time(), 'diff_capped': 0}
|
||||
'start': time.time(), 'diff_capped': 0,
|
||||
'failure_nodes': {}}
|
||||
|
||||
def _report_stats(self):
|
||||
"""Report the current stats to the logs."""
|
||||
|
@ -212,6 +213,13 @@ class Replicator(Daemon):
|
|||
('no_change', 'hashmatch', 'rsync', 'diff', 'ts_repl',
|
||||
'empty', 'diff_capped')]))
|
||||
|
||||
def _add_failure_stats(self, failure_devs_info):
|
||||
for node, dev in failure_devs_info:
|
||||
self.stats['failure'] += 1
|
||||
failure_devs = self.stats['failure_nodes'].setdefault(node, {})
|
||||
failure_devs.setdefault(dev, 0)
|
||||
failure_devs[dev] += 1
|
||||
|
||||
def _rsync_file(self, db_file, remote_file, whole_file=True,
|
||||
different_region=False):
|
||||
"""
|
||||
|
@ -479,7 +487,10 @@ class Replicator(Daemon):
|
|||
quarantine_db(broker.db_file, broker.db_type)
|
||||
else:
|
||||
self.logger.exception(_('ERROR reading db %s'), object_file)
|
||||
self.stats['failure'] += 1
|
||||
nodes = self.ring.get_part_nodes(int(partition))
|
||||
self._add_failure_stats([(failure_dev['replication_ip'],
|
||||
failure_dev['device'])
|
||||
for failure_dev in nodes])
|
||||
self.logger.increment('failures')
|
||||
return
|
||||
# The db is considered deleted if the delete_timestamp value is greater
|
||||
|
@ -494,6 +505,7 @@ class Replicator(Daemon):
|
|||
self.logger.timing_since('timing', start_time)
|
||||
return
|
||||
responses = []
|
||||
failure_devs_info = set()
|
||||
nodes = self.ring.get_part_nodes(int(partition))
|
||||
local_dev = None
|
||||
for node in nodes:
|
||||
|
@ -532,7 +544,8 @@ class Replicator(Daemon):
|
|||
self.logger.exception(_('ERROR syncing %(file)s with node'
|
||||
' %(node)s'),
|
||||
{'file': object_file, 'node': node})
|
||||
self.stats['success' if success else 'failure'] += 1
|
||||
if not success:
|
||||
failure_devs_info.add((node['replication_ip'], node['device']))
|
||||
self.logger.increment('successes' if success else 'failures')
|
||||
responses.append(success)
|
||||
try:
|
||||
|
@ -543,7 +556,17 @@ class Replicator(Daemon):
|
|||
if not shouldbehere and all(responses):
|
||||
# If the db shouldn't be on this node and has been successfully
|
||||
# synced to all of its peers, it can be removed.
|
||||
self.delete_db(broker)
|
||||
if not self.delete_db(broker):
|
||||
failure_devs_info.update(
|
||||
[(failure_dev['replication_ip'], failure_dev['device'])
|
||||
for failure_dev in repl_nodes])
|
||||
|
||||
target_devs_info = set([(target_dev['replication_ip'],
|
||||
target_dev['device'])
|
||||
for target_dev in repl_nodes])
|
||||
self.stats['success'] += len(target_devs_info - failure_devs_info)
|
||||
self._add_failure_stats(failure_devs_info)
|
||||
|
||||
self.logger.timing_since('timing', start_time)
|
||||
|
||||
def delete_db(self, broker):
|
||||
|
@ -558,9 +581,11 @@ class Replicator(Daemon):
|
|||
if err.errno not in (errno.ENOENT, errno.ENOTEMPTY):
|
||||
self.logger.exception(
|
||||
_('ERROR while trying to clean up %s') % suf_dir)
|
||||
return False
|
||||
self.stats['remove'] += 1
|
||||
device_name = self.extract_device(object_file)
|
||||
self.logger.increment('removes.' + device_name)
|
||||
return True
|
||||
|
||||
def extract_device(self, object_file):
|
||||
"""
|
||||
|
@ -592,6 +617,10 @@ class Replicator(Daemon):
|
|||
node['replication_port']):
|
||||
if self.mount_check and not ismount(
|
||||
os.path.join(self.root, node['device'])):
|
||||
self._add_failure_stats(
|
||||
[(failure_dev['replication_ip'],
|
||||
failure_dev['device'])
|
||||
for failure_dev in self.ring.devs if failure_dev])
|
||||
self.logger.warn(
|
||||
_('Skipping %(device)s as it is not mounted') % node)
|
||||
continue
|
||||
|
|
|
@ -20,10 +20,10 @@ through the proxy.
|
|||
|
||||
import os
|
||||
import socket
|
||||
from httplib import HTTPException
|
||||
from time import time
|
||||
|
||||
from eventlet import sleep, Timeout
|
||||
from six.moves.http_client import HTTPException
|
||||
|
||||
from swift.common.bufferedhttp import http_connect
|
||||
from swift.common.exceptions import ClientException
|
||||
|
@ -401,7 +401,7 @@ def direct_put_object(node, part, account, container, name, contents,
|
|||
headers['Content-Length'] = '0'
|
||||
if isinstance(contents, basestring):
|
||||
contents = [contents]
|
||||
#Incase the caller want to insert an object with specific age
|
||||
# Incase the caller want to insert an object with specific age
|
||||
add_ts = 'X-Timestamp' not in headers
|
||||
|
||||
if content_length is None:
|
||||
|
@ -543,8 +543,8 @@ def retry(func, *args, **kwargs):
|
|||
# Shouldn't actually get down here, but just in case.
|
||||
if args and 'ip' in args[0]:
|
||||
raise ClientException('Raise too many retries',
|
||||
http_host=args[
|
||||
0]['ip'], http_port=args[0]['port'],
|
||||
http_host=args[0]['ip'],
|
||||
http_port=args[0]['port'],
|
||||
http_device=args[0]['device'])
|
||||
else:
|
||||
raise ClientException('Raise too many retries')
|
||||
|
|
|
@ -67,7 +67,7 @@ def is_server_error(status):
|
|||
# List of HTTP status codes
|
||||
|
||||
###############################################################################
|
||||
## 1xx Informational
|
||||
# 1xx Informational
|
||||
###############################################################################
|
||||
|
||||
HTTP_CONTINUE = 100
|
||||
|
@ -77,7 +77,7 @@ HTTP_CHECKPOINT = 103
|
|||
HTTP_REQUEST_URI_TOO_LONG = 122
|
||||
|
||||
###############################################################################
|
||||
## 2xx Success
|
||||
# 2xx Success
|
||||
###############################################################################
|
||||
|
||||
HTTP_OK = 200
|
||||
|
@ -91,7 +91,7 @@ HTTP_MULTI_STATUS = 207 # WebDAV
|
|||
HTTP_IM_USED = 226
|
||||
|
||||
###############################################################################
|
||||
## 3xx Redirection
|
||||
# 3xx Redirection
|
||||
###############################################################################
|
||||
|
||||
HTTP_MULTIPLE_CHOICES = 300
|
||||
|
@ -105,7 +105,7 @@ HTTP_TEMPORARY_REDIRECT = 307
|
|||
HTTP_RESUME_INCOMPLETE = 308
|
||||
|
||||
###############################################################################
|
||||
## 4xx Client Error
|
||||
# 4xx Client Error
|
||||
###############################################################################
|
||||
|
||||
HTTP_BAD_REQUEST = 400
|
||||
|
@ -141,7 +141,7 @@ HTTP_BLOCKED_BY_WINDOWS_PARENTAL_CONTROLS = 450
|
|||
HTTP_CLIENT_CLOSED_REQUEST = 499
|
||||
|
||||
###############################################################################
|
||||
## 5xx Server Error
|
||||
# 5xx Server Error
|
||||
###############################################################################
|
||||
|
||||
HTTP_INTERNAL_SERVER_ERROR = 500
|
||||
|
|
|
@ -13,6 +13,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
import functools
|
||||
import errno
|
||||
import os
|
||||
|
@ -62,22 +63,22 @@ def setup_env():
|
|||
resource.setrlimit(resource.RLIMIT_NOFILE,
|
||||
(MAX_DESCRIPTORS, MAX_DESCRIPTORS))
|
||||
except ValueError:
|
||||
print _("WARNING: Unable to modify file descriptor limit. "
|
||||
"Running as non-root?")
|
||||
print(_("WARNING: Unable to modify file descriptor limit. "
|
||||
"Running as non-root?"))
|
||||
|
||||
try:
|
||||
resource.setrlimit(resource.RLIMIT_DATA,
|
||||
(MAX_MEMORY, MAX_MEMORY))
|
||||
except ValueError:
|
||||
print _("WARNING: Unable to modify memory limit. "
|
||||
"Running as non-root?")
|
||||
print(_("WARNING: Unable to modify memory limit. "
|
||||
"Running as non-root?"))
|
||||
|
||||
try:
|
||||
resource.setrlimit(resource.RLIMIT_NPROC,
|
||||
(MAX_PROCS, MAX_PROCS))
|
||||
except ValueError:
|
||||
print _("WARNING: Unable to modify max process limit. "
|
||||
"Running as non-root?")
|
||||
print(_("WARNING: Unable to modify max process limit. "
|
||||
"Running as non-root?"))
|
||||
|
||||
# Set PYTHON_EGG_CACHE if it isn't already set
|
||||
os.environ.setdefault('PYTHON_EGG_CACHE', '/tmp')
|
||||
|
@ -217,7 +218,7 @@ class Manager(object):
|
|||
try:
|
||||
status += server.interact(**kwargs)
|
||||
except KeyboardInterrupt:
|
||||
print _('\nuser quit')
|
||||
print(_('\nuser quit'))
|
||||
self.stop(**kwargs)
|
||||
break
|
||||
elif kwargs.get('wait', True):
|
||||
|
@ -254,7 +255,7 @@ class Manager(object):
|
|||
for server in self.servers:
|
||||
signaled_pids = server.stop(**kwargs)
|
||||
if not signaled_pids:
|
||||
print _('No %s running') % server
|
||||
print(_('No %s running') % server)
|
||||
else:
|
||||
server_pids[server] = signaled_pids
|
||||
|
||||
|
@ -267,7 +268,7 @@ class Manager(object):
|
|||
for server, killed_pid in watch_server_pids(server_pids,
|
||||
interval=kill_wait,
|
||||
**kwargs):
|
||||
print _("%s (%s) appears to have stopped") % (server, killed_pid)
|
||||
print(_("%s (%s) appears to have stopped") % (server, killed_pid))
|
||||
killed_pids.add(killed_pid)
|
||||
if not killed_pids.symmetric_difference(signaled_pids):
|
||||
# all processes have been stopped
|
||||
|
@ -277,8 +278,8 @@ class Manager(object):
|
|||
for server, pids in server_pids.items():
|
||||
if not killed_pids.issuperset(pids):
|
||||
# some pids of this server were not killed
|
||||
print _('Waited %s seconds for %s to die; giving up') % (
|
||||
kill_wait, server)
|
||||
print(_('Waited %s seconds for %s to die; giving up') % (
|
||||
kill_wait, server))
|
||||
return 1
|
||||
|
||||
@command
|
||||
|
@ -461,15 +462,15 @@ class Server(object):
|
|||
# maybe there's a config file(s) out there, but I couldn't find it!
|
||||
if not kwargs.get('quiet'):
|
||||
if number:
|
||||
print _('Unable to locate config number %s for %s' % (
|
||||
number, self.server))
|
||||
print(_('Unable to locate config number %s for %s')
|
||||
% (number, self.server))
|
||||
else:
|
||||
print _('Unable to locate config for %s' % (self.server))
|
||||
print(_('Unable to locate config for %s') % self.server)
|
||||
if kwargs.get('verbose') and not kwargs.get('quiet'):
|
||||
if found_conf_files:
|
||||
print _('Found configs:')
|
||||
print(_('Found configs:'))
|
||||
for i, conf_file in enumerate(found_conf_files):
|
||||
print ' %d) %s' % (i + 1, conf_file)
|
||||
print(' %d) %s' % (i + 1, conf_file))
|
||||
|
||||
return conf_files
|
||||
|
||||
|
@ -514,27 +515,27 @@ class Server(object):
|
|||
pids = {}
|
||||
for pid_file, pid in self.iter_pid_files(**kwargs):
|
||||
if not pid: # Catches None and 0
|
||||
print _('Removing pid file %s with invalid pid') % pid_file
|
||||
print (_('Removing pid file %s with invalid pid') % pid_file)
|
||||
remove_file(pid_file)
|
||||
continue
|
||||
try:
|
||||
if sig != signal.SIG_DFL:
|
||||
print _('Signal %s pid: %s signal: %s') % (self.server,
|
||||
pid, sig)
|
||||
print(_('Signal %s pid: %s signal: %s') % (self.server,
|
||||
pid, sig))
|
||||
safe_kill(pid, sig, 'swift-%s' % self.server)
|
||||
except InvalidPidFileException as e:
|
||||
if kwargs.get('verbose'):
|
||||
print _('Removing pid file %s with wrong pid %d') \
|
||||
% (pid_file, pid)
|
||||
print(_('Removing pid file %s with wrong pid %d') % (
|
||||
pid_file, pid))
|
||||
remove_file(pid_file)
|
||||
except OSError as e:
|
||||
if e.errno == errno.ESRCH:
|
||||
# pid does not exist
|
||||
if kwargs.get('verbose'):
|
||||
print _("Removing stale pid file %s") % pid_file
|
||||
print(_("Removing stale pid file %s") % pid_file)
|
||||
remove_file(pid_file)
|
||||
elif e.errno == errno.EPERM:
|
||||
print _("No permission to signal PID %d") % pid
|
||||
print(_("No permission to signal PID %d") % pid)
|
||||
else:
|
||||
# process exists
|
||||
pids[pid] = pid_file
|
||||
|
@ -579,14 +580,14 @@ class Server(object):
|
|||
kwargs['quiet'] = True
|
||||
conf_files = self.conf_files(**kwargs)
|
||||
if conf_files:
|
||||
print _("%s #%d not running (%s)") % (self.server, number,
|
||||
conf_files[0])
|
||||
print(_("%s #%d not running (%s)") % (self.server, number,
|
||||
conf_files[0]))
|
||||
else:
|
||||
print _("No %s running") % self.server
|
||||
print(_("No %s running") % self.server)
|
||||
return 1
|
||||
for pid, pid_file in pids.items():
|
||||
conf_file = self.get_conf_file_name(pid_file)
|
||||
print _("%s running (%s - %s)") % (self.server, pid, conf_file)
|
||||
print(_("%s running (%s - %s)") % (self.server, pid, conf_file))
|
||||
return 0
|
||||
|
||||
def spawn(self, conf_file, once=False, wait=True, daemon=True, **kwargs):
|
||||
|
@ -638,7 +639,7 @@ class Server(object):
|
|||
# no-daemon anyway, but this is quieter
|
||||
proc.wait()
|
||||
if output:
|
||||
print output
|
||||
print(output)
|
||||
start = time.time()
|
||||
# wait for process to die (output may just be a warning)
|
||||
while time.time() - start < WARNING_WAIT:
|
||||
|
@ -679,13 +680,14 @@ class Server(object):
|
|||
# any unstarted instances
|
||||
if conf_file in conf_files:
|
||||
already_started = True
|
||||
print _("%s running (%s - %s)") % (self.server, pid, conf_file)
|
||||
print(_("%s running (%s - %s)") %
|
||||
(self.server, pid, conf_file))
|
||||
elif not kwargs.get('number', 0):
|
||||
already_started = True
|
||||
print _("%s running (%s - %s)") % (self.server, pid, pid_file)
|
||||
print(_("%s running (%s - %s)") % (self.server, pid, pid_file))
|
||||
|
||||
if already_started:
|
||||
print _("%s already started...") % self.server
|
||||
print(_("%s already started...") % self.server)
|
||||
return {}
|
||||
|
||||
if self.server not in START_ONCE_SERVERS:
|
||||
|
@ -697,13 +699,13 @@ class Server(object):
|
|||
msg = _('Running %s once') % self.server
|
||||
else:
|
||||
msg = _('Starting %s') % self.server
|
||||
print '%s...(%s)' % (msg, conf_file)
|
||||
print('%s...(%s)' % (msg, conf_file))
|
||||
try:
|
||||
pid = self.spawn(conf_file, **kwargs)
|
||||
except OSError as e:
|
||||
if e.errno == errno.ENOENT:
|
||||
#TODO(clayg): should I check if self.cmd exists earlier?
|
||||
print _("%s does not exist") % self.cmd
|
||||
# TODO(clayg): should I check if self.cmd exists earlier?
|
||||
print(_("%s does not exist") % self.cmd)
|
||||
break
|
||||
else:
|
||||
raise
|
||||
|
|
|
@ -44,7 +44,7 @@ version is at:
|
|||
http://github.com/memcached/memcached/blob/1.4.2/doc/protocol.txt
|
||||
"""
|
||||
|
||||
import cPickle as pickle
|
||||
import six.moves.cPickle as pickle
|
||||
import logging
|
||||
import time
|
||||
from bisect import bisect
|
||||
|
@ -443,7 +443,7 @@ class MemcacheRing(object):
|
|||
with Timeout(self._io_timeout):
|
||||
sock.sendall(msg)
|
||||
# Wait for the set to complete
|
||||
for _ in range(len(mapping)):
|
||||
for line in range(len(mapping)):
|
||||
fp.readline()
|
||||
self._return_conn(server, fp, sock)
|
||||
return
|
||||
|
|
|
@ -14,7 +14,9 @@
|
|||
# limitations under the License.
|
||||
|
||||
import os
|
||||
from ConfigParser import ConfigParser, NoSectionError, NoOptionError
|
||||
|
||||
from six.moves.configparser import ConfigParser, NoSectionError, NoOptionError
|
||||
|
||||
from hashlib import md5
|
||||
from swift.common import constraints
|
||||
from swift.common.exceptions import ListingIterError, SegmentError
|
||||
|
|
|
@ -410,7 +410,7 @@ class KeystoneAuth(object):
|
|||
user_id, user_name = env_identity['user']
|
||||
referrers, roles = swift_acl.parse_acl(getattr(req, 'acl', None))
|
||||
|
||||
#allow OPTIONS requests to proceed as normal
|
||||
# allow OPTIONS requests to proceed as normal
|
||||
if req.method == 'OPTIONS':
|
||||
return
|
||||
|
||||
|
@ -526,7 +526,7 @@ class KeystoneAuth(object):
|
|||
except ValueError:
|
||||
return HTTPNotFound(request=req)
|
||||
|
||||
#allow OPTIONS requests to proceed as normal
|
||||
# allow OPTIONS requests to proceed as normal
|
||||
if req.method == 'OPTIONS':
|
||||
return
|
||||
|
||||
|
|
|
@ -14,7 +14,8 @@
|
|||
# limitations under the License.
|
||||
|
||||
import os
|
||||
from ConfigParser import ConfigParser, NoSectionError, NoOptionError
|
||||
|
||||
from six.moves.configparser import ConfigParser, NoSectionError, NoOptionError
|
||||
|
||||
from swift.common.memcached import (MemcacheRing, CONN_TIMEOUT, POOL_TIMEOUT,
|
||||
IO_TIMEOUT, TRY_COUNT)
|
||||
|
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
import errno
|
||||
import os
|
||||
import time
|
||||
from swift import gettext_ as _
|
||||
|
||||
from swift import __version__ as swiftver
|
||||
|
@ -133,19 +134,19 @@ class ReconMiddleware(object):
|
|||
|
||||
def get_replication_info(self, recon_type):
|
||||
"""get replication info"""
|
||||
replication_list = ['replication_time',
|
||||
'replication_stats',
|
||||
'replication_last']
|
||||
if recon_type == 'account':
|
||||
return self._from_recon_cache(['replication_time',
|
||||
'replication_stats',
|
||||
'replication_last'],
|
||||
return self._from_recon_cache(replication_list,
|
||||
self.account_recon_cache)
|
||||
elif recon_type == 'container':
|
||||
return self._from_recon_cache(['replication_time',
|
||||
'replication_stats',
|
||||
'replication_last'],
|
||||
return self._from_recon_cache(replication_list,
|
||||
self.container_recon_cache)
|
||||
elif recon_type == 'object':
|
||||
return self._from_recon_cache(['object_replication_time',
|
||||
'object_replication_last'],
|
||||
replication_list += ['object_replication_time',
|
||||
'object_replication_last']
|
||||
return self._from_recon_cache(replication_list,
|
||||
self.object_recon_cache)
|
||||
else:
|
||||
return None
|
||||
|
@ -328,6 +329,11 @@ class ReconMiddleware(object):
|
|||
raise
|
||||
return sockstat
|
||||
|
||||
def get_time(self):
|
||||
"""get current time"""
|
||||
|
||||
return time.time()
|
||||
|
||||
def GET(self, req):
|
||||
root, rcheck, rtype = req.split_path(1, 3, True)
|
||||
all_rtypes = ['account', 'container', 'object']
|
||||
|
@ -340,7 +346,7 @@ class ReconMiddleware(object):
|
|||
elif rcheck == 'replication' and rtype in all_rtypes:
|
||||
content = self.get_replication_info(rtype)
|
||||
elif rcheck == 'replication' and rtype is None:
|
||||
#handle old style object replication requests
|
||||
# handle old style object replication requests
|
||||
content = self.get_replication_info('object')
|
||||
elif rcheck == "devices":
|
||||
content = self.get_device_info()
|
||||
|
@ -368,6 +374,8 @@ class ReconMiddleware(object):
|
|||
content = self.get_version()
|
||||
elif rcheck == "driveaudit":
|
||||
content = self.get_driveaudit_error()
|
||||
elif rcheck == "time":
|
||||
content = self.get_time()
|
||||
else:
|
||||
content = "Invalid path: %s" % req.path
|
||||
return Response(request=req, status="404 Not Found",
|
||||
|
|
|
@ -148,10 +148,11 @@ metadata which can be used for stats purposes.
|
|||
|
||||
from six.moves import range
|
||||
|
||||
from cStringIO import StringIO
|
||||
from datetime import datetime
|
||||
import mimetypes
|
||||
import re
|
||||
import six
|
||||
from six import BytesIO
|
||||
from hashlib import md5
|
||||
from swift.common.exceptions import ListingIterError, SegmentError
|
||||
from swift.common.swob import Request, HTTPBadRequest, HTTPServerError, \
|
||||
|
@ -681,8 +682,10 @@ class StaticLargeObject(object):
|
|||
env['CONTENT_TYPE'] += ";swift_bytes=%d" % total_size
|
||||
env['HTTP_X_STATIC_LARGE_OBJECT'] = 'True'
|
||||
json_data = json.dumps(data_for_storage)
|
||||
if six.PY3:
|
||||
json_data = json_data.encode('utf-8')
|
||||
env['CONTENT_LENGTH'] = str(len(json_data))
|
||||
env['wsgi.input'] = StringIO(json_data)
|
||||
env['wsgi.input'] = BytesIO(json_data)
|
||||
|
||||
slo_put_context = SloPutContext(self, slo_etag)
|
||||
return slo_put_context.handle_slo_put(req, start_response)
|
||||
|
|
|
@ -531,7 +531,7 @@ class TempAuth(object):
|
|||
return None
|
||||
|
||||
if req.method == 'OPTIONS':
|
||||
#allow OPTIONS requests to proceed as normal
|
||||
# allow OPTIONS requests to proceed as normal
|
||||
self.logger.debug("Allow OPTIONS request.")
|
||||
return None
|
||||
|
||||
|
@ -674,15 +674,15 @@ class TempAuth(object):
|
|||
user = req.headers.get('x-auth-user')
|
||||
if not user or ':' not in user:
|
||||
self.logger.increment('token_denied')
|
||||
return HTTPUnauthorized(request=req, headers=
|
||||
{'Www-Authenticate':
|
||||
'Swift realm="%s"' % account})
|
||||
auth = 'Swift realm="%s"' % account
|
||||
return HTTPUnauthorized(request=req,
|
||||
headers={'Www-Authenticate': auth})
|
||||
account2, user = user.split(':', 1)
|
||||
if account != account2:
|
||||
self.logger.increment('token_denied')
|
||||
return HTTPUnauthorized(request=req, headers=
|
||||
{'Www-Authenticate':
|
||||
'Swift realm="%s"' % account})
|
||||
auth = 'Swift realm="%s"' % account
|
||||
return HTTPUnauthorized(request=req,
|
||||
headers={'Www-Authenticate': auth})
|
||||
key = req.headers.get('x-storage-pass')
|
||||
if not key:
|
||||
key = req.headers.get('x-auth-key')
|
||||
|
@ -692,9 +692,9 @@ class TempAuth(object):
|
|||
user = req.headers.get('x-storage-user')
|
||||
if not user or ':' not in user:
|
||||
self.logger.increment('token_denied')
|
||||
return HTTPUnauthorized(request=req, headers=
|
||||
{'Www-Authenticate':
|
||||
'Swift realm="unknown"'})
|
||||
auth = 'Swift realm="unknown"'
|
||||
return HTTPUnauthorized(request=req,
|
||||
headers={'Www-Authenticate': auth})
|
||||
account, user = user.split(':', 1)
|
||||
key = req.headers.get('x-auth-key')
|
||||
if not key:
|
||||
|
@ -711,14 +711,14 @@ class TempAuth(object):
|
|||
account_user = account + ':' + user
|
||||
if account_user not in self.users:
|
||||
self.logger.increment('token_denied')
|
||||
return HTTPUnauthorized(request=req, headers=
|
||||
{'Www-Authenticate':
|
||||
'Swift realm="%s"' % account})
|
||||
auth = 'Swift realm="%s"' % account
|
||||
return HTTPUnauthorized(request=req,
|
||||
headers={'Www-Authenticate': auth})
|
||||
if self.users[account_user]['key'] != key:
|
||||
self.logger.increment('token_denied')
|
||||
return HTTPUnauthorized(request=req, headers=
|
||||
{'Www-Authenticate':
|
||||
'Swift realm="unknown"'})
|
||||
auth = 'Swift realm="unknown"'
|
||||
return HTTPUnauthorized(request=req,
|
||||
headers={'Www-Authenticate': auth})
|
||||
account_id = self.users[account_user]['url'].rsplit('/', 1)[-1]
|
||||
# Get memcache client
|
||||
memcache_client = cache_from_env(req.environ)
|
||||
|
|
|
@ -122,11 +122,13 @@ from urllib import urlencode
|
|||
from urlparse import parse_qs
|
||||
|
||||
from swift.proxy.controllers.base import get_account_info, get_container_info
|
||||
from swift.common.swob import HeaderKeyDict, HTTPUnauthorized
|
||||
from swift.common.swob import HeaderKeyDict, HTTPUnauthorized, HTTPBadRequest
|
||||
from swift.common.utils import split_path, get_valid_utf8_str, \
|
||||
register_swift_info, get_hmac, streq_const_time, quote
|
||||
|
||||
|
||||
DISALLOWED_INCOMING_HEADERS = 'x-object-manifest'
|
||||
|
||||
#: Default headers to remove from incoming requests. Simply a whitespace
|
||||
#: delimited list of header names and names can optionally end with '*' to
|
||||
#: indicate a prefix match. DEFAULT_INCOMING_ALLOW_HEADERS is a list of
|
||||
|
@ -150,6 +152,10 @@ DEFAULT_OUTGOING_REMOVE_HEADERS = 'x-object-meta-*'
|
|||
DEFAULT_OUTGOING_ALLOW_HEADERS = 'x-object-meta-public-*'
|
||||
|
||||
|
||||
CONTAINER_SCOPE = 'container'
|
||||
ACCOUNT_SCOPE = 'account'
|
||||
|
||||
|
||||
def get_tempurl_keys_from_metadata(meta):
|
||||
"""
|
||||
Extracts the tempurl keys from metadata.
|
||||
|
@ -170,6 +176,38 @@ def disposition_format(filename):
|
|||
quote(filename, safe=' /'), quote(filename))
|
||||
|
||||
|
||||
def authorize_same_account(account_to_match):
|
||||
|
||||
def auth_callback_same_account(req):
|
||||
try:
|
||||
_ver, acc, _rest = req.split_path(2, 3, True)
|
||||
except ValueError:
|
||||
return HTTPUnauthorized(request=req)
|
||||
|
||||
if acc == account_to_match:
|
||||
return None
|
||||
else:
|
||||
return HTTPUnauthorized(request=req)
|
||||
|
||||
return auth_callback_same_account
|
||||
|
||||
|
||||
def authorize_same_container(account_to_match, container_to_match):
|
||||
|
||||
def auth_callback_same_container(req):
|
||||
try:
|
||||
_ver, acc, con, _rest = req.split_path(3, 4, True)
|
||||
except ValueError:
|
||||
return HTTPUnauthorized(request=req)
|
||||
|
||||
if acc == account_to_match and con == container_to_match:
|
||||
return None
|
||||
else:
|
||||
return HTTPUnauthorized(request=req)
|
||||
|
||||
return auth_callback_same_container
|
||||
|
||||
|
||||
class TempURL(object):
|
||||
"""
|
||||
WSGI Middleware to grant temporary URLs specific access to Swift
|
||||
|
@ -230,6 +268,10 @@ class TempURL(object):
|
|||
#: The methods allowed with Temp URLs.
|
||||
self.methods = methods
|
||||
|
||||
self.disallowed_headers = set(
|
||||
'HTTP_' + h.upper().replace('-', '_')
|
||||
for h in DISALLOWED_INCOMING_HEADERS.split())
|
||||
|
||||
headers = DEFAULT_INCOMING_REMOVE_HEADERS
|
||||
if 'incoming_remove_headers' in conf:
|
||||
headers = conf['incoming_remove_headers']
|
||||
|
@ -298,10 +340,10 @@ class TempURL(object):
|
|||
return self.app(env, start_response)
|
||||
if not temp_url_sig or not temp_url_expires:
|
||||
return self._invalid(env, start_response)
|
||||
account = self._get_account(env)
|
||||
account, container = self._get_account_and_container(env)
|
||||
if not account:
|
||||
return self._invalid(env, start_response)
|
||||
keys = self._get_keys(env, account)
|
||||
keys = self._get_keys(env)
|
||||
if not keys:
|
||||
return self._invalid(env, start_response)
|
||||
if env['REQUEST_METHOD'] == 'HEAD':
|
||||
|
@ -316,15 +358,32 @@ class TempURL(object):
|
|||
else:
|
||||
hmac_vals = self._get_hmacs(env, temp_url_expires, keys)
|
||||
|
||||
# While it's true that any() will short-circuit, this doesn't affect
|
||||
# the timing-attack resistance since the only way this will
|
||||
# short-circuit is when a valid signature is passed in.
|
||||
is_valid_hmac = any(streq_const_time(temp_url_sig, hmac)
|
||||
for hmac in hmac_vals)
|
||||
is_valid_hmac = False
|
||||
hmac_scope = None
|
||||
for hmac, scope in hmac_vals:
|
||||
# While it's true that we short-circuit, this doesn't affect the
|
||||
# timing-attack resistance since the only way this will
|
||||
# short-circuit is when a valid signature is passed in.
|
||||
if streq_const_time(temp_url_sig, hmac):
|
||||
is_valid_hmac = True
|
||||
hmac_scope = scope
|
||||
break
|
||||
if not is_valid_hmac:
|
||||
return self._invalid(env, start_response)
|
||||
# disallowed headers prevent accidently allowing upload of a pointer
|
||||
# to data that the PUT tempurl would not otherwise allow access for.
|
||||
# It should be safe to provide a GET tempurl for data that an
|
||||
# untrusted client just uploaded with a PUT tempurl.
|
||||
resp = self._clean_disallowed_headers(env, start_response)
|
||||
if resp:
|
||||
return resp
|
||||
self._clean_incoming_headers(env)
|
||||
env['swift.authorize'] = lambda req: None
|
||||
|
||||
if hmac_scope == ACCOUNT_SCOPE:
|
||||
env['swift.authorize'] = authorize_same_account(account)
|
||||
else:
|
||||
env['swift.authorize'] = authorize_same_container(account,
|
||||
container)
|
||||
env['swift.authorize_override'] = True
|
||||
env['REMOTE_USER'] = '.wsgi.tempurl'
|
||||
qs = {'temp_url_sig': temp_url_sig,
|
||||
|
@ -365,22 +424,23 @@ class TempURL(object):
|
|||
|
||||
return self.app(env, _start_response)
|
||||
|
||||
def _get_account(self, env):
|
||||
def _get_account_and_container(self, env):
|
||||
"""
|
||||
Returns just the account for the request, if it's an object
|
||||
request and one of the configured methods; otherwise, None is
|
||||
Returns just the account and container for the request, if it's an
|
||||
object request and one of the configured methods; otherwise, None is
|
||||
returned.
|
||||
|
||||
:param env: The WSGI environment for the request.
|
||||
:returns: Account str or None.
|
||||
:returns: (Account str, container str) or (None, None).
|
||||
"""
|
||||
if env['REQUEST_METHOD'] in self.methods:
|
||||
try:
|
||||
ver, acc, cont, obj = split_path(env['PATH_INFO'], 4, 4, True)
|
||||
except ValueError:
|
||||
return None
|
||||
return (None, None)
|
||||
if ver == 'v1' and obj.strip('/'):
|
||||
return acc
|
||||
return (acc, cont)
|
||||
return (None, None)
|
||||
|
||||
def _get_temp_url_info(self, env):
|
||||
"""
|
||||
|
@ -410,18 +470,23 @@ class TempURL(object):
|
|||
inline = True
|
||||
return temp_url_sig, temp_url_expires, filename, inline
|
||||
|
||||
def _get_keys(self, env, account):
|
||||
def _get_keys(self, env):
|
||||
"""
|
||||
Returns the X-[Account|Container]-Meta-Temp-URL-Key[-2] header values
|
||||
for the account or container, or an empty list if none are set.
|
||||
for the account or container, or an empty list if none are set. Each
|
||||
value comes as a 2-tuple (key, scope), where scope is either
|
||||
CONTAINER_SCOPE or ACCOUNT_SCOPE.
|
||||
|
||||
Returns 0-4 elements depending on how many keys are set in the
|
||||
account's or container's metadata.
|
||||
|
||||
:param env: The WSGI environment for the request.
|
||||
:param account: Account str.
|
||||
:returns: [X-Account-Meta-Temp-URL-Key str value if set,
|
||||
X-Account-Meta-Temp-URL-Key-2 str value if set]
|
||||
:returns: [
|
||||
(X-Account-Meta-Temp-URL-Key str value, ACCOUNT_SCOPE) if set,
|
||||
(X-Account-Meta-Temp-URL-Key-2 str value, ACCOUNT_SCOPE if set,
|
||||
(X-Container-Meta-Temp-URL-Key str value, CONTAINER_SCOPE) if set,
|
||||
(X-Container-Meta-Temp-URL-Key-2 str value, CONTAINER_SCOPE if set,
|
||||
]
|
||||
"""
|
||||
account_info = get_account_info(env, self.app, swift_source='TU')
|
||||
account_keys = get_tempurl_keys_from_metadata(account_info['meta'])
|
||||
|
@ -430,25 +495,28 @@ class TempURL(object):
|
|||
container_keys = get_tempurl_keys_from_metadata(
|
||||
container_info.get('meta', []))
|
||||
|
||||
return account_keys + container_keys
|
||||
return ([(ak, ACCOUNT_SCOPE) for ak in account_keys] +
|
||||
[(ck, CONTAINER_SCOPE) for ck in container_keys])
|
||||
|
||||
def _get_hmacs(self, env, expires, keys, request_method=None):
|
||||
def _get_hmacs(self, env, expires, scoped_keys, request_method=None):
|
||||
"""
|
||||
:param env: The WSGI environment for the request.
|
||||
:param expires: Unix timestamp as an int for when the URL
|
||||
expires.
|
||||
:param keys: Key strings, from the X-Account-Meta-Temp-URL-Key[-2] of
|
||||
the account.
|
||||
:param scoped_keys: (key, scope) tuples like _get_keys() returns
|
||||
:param request_method: Optional override of the request in
|
||||
the WSGI env. For example, if a HEAD
|
||||
does not match, you may wish to
|
||||
override with GET to still allow the
|
||||
HEAD.
|
||||
|
||||
:returns: a list of (hmac, scope) 2-tuples
|
||||
"""
|
||||
if not request_method:
|
||||
request_method = env['REQUEST_METHOD']
|
||||
return [get_hmac(
|
||||
request_method, env['PATH_INFO'], expires, key) for key in keys]
|
||||
return [
|
||||
(get_hmac(request_method, env['PATH_INFO'], expires, key), scope)
|
||||
for (key, scope) in scoped_keys]
|
||||
|
||||
def _invalid(self, env, start_response):
|
||||
"""
|
||||
|
@ -465,6 +533,22 @@ class TempURL(object):
|
|||
body = '401 Unauthorized: Temp URL invalid\n'
|
||||
return HTTPUnauthorized(body=body)(env, start_response)
|
||||
|
||||
def _clean_disallowed_headers(self, env, start_response):
|
||||
"""
|
||||
Validate the absense of disallowed headers for "unsafe" operations.
|
||||
|
||||
:returns: None for safe operations or swob.HTTPBadResponse if the
|
||||
request includes disallowed headers.
|
||||
"""
|
||||
if env['REQUEST_METHOD'] in ('GET', 'HEAD', 'OPTIONS'):
|
||||
return
|
||||
for h in env:
|
||||
if h in self.disallowed_headers:
|
||||
return HTTPBadRequest(
|
||||
body='The header %r is not allowed in this tempurl' %
|
||||
h[len('HTTP_'):].title().replace('_', '-'))(
|
||||
env, start_response)
|
||||
|
||||
def _clean_incoming_headers(self, env):
|
||||
"""
|
||||
Removes any headers from the WSGI environment as per the
|
||||
|
|
|
@ -0,0 +1,496 @@
|
|||
# Copyright (c) 2014 OpenStack Foundation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Object versioning in swift is implemented by setting a flag on the container
|
||||
to tell swift to version all objects in the container. The flag is the
|
||||
``X-Versions-Location`` header on the container, and its value is the
|
||||
container where the versions are stored. It is recommended to use a different
|
||||
``X-Versions-Location`` container for each container that is being versioned.
|
||||
|
||||
When data is ``PUT`` into a versioned container (a container with the
|
||||
versioning flag turned on), the existing data in the file is redirected to a
|
||||
new object and the data in the ``PUT`` request is saved as the data for the
|
||||
versioned object. The new object name (for the previous version) is
|
||||
``<versions_container>/<length><object_name>/<timestamp>``, where ``length``
|
||||
is the 3-character zero-padded hexadecimal length of the ``<object_name>`` and
|
||||
``<timestamp>`` is the timestamp of when the previous version was created.
|
||||
|
||||
A ``GET`` to a versioned object will return the current version of the object
|
||||
without having to do any request redirects or metadata lookups.
|
||||
|
||||
A ``POST`` to a versioned object will update the object metadata as normal,
|
||||
but will not create a new version of the object. In other words, new versions
|
||||
are only created when the content of the object changes.
|
||||
|
||||
A ``DELETE`` to a versioned object will only remove the current version of the
|
||||
object. If you have 5 total versions of the object, you must delete the
|
||||
object 5 times to completely remove the object.
|
||||
|
||||
--------------------------------------------------
|
||||
How to Enable Object Versioning in a Swift Cluster
|
||||
--------------------------------------------------
|
||||
|
||||
This middleware was written as an effort to refactor parts of the proxy server,
|
||||
so this functionality was already available in previous releases and every
|
||||
attempt was made to maintain backwards compatibility. To allow operators to
|
||||
perform a seamless upgrade, it is not required to add the middleware to the
|
||||
proxy pipeline and the flag ``allow_versions`` in the container server
|
||||
configuration files are still valid. In future releases, ``allow_versions``
|
||||
will be deprecated in favor of adding this middleware to the pipeline to enable
|
||||
or disable the feature.
|
||||
|
||||
In case the middleware is added to the proxy pipeline, you must also
|
||||
set ``allow_versioned_writes`` to ``True`` in the middleware options
|
||||
to enable the information about this middleware to be returned in a /info
|
||||
request.
|
||||
|
||||
Upgrade considerations: If ``allow_versioned_writes`` is set in the filter
|
||||
configuration, you can leave the ``allow_versions`` flag in the container
|
||||
server configuration files untouched. If you decide to disable or remove the
|
||||
``allow_versions`` flag, you must re-set any existing containers that had
|
||||
the 'X-Versions-Location' flag configured so that it can now be tracked by the
|
||||
versioned_writes middleware.
|
||||
|
||||
-----------------------
|
||||
Examples Using ``curl``
|
||||
-----------------------
|
||||
|
||||
First, create a container with the ``X-Versions-Location`` header or add the
|
||||
header to an existing container. Also make sure the container referenced by
|
||||
the ``X-Versions-Location`` exists. In this example, the name of that
|
||||
container is "versions"::
|
||||
|
||||
curl -i -XPUT -H "X-Auth-Token: <token>" \
|
||||
-H "X-Versions-Location: versions" http://<storage_url>/container
|
||||
curl -i -XPUT -H "X-Auth-Token: <token>" http://<storage_url>/versions
|
||||
|
||||
Create an object (the first version)::
|
||||
|
||||
curl -i -XPUT --data-binary 1 -H "X-Auth-Token: <token>" \
|
||||
http://<storage_url>/container/myobject
|
||||
|
||||
Now create a new version of that object::
|
||||
|
||||
curl -i -XPUT --data-binary 2 -H "X-Auth-Token: <token>" \
|
||||
http://<storage_url>/container/myobject
|
||||
|
||||
See a listing of the older versions of the object::
|
||||
|
||||
curl -i -H "X-Auth-Token: <token>" \
|
||||
http://<storage_url>/versions?prefix=008myobject/
|
||||
|
||||
Now delete the current version of the object and see that the older version is
|
||||
gone::
|
||||
|
||||
curl -i -XDELETE -H "X-Auth-Token: <token>" \
|
||||
http://<storage_url>/container/myobject
|
||||
curl -i -H "X-Auth-Token: <token>" \
|
||||
http://<storage_url>/versions?prefix=008myobject/
|
||||
|
||||
---------------------------------------------------
|
||||
How to Disable Object Versioning in a Swift Cluster
|
||||
---------------------------------------------------
|
||||
|
||||
If you want to disable all functionality, set ``allow_versioned_writes`` to
|
||||
``False`` in the middleware options.
|
||||
|
||||
Disable versioning from a container (x is any value except empty)::
|
||||
|
||||
curl -i -XPOST -H "X-Auth-Token: <token>" \
|
||||
-H "X-Remove-Versions-Location: x" http://<storage_url>/container
|
||||
"""
|
||||
|
||||
import time
|
||||
from urllib import quote, unquote
|
||||
from swift.common.utils import get_logger, Timestamp, json, \
|
||||
register_swift_info, config_true_value
|
||||
from swift.common.request_helpers import get_sys_meta_prefix
|
||||
from swift.common.wsgi import WSGIContext, make_pre_authed_request
|
||||
from swift.common.swob import Request, HTTPException
|
||||
from swift.common.constraints import (
|
||||
check_account_format, check_container_format, check_destination_header)
|
||||
from swift.proxy.controllers.base import get_container_info
|
||||
from swift.common.http import (
|
||||
is_success, is_client_error, HTTP_NOT_FOUND)
|
||||
from swift.common.swob import HTTPPreconditionFailed, HTTPServiceUnavailable, \
|
||||
HTTPServerError
|
||||
from swift.common.exceptions import (
|
||||
ListingIterNotFound, ListingIterError)
|
||||
|
||||
|
||||
class VersionedWritesContext(WSGIContext):
|
||||
|
||||
def __init__(self, wsgi_app, logger):
|
||||
WSGIContext.__init__(self, wsgi_app)
|
||||
self.logger = logger
|
||||
|
||||
def _listing_iter(self, account_name, lcontainer, lprefix, env):
|
||||
for page in self._listing_pages_iter(account_name,
|
||||
lcontainer, lprefix, env):
|
||||
for item in page:
|
||||
yield item
|
||||
|
||||
def _listing_pages_iter(self, account_name, lcontainer, lprefix, env):
|
||||
marker = ''
|
||||
while True:
|
||||
lreq = make_pre_authed_request(
|
||||
env, method='GET', swift_source='VW',
|
||||
path='/v1/%s/%s' % (account_name, lcontainer))
|
||||
lreq.environ['QUERY_STRING'] = \
|
||||
'format=json&prefix=%s&marker=%s' % (quote(lprefix),
|
||||
quote(marker))
|
||||
lresp = lreq.get_response(self.app)
|
||||
if not is_success(lresp.status_int):
|
||||
if lresp.status_int == HTTP_NOT_FOUND:
|
||||
raise ListingIterNotFound()
|
||||
elif is_client_error(lresp.status_int):
|
||||
raise HTTPPreconditionFailed()
|
||||
else:
|
||||
raise ListingIterError()
|
||||
|
||||
if not lresp.body:
|
||||
break
|
||||
|
||||
sublisting = json.loads(lresp.body)
|
||||
if not sublisting:
|
||||
break
|
||||
marker = sublisting[-1]['name'].encode('utf-8')
|
||||
yield sublisting
|
||||
|
||||
def handle_obj_versions_put(self, req, object_versions,
|
||||
object_name, policy_index):
|
||||
ret = None
|
||||
|
||||
# do a HEAD request to check object versions
|
||||
_headers = {'X-Newest': 'True',
|
||||
'X-Backend-Storage-Policy-Index': policy_index,
|
||||
'x-auth-token': req.headers.get('x-auth-token')}
|
||||
|
||||
# make a pre_auth request in case the user has write access
|
||||
# to container, but not READ. This was allowed in previous version
|
||||
# (i.e., before middleware) so keeping the same behavior here
|
||||
head_req = make_pre_authed_request(
|
||||
req.environ, path=req.path_info,
|
||||
headers=_headers, method='HEAD', swift_source='VW')
|
||||
hresp = head_req.get_response(self.app)
|
||||
|
||||
is_dlo_manifest = 'X-Object-Manifest' in req.headers or \
|
||||
'X-Object-Manifest' in hresp.headers
|
||||
|
||||
# if there's an existing object, then copy it to
|
||||
# X-Versions-Location
|
||||
if is_success(hresp.status_int) and not is_dlo_manifest:
|
||||
lcontainer = object_versions.split('/')[0]
|
||||
prefix_len = '%03x' % len(object_name)
|
||||
lprefix = prefix_len + object_name + '/'
|
||||
ts_source = hresp.environ.get('swift_x_timestamp')
|
||||
if ts_source is None:
|
||||
ts_source = time.mktime(time.strptime(
|
||||
hresp.headers['last-modified'],
|
||||
'%a, %d %b %Y %H:%M:%S GMT'))
|
||||
new_ts = Timestamp(ts_source).internal
|
||||
vers_obj_name = lprefix + new_ts
|
||||
copy_headers = {
|
||||
'Destination': '%s/%s' % (lcontainer, vers_obj_name),
|
||||
'x-auth-token': req.headers.get('x-auth-token')}
|
||||
|
||||
# COPY implementation sets X-Newest to True when it internally
|
||||
# does a GET on source object. So, we don't have to explicity
|
||||
# set it in request headers here.
|
||||
copy_req = make_pre_authed_request(
|
||||
req.environ, path=req.path_info,
|
||||
headers=copy_headers, method='COPY', swift_source='VW')
|
||||
copy_resp = copy_req.get_response(self.app)
|
||||
|
||||
if is_success(copy_resp.status_int):
|
||||
# success versioning previous existing object
|
||||
# return None and handle original request
|
||||
ret = None
|
||||
else:
|
||||
if is_client_error(copy_resp.status_int):
|
||||
# missing container or bad permissions
|
||||
ret = HTTPPreconditionFailed(request=req)
|
||||
else:
|
||||
# could not copy the data, bail
|
||||
ret = HTTPServiceUnavailable(request=req)
|
||||
|
||||
else:
|
||||
if hresp.status_int == HTTP_NOT_FOUND or is_dlo_manifest:
|
||||
# nothing to version
|
||||
# return None and handle original request
|
||||
ret = None
|
||||
else:
|
||||
# if not HTTP_NOT_FOUND, return error immediately
|
||||
ret = hresp
|
||||
|
||||
return ret
|
||||
|
||||
def handle_obj_versions_delete(self, req, object_versions,
|
||||
account_name, container_name, object_name):
|
||||
lcontainer = object_versions.split('/')[0]
|
||||
prefix_len = '%03x' % len(object_name)
|
||||
lprefix = prefix_len + object_name + '/'
|
||||
item_list = []
|
||||
try:
|
||||
for _item in self._listing_iter(account_name, lcontainer, lprefix,
|
||||
req.environ):
|
||||
item_list.append(_item)
|
||||
except ListingIterNotFound:
|
||||
pass
|
||||
except HTTPPreconditionFailed:
|
||||
return HTTPPreconditionFailed(request=req)
|
||||
except ListingIterError:
|
||||
return HTTPServerError(request=req)
|
||||
|
||||
if item_list:
|
||||
# we're about to start making COPY requests - need to validate the
|
||||
# write access to the versioned container
|
||||
if 'swift.authorize' in req.environ:
|
||||
container_info = get_container_info(
|
||||
req.environ, self.app)
|
||||
req.acl = container_info.get('write_acl')
|
||||
aresp = req.environ['swift.authorize'](req)
|
||||
if aresp:
|
||||
return aresp
|
||||
|
||||
while len(item_list) > 0:
|
||||
previous_version = item_list.pop()
|
||||
|
||||
# there are older versions so copy the previous version to the
|
||||
# current object and delete the previous version
|
||||
prev_obj_name = previous_version['name'].encode('utf-8')
|
||||
|
||||
copy_path = '/v1/' + account_name + '/' + \
|
||||
lcontainer + '/' + prev_obj_name
|
||||
|
||||
copy_headers = {'X-Newest': 'True',
|
||||
'Destination': container_name + '/' + object_name,
|
||||
'x-auth-token': req.headers.get('x-auth-token')}
|
||||
|
||||
copy_req = make_pre_authed_request(
|
||||
req.environ, path=copy_path,
|
||||
headers=copy_headers, method='COPY', swift_source='VW')
|
||||
copy_resp = copy_req.get_response(self.app)
|
||||
|
||||
# if the version isn't there, keep trying with previous version
|
||||
if copy_resp.status_int == HTTP_NOT_FOUND:
|
||||
continue
|
||||
|
||||
if not is_success(copy_resp.status_int):
|
||||
if is_client_error(copy_resp.status_int):
|
||||
# some user error, maybe permissions
|
||||
return HTTPPreconditionFailed(request=req)
|
||||
else:
|
||||
# could not copy the data, bail
|
||||
return HTTPServiceUnavailable(request=req)
|
||||
|
||||
# reset these because the COPY changed them
|
||||
new_del_req = make_pre_authed_request(
|
||||
req.environ, path=copy_path, method='DELETE',
|
||||
swift_source='VW')
|
||||
req = new_del_req
|
||||
|
||||
# remove 'X-If-Delete-At', since it is not for the older copy
|
||||
if 'X-If-Delete-At' in req.headers:
|
||||
del req.headers['X-If-Delete-At']
|
||||
break
|
||||
|
||||
# handle DELETE request here in case it was modified
|
||||
return req.get_response(self.app)
|
||||
|
||||
def handle_container_request(self, env, start_response):
|
||||
app_resp = self._app_call(env)
|
||||
if self._response_headers is None:
|
||||
self._response_headers = []
|
||||
sysmeta_version_hdr = get_sys_meta_prefix('container') + \
|
||||
'versions-location'
|
||||
location = ''
|
||||
for key, val in self._response_headers:
|
||||
if key.lower() == sysmeta_version_hdr:
|
||||
location = val
|
||||
|
||||
if location:
|
||||
self._response_headers.extend([('X-Versions-Location', location)])
|
||||
|
||||
start_response(self._response_status,
|
||||
self._response_headers,
|
||||
self._response_exc_info)
|
||||
return app_resp
|
||||
|
||||
|
||||
class VersionedWritesMiddleware(object):
|
||||
|
||||
def __init__(self, app, conf):
|
||||
self.app = app
|
||||
self.conf = conf
|
||||
self.logger = get_logger(conf, log_route='versioned_writes')
|
||||
|
||||
def container_request(self, req, start_response, enabled):
|
||||
sysmeta_version_hdr = get_sys_meta_prefix('container') + \
|
||||
'versions-location'
|
||||
|
||||
# set version location header as sysmeta
|
||||
if 'X-Versions-Location' in req.headers:
|
||||
val = req.headers.get('X-Versions-Location')
|
||||
if val:
|
||||
# diferently from previous version, we are actually
|
||||
# returning an error if user tries to set versions location
|
||||
# while feature is explicitly disabled.
|
||||
if not config_true_value(enabled) and \
|
||||
req.method in ('PUT', 'POST'):
|
||||
raise HTTPPreconditionFailed(
|
||||
request=req, content_type='text/plain',
|
||||
body='Versioned Writes is disabled')
|
||||
|
||||
location = check_container_format(req, val)
|
||||
req.headers[sysmeta_version_hdr] = location
|
||||
|
||||
# reset original header to maintain sanity
|
||||
# now only sysmeta is source of Versions Location
|
||||
req.headers['X-Versions-Location'] = ''
|
||||
|
||||
# if both headers are in the same request
|
||||
# adding location takes precendence over removing
|
||||
if 'X-Remove-Versions-Location' in req.headers:
|
||||
del req.headers['X-Remove-Versions-Location']
|
||||
else:
|
||||
# empty value is the same as X-Remove-Versions-Location
|
||||
req.headers['X-Remove-Versions-Location'] = 'x'
|
||||
|
||||
# handle removing versions container
|
||||
val = req.headers.get('X-Remove-Versions-Location')
|
||||
if val:
|
||||
req.headers.update({sysmeta_version_hdr: ''})
|
||||
req.headers.update({'X-Versions-Location': ''})
|
||||
del req.headers['X-Remove-Versions-Location']
|
||||
|
||||
# send request and translate sysmeta headers from response
|
||||
vw_ctx = VersionedWritesContext(self.app, self.logger)
|
||||
return vw_ctx.handle_container_request(req.environ, start_response)
|
||||
|
||||
def object_request(self, req, version, account, container, obj,
|
||||
allow_versioned_writes):
|
||||
account_name = unquote(account)
|
||||
container_name = unquote(container)
|
||||
object_name = unquote(obj)
|
||||
container_info = None
|
||||
resp = None
|
||||
is_enabled = config_true_value(allow_versioned_writes)
|
||||
if req.method in ('PUT', 'DELETE'):
|
||||
container_info = get_container_info(
|
||||
req.environ, self.app)
|
||||
elif req.method == 'COPY' and 'Destination' in req.headers:
|
||||
if 'Destination-Account' in req.headers:
|
||||
account_name = req.headers.get('Destination-Account')
|
||||
account_name = check_account_format(req, account_name)
|
||||
container_name, object_name = check_destination_header(req)
|
||||
req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % (
|
||||
version, account_name, container_name, object_name)
|
||||
container_info = get_container_info(
|
||||
req.environ, self.app)
|
||||
|
||||
if not container_info:
|
||||
return self.app
|
||||
|
||||
# To maintain backwards compatibility, container version
|
||||
# location could be stored as sysmeta or not, need to check both.
|
||||
# If stored as sysmeta, check if middleware is enabled. If sysmeta
|
||||
# is not set, but versions property is set in container_info, then
|
||||
# for backwards compatibility feature is enabled.
|
||||
object_versions = container_info.get(
|
||||
'sysmeta', {}).get('versions-location')
|
||||
if object_versions and isinstance(object_versions, unicode):
|
||||
object_versions = object_versions.encode('utf-8')
|
||||
elif not object_versions:
|
||||
object_versions = container_info.get('versions')
|
||||
# if allow_versioned_writes is not set in the configuration files
|
||||
# but 'versions' is configured, enable feature to maintain
|
||||
# backwards compatibility
|
||||
if not allow_versioned_writes and object_versions:
|
||||
is_enabled = True
|
||||
|
||||
if is_enabled and object_versions:
|
||||
object_versions = unquote(object_versions)
|
||||
vw_ctx = VersionedWritesContext(self.app, self.logger)
|
||||
if req.method in ('PUT', 'COPY'):
|
||||
policy_idx = req.headers.get(
|
||||
'X-Backend-Storage-Policy-Index',
|
||||
container_info['storage_policy'])
|
||||
resp = vw_ctx.handle_obj_versions_put(
|
||||
req, object_versions, object_name, policy_idx)
|
||||
else: # handle DELETE
|
||||
resp = vw_ctx.handle_obj_versions_delete(
|
||||
req, object_versions, account_name,
|
||||
container_name, object_name)
|
||||
|
||||
if resp:
|
||||
return resp
|
||||
else:
|
||||
return self.app
|
||||
|
||||
def __call__(self, env, start_response):
|
||||
# making a duplicate, because if this is a COPY request, we will
|
||||
# modify the PATH_INFO to find out if the 'Destination' is in a
|
||||
# versioned container
|
||||
req = Request(env.copy())
|
||||
try:
|
||||
(version, account, container, obj) = req.split_path(3, 4, True)
|
||||
except ValueError:
|
||||
return self.app(env, start_response)
|
||||
|
||||
# In case allow_versioned_writes is set in the filter configuration,
|
||||
# the middleware becomes the authority on whether object
|
||||
# versioning is enabled or not. In case it is not set, then
|
||||
# the option in the container configuration is still checked
|
||||
# for backwards compatibility
|
||||
|
||||
# For a container request, first just check if option is set,
|
||||
# can be either true or false.
|
||||
# If set, check if enabled when actually trying to set container
|
||||
# header. If not set, let request be handled by container server
|
||||
# for backwards compatibility.
|
||||
# For an object request, also check if option is set (either T or F).
|
||||
# If set, check if enabled when checking versions container in
|
||||
# sysmeta property. If it is not set check 'versions' property in
|
||||
# container_info
|
||||
allow_versioned_writes = self.conf.get('allow_versioned_writes')
|
||||
if allow_versioned_writes and container and not obj:
|
||||
try:
|
||||
return self.container_request(req, start_response,
|
||||
allow_versioned_writes)
|
||||
except HTTPException as error_response:
|
||||
return error_response(env, start_response)
|
||||
elif obj and req.method in ('PUT', 'COPY', 'DELETE'):
|
||||
try:
|
||||
return self.object_request(
|
||||
req, version, account, container, obj,
|
||||
allow_versioned_writes)(env, start_response)
|
||||
except HTTPException as error_response:
|
||||
return error_response(env, start_response)
|
||||
else:
|
||||
return self.app(env, start_response)
|
||||
|
||||
|
||||
def filter_factory(global_conf, **local_conf):
|
||||
conf = global_conf.copy()
|
||||
conf.update(local_conf)
|
||||
if config_true_value(conf.get('allow_versioned_writes')):
|
||||
register_swift_info('versioned_writes')
|
||||
|
||||
def obj_versions_filter(app):
|
||||
return VersionedWritesMiddleware(app, conf)
|
||||
|
||||
return obj_versions_filter
|
|
@ -423,7 +423,7 @@ class HTMLViewer(object):
|
|||
plt.yticks(y_pos, nfls)
|
||||
plt.xlabel(names[metric_selected])
|
||||
plt.title('Profile Statistics (by %s)' % names[metric_selected])
|
||||
#plt.gcf().tight_layout(pad=1.2)
|
||||
# plt.gcf().tight_layout(pad=1.2)
|
||||
with tempfile.TemporaryFile() as profile_img:
|
||||
plt.savefig(profile_img, format='png', dpi=300)
|
||||
profile_img.seek(0)
|
||||
|
|
|
@ -20,7 +20,7 @@ import itertools
|
|||
import logging
|
||||
import math
|
||||
import random
|
||||
import cPickle as pickle
|
||||
import six.moves.cPickle as pickle
|
||||
from copy import deepcopy
|
||||
|
||||
from array import array
|
||||
|
|
|
@ -14,7 +14,7 @@
|
|||
# limitations under the License.
|
||||
|
||||
import array
|
||||
import cPickle as pickle
|
||||
import six.moves.cPickle as pickle
|
||||
import inspect
|
||||
from collections import defaultdict
|
||||
from gzip import GzipFile
|
||||
|
|
|
@ -11,12 +11,14 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from ConfigParser import ConfigParser
|
||||
|
||||
import os
|
||||
import string
|
||||
import textwrap
|
||||
import six
|
||||
|
||||
from six.moves.configparser import ConfigParser
|
||||
|
||||
from swift.common.utils import (
|
||||
config_true_value, SWIFT_CONF_FILE, whataremyips)
|
||||
from swift.common.ring import Ring, RingData
|
||||
|
|
|
@ -36,7 +36,6 @@ needs to change.
|
|||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
from StringIO import StringIO
|
||||
import UserDict
|
||||
import time
|
||||
from functools import partial
|
||||
|
@ -49,6 +48,9 @@ import random
|
|||
import functools
|
||||
import inspect
|
||||
|
||||
from six import BytesIO
|
||||
from six import StringIO
|
||||
|
||||
from swift.common.utils import reiterate, split_path, Timestamp, pairs, \
|
||||
close_if_possible
|
||||
from swift.common.exceptions import InvalidTimestamp
|
||||
|
@ -129,10 +131,10 @@ class _UTC(tzinfo):
|
|||
UTC = _UTC()
|
||||
|
||||
|
||||
class WsgiStringIO(StringIO):
|
||||
class WsgiBytesIO(BytesIO):
|
||||
"""
|
||||
This class adds support for the additional wsgi.input methods defined on
|
||||
eventlet.wsgi.Input to the StringIO class which would otherwise be a fine
|
||||
eventlet.wsgi.Input to the BytesIO class which would otherwise be a fine
|
||||
stand-in for the file-like object in the WSGI environment.
|
||||
"""
|
||||
|
||||
|
@ -478,8 +480,8 @@ class Range(object):
|
|||
After initialization, "range.ranges" is populated with a list
|
||||
of (start, end) tuples denoting the requested ranges.
|
||||
|
||||
If there were any syntactically-invalid byte-range-spec values,
|
||||
"range.ranges" will be an empty list, per the relevant RFC:
|
||||
If there were any syntactically-invalid byte-range-spec values, the
|
||||
constructor will raise a ValueError, per the relevant RFC:
|
||||
|
||||
"The recipient of a byte-range-set that includes one or more syntactically
|
||||
invalid byte-range-spec values MUST ignore the header field that includes
|
||||
|
@ -758,16 +760,16 @@ def _req_environ_property(environ_field):
|
|||
def _req_body_property():
|
||||
"""
|
||||
Set and retrieve the Request.body parameter. It consumes wsgi.input and
|
||||
returns the results. On assignment, uses a WsgiStringIO to create a new
|
||||
returns the results. On assignment, uses a WsgiBytesIO to create a new
|
||||
wsgi.input.
|
||||
"""
|
||||
def getter(self):
|
||||
body = self.environ['wsgi.input'].read()
|
||||
self.environ['wsgi.input'] = WsgiStringIO(body)
|
||||
self.environ['wsgi.input'] = WsgiBytesIO(body)
|
||||
return body
|
||||
|
||||
def setter(self, value):
|
||||
self.environ['wsgi.input'] = WsgiStringIO(value)
|
||||
self.environ['wsgi.input'] = WsgiBytesIO(value)
|
||||
self.environ['CONTENT_LENGTH'] = str(len(value))
|
||||
|
||||
return property(getter, setter, doc="Get and set the request body str")
|
||||
|
@ -835,7 +837,7 @@ class Request(object):
|
|||
:param path: encoded, parsed, and unquoted into PATH_INFO
|
||||
:param environ: WSGI environ dictionary
|
||||
:param headers: HTTP headers
|
||||
:param body: stuffed in a WsgiStringIO and hung on wsgi.input
|
||||
:param body: stuffed in a WsgiBytesIO and hung on wsgi.input
|
||||
:param kwargs: any environ key with an property setter
|
||||
"""
|
||||
headers = headers or {}
|
||||
|
@ -864,16 +866,16 @@ class Request(object):
|
|||
'SERVER_PROTOCOL': 'HTTP/1.0',
|
||||
'wsgi.version': (1, 0),
|
||||
'wsgi.url_scheme': parsed_path.scheme or 'http',
|
||||
'wsgi.errors': StringIO(''),
|
||||
'wsgi.errors': StringIO(),
|
||||
'wsgi.multithread': False,
|
||||
'wsgi.multiprocess': False
|
||||
}
|
||||
env.update(environ)
|
||||
if body is not None:
|
||||
env['wsgi.input'] = WsgiStringIO(body)
|
||||
env['wsgi.input'] = WsgiBytesIO(body)
|
||||
env['CONTENT_LENGTH'] = str(len(body))
|
||||
elif 'wsgi.input' not in env:
|
||||
env['wsgi.input'] = WsgiStringIO('')
|
||||
env['wsgi.input'] = WsgiBytesIO()
|
||||
req = Request(env)
|
||||
for key, val in headers.items():
|
||||
req.headers[key] = val
|
||||
|
@ -980,7 +982,7 @@ class Request(object):
|
|||
env.update({
|
||||
'REQUEST_METHOD': 'GET',
|
||||
'CONTENT_LENGTH': '0',
|
||||
'wsgi.input': WsgiStringIO(''),
|
||||
'wsgi.input': WsgiBytesIO(),
|
||||
})
|
||||
return Request(env)
|
||||
|
||||
|
@ -1127,6 +1129,7 @@ class Response(object):
|
|||
self.request = request
|
||||
self.body = body
|
||||
self.app_iter = app_iter
|
||||
self.response_iter = None
|
||||
self.status = status
|
||||
self.boundary = "%.32x" % random.randint(0, 256 ** 16)
|
||||
if request:
|
||||
|
@ -1322,6 +1325,17 @@ class Response(object):
|
|||
return [body]
|
||||
return ['']
|
||||
|
||||
def fix_conditional_response(self):
|
||||
"""
|
||||
You may call this once you have set the content_length to the whole
|
||||
object length and body or app_iter to reset the content_length
|
||||
properties on the request.
|
||||
|
||||
It is ok to not call this method, the conditional resposne will be
|
||||
maintained for you when you __call__ the response.
|
||||
"""
|
||||
self.response_iter = self._response_iter(self.app_iter, self._body)
|
||||
|
||||
def absolute_location(self):
|
||||
"""
|
||||
Attempt to construct an absolute location.
|
||||
|
@ -1372,12 +1386,15 @@ class Response(object):
|
|||
if not self.request:
|
||||
self.request = Request(env)
|
||||
self.environ = env
|
||||
app_iter = self._response_iter(self.app_iter, self._body)
|
||||
|
||||
if not self.response_iter:
|
||||
self.response_iter = self._response_iter(self.app_iter, self._body)
|
||||
|
||||
if 'location' in self.headers and \
|
||||
not env.get('swift.leave_relative_location'):
|
||||
self.location = self.absolute_location()
|
||||
start_response(self.status, self.headers.items())
|
||||
return app_iter
|
||||
return self.response_iter
|
||||
|
||||
|
||||
class HTTPException(Response, Exception):
|
||||
|
|
|
@ -38,16 +38,14 @@ from urllib import quote as _quote
|
|||
from contextlib import contextmanager, closing
|
||||
import ctypes
|
||||
import ctypes.util
|
||||
from ConfigParser import ConfigParser, NoSectionError, NoOptionError, \
|
||||
RawConfigParser
|
||||
from optparse import OptionParser
|
||||
from Queue import Queue, Empty
|
||||
|
||||
from tempfile import mkstemp, NamedTemporaryFile
|
||||
try:
|
||||
import simplejson as json
|
||||
except ImportError:
|
||||
import json
|
||||
import cPickle as pickle
|
||||
import six.moves.cPickle as pickle
|
||||
import glob
|
||||
from urlparse import urlparse as stdlib_urlparse, ParseResult
|
||||
import itertools
|
||||
|
@ -64,6 +62,9 @@ import netifaces
|
|||
import codecs
|
||||
utf8_decoder = codecs.getdecoder('utf-8')
|
||||
utf8_encoder = codecs.getencoder('utf-8')
|
||||
from six.moves.configparser import ConfigParser, NoSectionError, \
|
||||
NoOptionError, RawConfigParser
|
||||
from six.moves.queue import Queue, Empty
|
||||
from six.moves import range
|
||||
|
||||
from swift import gettext_ as _
|
||||
|
@ -568,9 +569,9 @@ class FallocateWrapper(object):
|
|||
self.func_name = 'posix_fallocate'
|
||||
self.fallocate = noop_libc_function
|
||||
return
|
||||
## fallocate is preferred because we need the on-disk size to match
|
||||
## the allocated size. Older versions of sqlite require that the
|
||||
## two sizes match. However, fallocate is Linux only.
|
||||
# fallocate is preferred because we need the on-disk size to match
|
||||
# the allocated size. Older versions of sqlite require that the
|
||||
# two sizes match. However, fallocate is Linux only.
|
||||
for func in ('fallocate', 'posix_fallocate'):
|
||||
self.func_name = func
|
||||
self.fallocate = load_libc_function(func, log_error=False)
|
||||
|
@ -1414,7 +1415,7 @@ class SwiftLogFormatter(logging.Formatter):
|
|||
if self.max_line_length < 7:
|
||||
msg = msg[:self.max_line_length]
|
||||
else:
|
||||
approxhalf = (self.max_line_length - 5) / 2
|
||||
approxhalf = (self.max_line_length - 5) // 2
|
||||
msg = msg[:approxhalf] + " ... " + msg[-approxhalf:]
|
||||
return msg
|
||||
|
||||
|
@ -2267,6 +2268,7 @@ class GreenAsyncPile(object):
|
|||
size = size_or_pool
|
||||
self._responses = eventlet.queue.LightQueue(size)
|
||||
self._inflight = 0
|
||||
self._pending = 0
|
||||
|
||||
def _run_func(self, func, args, kwargs):
|
||||
try:
|
||||
|
@ -2278,6 +2280,7 @@ class GreenAsyncPile(object):
|
|||
"""
|
||||
Spawn a job in a green thread on the pile.
|
||||
"""
|
||||
self._pending += 1
|
||||
self._inflight += 1
|
||||
self._pool.spawn(self._run_func, func, args, kwargs)
|
||||
|
||||
|
@ -2302,12 +2305,13 @@ class GreenAsyncPile(object):
|
|||
|
||||
def next(self):
|
||||
try:
|
||||
return self._responses.get_nowait()
|
||||
rv = self._responses.get_nowait()
|
||||
except Empty:
|
||||
if self._inflight == 0:
|
||||
raise StopIteration()
|
||||
else:
|
||||
return self._responses.get()
|
||||
rv = self._responses.get()
|
||||
self._pending -= 1
|
||||
return rv
|
||||
|
||||
|
||||
class ModifiedParseResult(ParseResult):
|
||||
|
|
|
@ -24,7 +24,6 @@ import signal
|
|||
import time
|
||||
import mimetools
|
||||
from swift import gettext_ as _
|
||||
from StringIO import StringIO
|
||||
from textwrap import dedent
|
||||
|
||||
import eventlet
|
||||
|
@ -32,6 +31,8 @@ import eventlet.debug
|
|||
from eventlet import greenio, GreenPool, sleep, wsgi, listen, Timeout
|
||||
from paste.deploy import loadwsgi
|
||||
from eventlet.green import socket, ssl, os as green_os
|
||||
from six import BytesIO
|
||||
from six import StringIO
|
||||
from urllib import unquote
|
||||
|
||||
from swift.common import utils, constraints
|
||||
|
@ -460,10 +461,14 @@ class WorkersStrategy(object):
|
|||
|
||||
def loop_timeout(self):
|
||||
"""
|
||||
:returns: None; to block in :py:func:`green.os.wait`
|
||||
We want to keep from busy-waiting, but we also need a non-None value so
|
||||
the main loop gets a chance to tell whether it should keep running or
|
||||
not (e.g. SIGHUP received).
|
||||
|
||||
So we return 0.5.
|
||||
"""
|
||||
|
||||
return None
|
||||
return 0.5
|
||||
|
||||
def bind_ports(self):
|
||||
"""
|
||||
|
@ -1079,13 +1084,13 @@ def make_env(env, method=None, path=None, agent='Swift', query_string=None,
|
|||
:returns: Fresh WSGI environment.
|
||||
"""
|
||||
newenv = {}
|
||||
for name in ('eventlet.posthooks', 'HTTP_USER_AGENT', 'HTTP_HOST',
|
||||
'PATH_INFO', 'QUERY_STRING', 'REMOTE_USER', 'REQUEST_METHOD',
|
||||
for name in ('HTTP_USER_AGENT', 'HTTP_HOST', 'PATH_INFO',
|
||||
'QUERY_STRING', 'REMOTE_USER', 'REQUEST_METHOD',
|
||||
'SCRIPT_NAME', 'SERVER_NAME', 'SERVER_PORT',
|
||||
'HTTP_ORIGIN', 'HTTP_ACCESS_CONTROL_REQUEST_METHOD',
|
||||
'SERVER_PROTOCOL', 'swift.cache', 'swift.source',
|
||||
'swift.trans_id', 'swift.authorize_override',
|
||||
'swift.authorize'):
|
||||
'swift.authorize', 'HTTP_X_USER_ID', 'HTTP_X_PROJECT_ID'):
|
||||
if name in env:
|
||||
newenv[name] = env[name]
|
||||
if method:
|
||||
|
@ -1102,7 +1107,7 @@ def make_env(env, method=None, path=None, agent='Swift', query_string=None,
|
|||
del newenv['HTTP_USER_AGENT']
|
||||
if swift_source:
|
||||
newenv['swift.source'] = swift_source
|
||||
newenv['wsgi.input'] = StringIO('')
|
||||
newenv['wsgi.input'] = BytesIO()
|
||||
if 'SCRIPT_NAME' not in newenv:
|
||||
newenv['SCRIPT_NAME'] = ''
|
||||
return newenv
|
||||
|
|
|
@ -19,7 +19,7 @@ Pluggable Back-ends for Container Server
|
|||
import os
|
||||
from uuid import uuid4
|
||||
import time
|
||||
import cPickle as pickle
|
||||
import six.moves.cPickle as pickle
|
||||
|
||||
from six.moves import range
|
||||
import sqlite3
|
||||
|
|
|
@ -185,7 +185,11 @@ class ContainerController(BaseStorageServer):
|
|||
return HTTPBadRequest(req=req)
|
||||
|
||||
if account_partition:
|
||||
updates = zip(account_hosts, account_devices)
|
||||
# zip is lazy on py3, but we need a list, so force evaluation.
|
||||
# On py2 it's an extra list copy, but the list is so small
|
||||
# (one element per replica in account ring, usually 3) that it
|
||||
# doesn't matter.
|
||||
updates = list(zip(account_hosts, account_devices))
|
||||
else:
|
||||
updates = []
|
||||
|
||||
|
|
|
@ -13,6 +13,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
|
@ -254,8 +255,8 @@ class ContainerUpdater(Daemon):
|
|||
self.account_suppressions[info['account']] = until = \
|
||||
time.time() + self.account_suppression_time
|
||||
if self.new_account_suppressions:
|
||||
print >>self.new_account_suppressions, \
|
||||
info['account'], until
|
||||
print(info['account'], until,
|
||||
file=self.new_account_suppressions)
|
||||
# Only track timing data for attempted updates:
|
||||
self.logger.timing_since('timing', start_time)
|
||||
else:
|
||||
|
|
|
@ -1,19 +1,19 @@
|
|||
# Translations template for swift.
|
||||
# Copyright (C) 2014 ORGANIZATION
|
||||
# Copyright (C) 2015 ORGANIZATION
|
||||
# This file is distributed under the same license as the swift project.
|
||||
# FIRST AUTHOR <EMAIL@ADDRESS>, 2014.
|
||||
# FIRST AUTHOR <EMAIL@ADDRESS>, 2015.
|
||||
#
|
||||
#, fuzzy
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: swift 2.1.0.77.g0d0c16d\n"
|
||||
"Project-Id-Version: swift 2.3.1.dev213\n"
|
||||
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
||||
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
|
||||
"POT-Creation-Date: 2015-07-29 06:35+0000\n"
|
||||
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
|
||||
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
|
||||
"Language-Team: LANGUAGE <LL@li.org>\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=utf-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Generated-By: Babel 1.3\n"
|
||||
"Generated-By: Babel 2.0\n"
|
||||
|
||||
|
|
|
@ -1,19 +1,19 @@
|
|||
# Translations template for swift.
|
||||
# Copyright (C) 2014 ORGANIZATION
|
||||
# Copyright (C) 2015 ORGANIZATION
|
||||
# This file is distributed under the same license as the swift project.
|
||||
# FIRST AUTHOR <EMAIL@ADDRESS>, 2014.
|
||||
# FIRST AUTHOR <EMAIL@ADDRESS>, 2015.
|
||||
#
|
||||
#, fuzzy
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: swift 2.1.0.77.g0d0c16d\n"
|
||||
"Project-Id-Version: swift 2.3.1.dev213\n"
|
||||
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
||||
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
|
||||
"POT-Creation-Date: 2015-07-29 06:35+0000\n"
|
||||
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
|
||||
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
|
||||
"Language-Team: LANGUAGE <LL@li.org>\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=utf-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Generated-By: Babel 1.3\n"
|
||||
"Generated-By: Babel 2.0\n"
|
||||
|
||||
|
|
|
@ -1,19 +1,19 @@
|
|||
# Translations template for swift.
|
||||
# Copyright (C) 2014 ORGANIZATION
|
||||
# Copyright (C) 2015 ORGANIZATION
|
||||
# This file is distributed under the same license as the swift project.
|
||||
# FIRST AUTHOR <EMAIL@ADDRESS>, 2014.
|
||||
# FIRST AUTHOR <EMAIL@ADDRESS>, 2015.
|
||||
#
|
||||
#, fuzzy
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: swift 2.1.0.77.g0d0c16d\n"
|
||||
"Project-Id-Version: swift 2.3.1.dev213\n"
|
||||
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
||||
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
|
||||
"POT-Creation-Date: 2015-07-29 06:35+0000\n"
|
||||
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
|
||||
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
|
||||
"Language-Team: LANGUAGE <LL@li.org>\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=utf-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Generated-By: Babel 1.3\n"
|
||||
"Generated-By: Babel 2.0\n"
|
||||
|
||||
|
|
|
@ -1,19 +1,19 @@
|
|||
# Translations template for swift.
|
||||
# Copyright (C) 2014 ORGANIZATION
|
||||
# Copyright (C) 2015 ORGANIZATION
|
||||
# This file is distributed under the same license as the swift project.
|
||||
# FIRST AUTHOR <EMAIL@ADDRESS>, 2014.
|
||||
# FIRST AUTHOR <EMAIL@ADDRESS>, 2015.
|
||||
#
|
||||
#, fuzzy
|
||||
msgid ""
|
||||
msgstr ""
|
||||
"Project-Id-Version: swift 2.1.0.77.g0d0c16d\n"
|
||||
"Project-Id-Version: swift 2.3.1.dev213\n"
|
||||
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
||||
"POT-Creation-Date: 2014-09-22 06:07+0000\n"
|
||||
"POT-Creation-Date: 2015-07-29 06:35+0000\n"
|
||||
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
|
||||
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
|
||||
"Language-Team: LANGUAGE <LL@li.org>\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=utf-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Generated-By: Babel 1.3\n"
|
||||
"Generated-By: Babel 2.0\n"
|
||||
|
||||
|
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
@ -8,16 +8,16 @@ msgid ""
|
|||
msgstr ""
|
||||
"Project-Id-Version: Swift\n"
|
||||
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
||||
"POT-Creation-Date: 2015-07-09 06:14+0000\n"
|
||||
"PO-Revision-Date: 2015-07-09 05:58+0000\n"
|
||||
"POT-Creation-Date: 2015-08-04 06:29+0000\n"
|
||||
"PO-Revision-Date: 2015-07-28 00:33+0000\n"
|
||||
"Last-Translator: openstackjenkins <jenkins@openstack.org>\n"
|
||||
"Language-Team: Chinese (China) (http://www.transifex.com/p/swift/language/"
|
||||
"zh_CN/)\n"
|
||||
"Language-Team: Chinese (China) (http://www.transifex.com/openstack/swift/"
|
||||
"language/zh_CN/)\n"
|
||||
"Plural-Forms: nplurals=1; plural=0\n"
|
||||
"MIME-Version: 1.0\n"
|
||||
"Content-Type: text/plain; charset=utf-8\n"
|
||||
"Content-Transfer-Encoding: 8bit\n"
|
||||
"Generated-By: Babel 1.3\n"
|
||||
"Generated-By: Babel 2.0\n"
|
||||
|
||||
msgid ""
|
||||
"\n"
|
||||
|
|
|
@ -79,7 +79,7 @@ class AuditorWorker(object):
|
|||
else:
|
||||
description = _(' - %s') % device_dir_str
|
||||
self.logger.info(_('Begin object audit "%s" mode (%s%s)') %
|
||||
(mode, self.auditor_type, description))
|
||||
(mode, self.auditor_type, description))
|
||||
begin = reported = time.time()
|
||||
self.total_bytes_processed = 0
|
||||
self.total_files_processed = 0
|
||||
|
@ -331,7 +331,7 @@ class ObjectAuditor(Daemon):
|
|||
try:
|
||||
self.audit_loop(parent, zbo_fps, **kwargs)
|
||||
except (Exception, Timeout) as err:
|
||||
self.logger.exception(_('ERROR auditing: %s' % err))
|
||||
self.logger.exception(_('ERROR auditing: %s'), err)
|
||||
self._sleep()
|
||||
|
||||
def run_once(self, *args, **kwargs):
|
||||
|
@ -352,4 +352,4 @@ class ObjectAuditor(Daemon):
|
|||
self.audit_loop(parent, zbo_fps, override_devices=override_devices,
|
||||
**kwargs)
|
||||
except (Exception, Timeout) as err:
|
||||
self.logger.exception(_('ERROR auditing: %s' % err))
|
||||
self.logger.exception(_('ERROR auditing: %s'), err)
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -15,12 +15,12 @@
|
|||
|
||||
""" In-Memory Disk File Interface for Swift Object Server"""
|
||||
|
||||
import cStringIO
|
||||
import time
|
||||
import hashlib
|
||||
from contextlib import contextmanager
|
||||
|
||||
from eventlet import Timeout
|
||||
from six import moves
|
||||
|
||||
from swift.common.utils import Timestamp
|
||||
from swift.common.exceptions import DiskFileQuarantined, DiskFileNotExist, \
|
||||
|
@ -385,7 +385,7 @@ class DiskFile(object):
|
|||
disk
|
||||
:raises DiskFileNoSpace: if a size is specified and allocation fails
|
||||
"""
|
||||
fp = cStringIO.StringIO()
|
||||
fp = moves.cStringIO()
|
||||
try:
|
||||
yield DiskFileWriter(self._filesystem, self._name, fp)
|
||||
finally:
|
||||
|
|
|
@ -19,7 +19,7 @@ import random
|
|||
import time
|
||||
import itertools
|
||||
from collections import defaultdict
|
||||
import cPickle as pickle
|
||||
import six.moves.cPickle as pickle
|
||||
import shutil
|
||||
|
||||
from eventlet import (GreenPile, GreenPool, Timeout, sleep, hubs, tpool,
|
||||
|
@ -319,11 +319,11 @@ class ObjectReconstructor(Daemon):
|
|||
except (Exception, Timeout):
|
||||
self.logger.exception(
|
||||
_("Error trying to rebuild %(path)s "
|
||||
"policy#%(policy)d frag#%(frag_index)s"), {
|
||||
'path': path,
|
||||
'policy': policy,
|
||||
'frag_index': frag_index,
|
||||
})
|
||||
"policy#%(policy)d frag#%(frag_index)s"),
|
||||
{'path': path,
|
||||
'policy': policy,
|
||||
'frag_index': frag_index,
|
||||
})
|
||||
break
|
||||
if not all(fragment_payload):
|
||||
break
|
||||
|
@ -337,22 +337,34 @@ class ObjectReconstructor(Daemon):
|
|||
"""
|
||||
Logs various stats for the currently running reconstruction pass.
|
||||
"""
|
||||
if self.reconstruction_count:
|
||||
if (self.device_count and self.part_count and
|
||||
self.reconstruction_device_count):
|
||||
elapsed = (time.time() - self.start) or 0.000001
|
||||
rate = self.reconstruction_count / elapsed
|
||||
rate = self.reconstruction_part_count / elapsed
|
||||
total_part_count = (self.part_count *
|
||||
self.device_count /
|
||||
self.reconstruction_device_count)
|
||||
self.logger.info(
|
||||
_("%(reconstructed)d/%(total)d (%(percentage).2f%%)"
|
||||
" partitions reconstructed in %(time).2fs (%(rate).2f/sec, "
|
||||
"%(remaining)s remaining)"),
|
||||
{'reconstructed': self.reconstruction_count,
|
||||
'total': self.job_count,
|
||||
" partitions of %(device)d/%(dtotal)d "
|
||||
"(%(dpercentage).2f%%) devices"
|
||||
" reconstructed in %(time).2fs "
|
||||
"(%(rate).2f/sec, %(remaining)s remaining)"),
|
||||
{'reconstructed': self.reconstruction_part_count,
|
||||
'total': self.part_count,
|
||||
'percentage':
|
||||
self.reconstruction_count * 100.0 / self.job_count,
|
||||
self.reconstruction_part_count * 100.0 / self.part_count,
|
||||
'device': self.reconstruction_device_count,
|
||||
'dtotal': self.device_count,
|
||||
'dpercentage':
|
||||
self.reconstruction_device_count * 100.0 / self.device_count,
|
||||
'time': time.time() - self.start, 'rate': rate,
|
||||
'remaining': '%d%s' % compute_eta(self.start,
|
||||
self.reconstruction_count,
|
||||
self.job_count)})
|
||||
if self.suffix_count:
|
||||
'remaining': '%d%s' %
|
||||
compute_eta(self.start,
|
||||
self.reconstruction_part_count,
|
||||
total_part_count)})
|
||||
|
||||
if self.suffix_count and self.partition_times:
|
||||
self.logger.info(
|
||||
_("%(checked)d suffixes checked - "
|
||||
"%(hashed).2f%% hashed, %(synced).2f%% synced"),
|
||||
|
@ -474,14 +486,11 @@ class ObjectReconstructor(Daemon):
|
|||
self._full_path(node, job['partition'], '',
|
||||
job['policy']))
|
||||
elif resp.status != HTTP_OK:
|
||||
full_path = self._full_path(node, job['partition'], '',
|
||||
job['policy'])
|
||||
self.logger.error(
|
||||
_("Invalid response %(resp)s "
|
||||
"from %(full_path)s"), {
|
||||
'resp': resp.status,
|
||||
'full_path': self._full_path(
|
||||
node, job['partition'], '',
|
||||
job['policy'])
|
||||
})
|
||||
_("Invalid response %(resp)s from %(full_path)s"),
|
||||
{'resp': resp.status, 'full_path': full_path})
|
||||
else:
|
||||
remote_suffixes = pickle.loads(resp.read())
|
||||
except (Exception, Timeout):
|
||||
|
@ -781,16 +790,22 @@ class ObjectReconstructor(Daemon):
|
|||
self._diskfile_mgr = self._df_router[policy]
|
||||
self.load_object_ring(policy)
|
||||
data_dir = get_data_dir(policy)
|
||||
local_devices = itertools.ifilter(
|
||||
local_devices = list(itertools.ifilter(
|
||||
lambda dev: dev and is_local_device(
|
||||
ips, self.port,
|
||||
dev['replication_ip'], dev['replication_port']),
|
||||
policy.object_ring.devs)
|
||||
policy.object_ring.devs))
|
||||
|
||||
if override_devices:
|
||||
self.device_count = len(override_devices)
|
||||
else:
|
||||
self.device_count = len(local_devices)
|
||||
|
||||
for local_dev in local_devices:
|
||||
if override_devices and (local_dev['device'] not in
|
||||
override_devices):
|
||||
continue
|
||||
self.reconstruction_device_count += 1
|
||||
dev_path = self._df_router[policy].get_dev_path(
|
||||
local_dev['device'])
|
||||
if not dev_path:
|
||||
|
@ -814,6 +829,8 @@ class ObjectReconstructor(Daemon):
|
|||
self.logger.exception(
|
||||
'Unable to list partitions in %r' % obj_path)
|
||||
continue
|
||||
|
||||
self.part_count += len(partitions)
|
||||
for partition in partitions:
|
||||
part_path = join(obj_path, partition)
|
||||
if not (partition.isdigit() and
|
||||
|
@ -821,6 +838,7 @@ class ObjectReconstructor(Daemon):
|
|||
self.logger.warning(
|
||||
'Unexpected entity in data dir: %r' % part_path)
|
||||
remove_file(part_path)
|
||||
self.reconstruction_part_count += 1
|
||||
continue
|
||||
partition = int(partition)
|
||||
if override_partitions and (partition not in
|
||||
|
@ -833,6 +851,7 @@ class ObjectReconstructor(Daemon):
|
|||
'part_path': part_path,
|
||||
}
|
||||
yield part_info
|
||||
self.reconstruction_part_count += 1
|
||||
|
||||
def build_reconstruction_jobs(self, part_info):
|
||||
"""
|
||||
|
@ -850,10 +869,14 @@ class ObjectReconstructor(Daemon):
|
|||
def _reset_stats(self):
|
||||
self.start = time.time()
|
||||
self.job_count = 0
|
||||
self.part_count = 0
|
||||
self.device_count = 0
|
||||
self.suffix_count = 0
|
||||
self.suffix_sync = 0
|
||||
self.suffix_hash = 0
|
||||
self.reconstruction_count = 0
|
||||
self.reconstruction_part_count = 0
|
||||
self.reconstruction_device_count = 0
|
||||
self.last_reconstruction_count = -1
|
||||
|
||||
def delete_partition(self, path):
|
||||
|
|
|
@ -20,7 +20,7 @@ import random
|
|||
import shutil
|
||||
import time
|
||||
import itertools
|
||||
import cPickle as pickle
|
||||
import six.moves.cPickle as pickle
|
||||
from swift import gettext_ as _
|
||||
|
||||
import eventlet
|
||||
|
@ -37,8 +37,7 @@ from swift.common.bufferedhttp import http_connect
|
|||
from swift.common.daemon import Daemon
|
||||
from swift.common.http import HTTP_OK, HTTP_INSUFFICIENT_STORAGE
|
||||
from swift.obj import ssync_sender
|
||||
from swift.obj.diskfile import (DiskFileManager, get_hashes, get_data_dir,
|
||||
get_tmp_dir)
|
||||
from swift.obj.diskfile import DiskFileManager, get_data_dir, get_tmp_dir
|
||||
from swift.common.storage_policy import POLICIES, REPL_POLICY
|
||||
|
||||
|
||||
|
@ -54,13 +53,13 @@ class ObjectReplicator(Daemon):
|
|||
caller to do this in a loop.
|
||||
"""
|
||||
|
||||
def __init__(self, conf):
|
||||
def __init__(self, conf, logger=None):
|
||||
"""
|
||||
:param conf: configuration object obtained from ConfigParser
|
||||
:param logger: logging object
|
||||
"""
|
||||
self.conf = conf
|
||||
self.logger = get_logger(conf, log_route='object-replicator')
|
||||
self.logger = logger or get_logger(conf, log_route='object-replicator')
|
||||
self.devices_dir = conf.get('devices', '/srv/node')
|
||||
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
|
||||
self.vm_test_mode = config_true_value(conf.get('vm_test_mode', 'no'))
|
||||
|
@ -91,7 +90,7 @@ class ObjectReplicator(Daemon):
|
|||
self.node_timeout = float(conf.get('node_timeout', 10))
|
||||
self.sync_method = getattr(self, conf.get('sync_method') or 'rsync')
|
||||
self.network_chunk_size = int(conf.get('network_chunk_size', 65536))
|
||||
self.headers = {
|
||||
self.default_headers = {
|
||||
'Content-Length': '0',
|
||||
'user-agent': 'object-replicator %s' % os.getpid()}
|
||||
self.rsync_error_log_line_length = \
|
||||
|
@ -100,8 +99,37 @@ class ObjectReplicator(Daemon):
|
|||
False))
|
||||
self.handoff_delete = config_auto_int_value(
|
||||
conf.get('handoff_delete', 'auto'), 0)
|
||||
if any((self.handoff_delete, self.handoffs_first)):
|
||||
self.logger.warn('Handoff only mode is not intended for normal '
|
||||
'operation, please disable handoffs_first and '
|
||||
'handoff_delete before the next '
|
||||
'normal rebalance')
|
||||
self._diskfile_mgr = DiskFileManager(conf, self.logger)
|
||||
|
||||
def _zero_stats(self):
|
||||
"""Zero out the stats."""
|
||||
self.stats = {'attempted': 0, 'success': 0, 'failure': 0,
|
||||
'hashmatch': 0, 'rsync': 0, 'remove': 0,
|
||||
'start': time.time(), 'failure_nodes': {}}
|
||||
|
||||
def _add_failure_stats(self, failure_devs_info):
|
||||
for node, dev in failure_devs_info:
|
||||
self.stats['failure'] += 1
|
||||
failure_devs = self.stats['failure_nodes'].setdefault(node, {})
|
||||
failure_devs.setdefault(dev, 0)
|
||||
failure_devs[dev] += 1
|
||||
|
||||
def _get_my_replication_ips(self):
|
||||
my_replication_ips = set()
|
||||
ips = whataremyips()
|
||||
for policy in POLICIES:
|
||||
self.load_object_ring(policy)
|
||||
for local_dev in [dev for dev in policy.object_ring.devs
|
||||
if dev and dev['replication_ip'] in ips and
|
||||
dev['replication_port'] == self.port]:
|
||||
my_replication_ips.add(local_dev['replication_ip'])
|
||||
return list(my_replication_ips)
|
||||
|
||||
# Just exists for doc anchor point
|
||||
def sync(self, node, job, suffixes, *args, **kwargs):
|
||||
"""
|
||||
|
@ -243,7 +271,9 @@ class ObjectReplicator(Daemon):
|
|||
if len(suff) == 3 and isdir(join(path, suff))]
|
||||
self.replication_count += 1
|
||||
self.logger.increment('partition.delete.count.%s' % (job['device'],))
|
||||
self.headers['X-Backend-Storage-Policy-Index'] = int(job['policy'])
|
||||
headers = dict(self.default_headers)
|
||||
headers['X-Backend-Storage-Policy-Index'] = int(job['policy'])
|
||||
failure_devs_info = set()
|
||||
begin = time.time()
|
||||
try:
|
||||
responses = []
|
||||
|
@ -252,6 +282,7 @@ class ObjectReplicator(Daemon):
|
|||
delete_objs = None
|
||||
if suffixes:
|
||||
for node in job['nodes']:
|
||||
self.stats['rsync'] += 1
|
||||
kwargs = {}
|
||||
if node['region'] in synced_remote_regions and \
|
||||
self.conf.get('sync_method', 'rsync') == 'ssync':
|
||||
|
@ -267,11 +298,14 @@ class ObjectReplicator(Daemon):
|
|||
node['replication_ip'],
|
||||
node['replication_port'],
|
||||
node['device'], job['partition'], 'REPLICATE',
|
||||
'/' + '-'.join(suffixes), headers=self.headers)
|
||||
'/' + '-'.join(suffixes), headers=headers)
|
||||
conn.getresponse().read()
|
||||
if node['region'] != job['region']:
|
||||
synced_remote_regions[node['region']] = \
|
||||
candidates.keys()
|
||||
else:
|
||||
failure_devs_info.add((node['replication_ip'],
|
||||
node['device']))
|
||||
responses.append(success)
|
||||
for region, cand_objs in synced_remote_regions.items():
|
||||
if delete_objs is None:
|
||||
|
@ -287,11 +321,23 @@ class ObjectReplicator(Daemon):
|
|||
delete_handoff = len(responses) == len(job['nodes']) and \
|
||||
all(responses)
|
||||
if delete_handoff:
|
||||
self.stats['remove'] += 1
|
||||
if (self.conf.get('sync_method', 'rsync') == 'ssync' and
|
||||
delete_objs is not None):
|
||||
self.logger.info(_("Removing %s objects"),
|
||||
len(delete_objs))
|
||||
self.delete_handoff_objs(job, delete_objs)
|
||||
_junk, error_paths = self.delete_handoff_objs(
|
||||
job, delete_objs)
|
||||
# if replication works for a hand-off device and it failed,
|
||||
# the remote devices which are target of the replication
|
||||
# from the hand-off device will be marked. Because cleanup
|
||||
# after replication failed means replicator needs to
|
||||
# replicate again with the same info.
|
||||
if error_paths:
|
||||
failure_devs_info.update(
|
||||
[(failure_dev['replication_ip'],
|
||||
failure_dev['device'])
|
||||
for failure_dev in job['nodes']])
|
||||
else:
|
||||
self.delete_partition(job['path'])
|
||||
elif not suffixes:
|
||||
|
@ -299,14 +345,21 @@ class ObjectReplicator(Daemon):
|
|||
except (Exception, Timeout):
|
||||
self.logger.exception(_("Error syncing handoff partition"))
|
||||
finally:
|
||||
target_devs_info = set([(target_dev['replication_ip'],
|
||||
target_dev['device'])
|
||||
for target_dev in job['nodes']])
|
||||
self.stats['success'] += len(target_devs_info - failure_devs_info)
|
||||
self._add_failure_stats(failure_devs_info)
|
||||
self.partition_times.append(time.time() - begin)
|
||||
self.logger.timing_since('partition.delete.timing', begin)
|
||||
|
||||
def delete_partition(self, path):
|
||||
self.logger.info(_("Removing partition: %s"), path)
|
||||
tpool.execute(shutil.rmtree, path, ignore_errors=True)
|
||||
tpool.execute(shutil.rmtree, path)
|
||||
|
||||
def delete_handoff_objs(self, job, delete_objs):
|
||||
success_paths = []
|
||||
error_paths = []
|
||||
for object_hash in delete_objs:
|
||||
object_path = storage_directory(job['obj_path'], job['partition'],
|
||||
object_hash)
|
||||
|
@ -314,11 +367,14 @@ class ObjectReplicator(Daemon):
|
|||
suffix_dir = dirname(object_path)
|
||||
try:
|
||||
os.rmdir(suffix_dir)
|
||||
success_paths.append(object_path)
|
||||
except OSError as e:
|
||||
if e.errno not in (errno.ENOENT, errno.ENOTEMPTY):
|
||||
error_paths.append(object_path)
|
||||
self.logger.exception(
|
||||
"Unexpected error trying to cleanup suffix dir:%r",
|
||||
suffix_dir)
|
||||
return success_paths, error_paths
|
||||
|
||||
def update(self, job):
|
||||
"""
|
||||
|
@ -328,11 +384,14 @@ class ObjectReplicator(Daemon):
|
|||
"""
|
||||
self.replication_count += 1
|
||||
self.logger.increment('partition.update.count.%s' % (job['device'],))
|
||||
self.headers['X-Backend-Storage-Policy-Index'] = int(job['policy'])
|
||||
headers = dict(self.default_headers)
|
||||
headers['X-Backend-Storage-Policy-Index'] = int(job['policy'])
|
||||
target_devs_info = set()
|
||||
failure_devs_info = set()
|
||||
begin = time.time()
|
||||
try:
|
||||
hashed, local_hash = tpool_reraise(
|
||||
get_hashes, job['path'],
|
||||
self._diskfile_mgr._get_hashes, job['path'],
|
||||
do_listdir=(self.replication_count % 10) == 0,
|
||||
reclaim_age=self.reclaim_age)
|
||||
self.suffix_hash += hashed
|
||||
|
@ -347,6 +406,7 @@ class ObjectReplicator(Daemon):
|
|||
while attempts_left > 0:
|
||||
# If this throws StopIteration it will be caught way below
|
||||
node = next(nodes)
|
||||
target_devs_info.add((node['replication_ip'], node['device']))
|
||||
attempts_left -= 1
|
||||
# if we have already synced to this remote region,
|
||||
# don't sync again on this replication pass
|
||||
|
@ -357,17 +417,21 @@ class ObjectReplicator(Daemon):
|
|||
resp = http_connect(
|
||||
node['replication_ip'], node['replication_port'],
|
||||
node['device'], job['partition'], 'REPLICATE',
|
||||
'', headers=self.headers).getresponse()
|
||||
'', headers=headers).getresponse()
|
||||
if resp.status == HTTP_INSUFFICIENT_STORAGE:
|
||||
self.logger.error(_('%(ip)s/%(device)s responded'
|
||||
' as unmounted'), node)
|
||||
attempts_left += 1
|
||||
failure_devs_info.add((node['replication_ip'],
|
||||
node['device']))
|
||||
continue
|
||||
if resp.status != HTTP_OK:
|
||||
self.logger.error(_("Invalid response %(resp)s "
|
||||
"from %(ip)s"),
|
||||
{'resp': resp.status,
|
||||
'ip': node['replication_ip']})
|
||||
failure_devs_info.add((node['replication_ip'],
|
||||
node['device']))
|
||||
continue
|
||||
remote_hash = pickle.loads(resp.read())
|
||||
del resp
|
||||
|
@ -375,9 +439,10 @@ class ObjectReplicator(Daemon):
|
|||
local_hash[suffix] !=
|
||||
remote_hash.get(suffix, -1)]
|
||||
if not suffixes:
|
||||
self.stats['hashmatch'] += 1
|
||||
continue
|
||||
hashed, recalc_hash = tpool_reraise(
|
||||
get_hashes,
|
||||
self._diskfile_mgr._get_hashes,
|
||||
job['path'], recalculate=suffixes,
|
||||
reclaim_age=self.reclaim_age)
|
||||
self.logger.update_stats('suffix.hashes', hashed)
|
||||
|
@ -385,26 +450,35 @@ class ObjectReplicator(Daemon):
|
|||
suffixes = [suffix for suffix in local_hash if
|
||||
local_hash[suffix] !=
|
||||
remote_hash.get(suffix, -1)]
|
||||
self.stats['rsync'] += 1
|
||||
success, _junk = self.sync(node, job, suffixes)
|
||||
with Timeout(self.http_timeout):
|
||||
conn = http_connect(
|
||||
node['replication_ip'], node['replication_port'],
|
||||
node['device'], job['partition'], 'REPLICATE',
|
||||
'/' + '-'.join(suffixes),
|
||||
headers=self.headers)
|
||||
headers=headers)
|
||||
conn.getresponse().read()
|
||||
if not success:
|
||||
failure_devs_info.add((node['replication_ip'],
|
||||
node['device']))
|
||||
# add only remote region when replicate succeeded
|
||||
if success and node['region'] != job['region']:
|
||||
synced_remote_regions.add(node['region'])
|
||||
self.suffix_sync += len(suffixes)
|
||||
self.logger.update_stats('suffix.syncs', len(suffixes))
|
||||
except (Exception, Timeout):
|
||||
failure_devs_info.add((node['replication_ip'],
|
||||
node['device']))
|
||||
self.logger.exception(_("Error syncing with node: %s") %
|
||||
node)
|
||||
self.suffix_count += len(local_hash)
|
||||
except (Exception, Timeout):
|
||||
failure_devs_info.update(target_devs_info)
|
||||
self.logger.exception(_("Error syncing partition"))
|
||||
finally:
|
||||
self.stats['success'] += len(target_devs_info - failure_devs_info)
|
||||
self._add_failure_stats(failure_devs_info)
|
||||
self.partition_times.append(time.time() - begin)
|
||||
self.logger.timing_since('partition.update.timing', begin)
|
||||
|
||||
|
@ -482,6 +556,9 @@ class ObjectReplicator(Daemon):
|
|||
using replication style storage policy
|
||||
"""
|
||||
jobs = []
|
||||
self.all_devs_info.update(
|
||||
[(dev['replication_ip'], dev['device'])
|
||||
for dev in policy.object_ring.devs if dev])
|
||||
data_dir = get_data_dir(policy)
|
||||
for local_dev in [dev for dev in policy.object_ring.devs
|
||||
if (dev
|
||||
|
@ -495,6 +572,11 @@ class ObjectReplicator(Daemon):
|
|||
obj_path = join(dev_path, data_dir)
|
||||
tmp_path = join(dev_path, get_tmp_dir(policy))
|
||||
if self.mount_check and not ismount(dev_path):
|
||||
self._add_failure_stats(
|
||||
[(failure_dev['replication_ip'],
|
||||
failure_dev['device'])
|
||||
for failure_dev in policy.object_ring.devs
|
||||
if failure_dev])
|
||||
self.logger.warn(_('%s is not mounted'), local_dev['device'])
|
||||
continue
|
||||
unlink_older_than(tmp_path, time.time() - self.reclaim_age)
|
||||
|
@ -509,6 +591,7 @@ class ObjectReplicator(Daemon):
|
|||
and partition not in override_partitions):
|
||||
continue
|
||||
|
||||
part_nodes = None
|
||||
try:
|
||||
job_path = join(obj_path, partition)
|
||||
part_nodes = policy.object_ring.get_part_nodes(
|
||||
|
@ -525,6 +608,17 @@ class ObjectReplicator(Daemon):
|
|||
partition=partition,
|
||||
region=local_dev['region']))
|
||||
except ValueError:
|
||||
if part_nodes:
|
||||
self._add_failure_stats(
|
||||
[(failure_dev['replication_ip'],
|
||||
failure_dev['device'])
|
||||
for failure_dev in nodes])
|
||||
else:
|
||||
self._add_failure_stats(
|
||||
[(failure_dev['replication_ip'],
|
||||
failure_dev['device'])
|
||||
for failure_dev in policy.object_ring.devs
|
||||
if failure_dev])
|
||||
continue
|
||||
return jobs
|
||||
|
||||
|
@ -570,19 +664,31 @@ class ObjectReplicator(Daemon):
|
|||
self.replication_count = 0
|
||||
self.last_replication_count = -1
|
||||
self.partition_times = []
|
||||
self.my_replication_ips = self._get_my_replication_ips()
|
||||
self.all_devs_info = set()
|
||||
|
||||
stats = eventlet.spawn(self.heartbeat)
|
||||
lockup_detector = eventlet.spawn(self.detect_lockups)
|
||||
eventlet.sleep() # Give spawns a cycle
|
||||
|
||||
current_nodes = None
|
||||
try:
|
||||
self.run_pool = GreenPool(size=self.concurrency)
|
||||
jobs = self.collect_jobs(override_devices=override_devices,
|
||||
override_partitions=override_partitions,
|
||||
override_policies=override_policies)
|
||||
for job in jobs:
|
||||
current_nodes = job['nodes']
|
||||
if override_devices and job['device'] not in override_devices:
|
||||
continue
|
||||
if override_partitions and \
|
||||
job['partition'] not in override_partitions:
|
||||
continue
|
||||
dev_path = join(self.devices_dir, job['device'])
|
||||
if self.mount_check and not ismount(dev_path):
|
||||
self._add_failure_stats([(failure_dev['replication_ip'],
|
||||
failure_dev['device'])
|
||||
for failure_dev in job['nodes']])
|
||||
self.logger.warn(_('%s is not mounted'), job['device'])
|
||||
continue
|
||||
if not self.check_ring(job['policy'].object_ring):
|
||||
|
@ -604,18 +710,26 @@ class ObjectReplicator(Daemon):
|
|||
self.run_pool.spawn(self.update_deleted, job)
|
||||
else:
|
||||
self.run_pool.spawn(self.update, job)
|
||||
current_nodes = None
|
||||
with Timeout(self.lockup_timeout):
|
||||
self.run_pool.waitall()
|
||||
except (Exception, Timeout):
|
||||
if current_nodes:
|
||||
self._add_failure_stats([(failure_dev['replication_ip'],
|
||||
failure_dev['device'])
|
||||
for failure_dev in current_nodes])
|
||||
else:
|
||||
self._add_failure_stats(self.all_devs_info)
|
||||
self.logger.exception(_("Exception in top-level replication loop"))
|
||||
self.kill_coros()
|
||||
finally:
|
||||
stats.kill()
|
||||
lockup_detector.kill()
|
||||
self.stats_line()
|
||||
self.stats['attempted'] = self.replication_count
|
||||
|
||||
def run_once(self, *args, **kwargs):
|
||||
start = time.time()
|
||||
self._zero_stats()
|
||||
self.logger.info(_("Running object replicator in script mode."))
|
||||
|
||||
override_devices = list_from_csv(kwargs.get('devices'))
|
||||
|
@ -632,27 +746,35 @@ class ObjectReplicator(Daemon):
|
|||
override_devices=override_devices,
|
||||
override_partitions=override_partitions,
|
||||
override_policies=override_policies)
|
||||
total = (time.time() - start) / 60
|
||||
total = (time.time() - self.stats['start']) / 60
|
||||
self.logger.info(
|
||||
_("Object replication complete (once). (%.02f minutes)"), total)
|
||||
if not (override_partitions or override_devices):
|
||||
dump_recon_cache({'object_replication_time': total,
|
||||
'object_replication_last': time.time()},
|
||||
replication_last = time.time()
|
||||
dump_recon_cache({'replication_stats': self.stats,
|
||||
'replication_time': total,
|
||||
'replication_last': replication_last,
|
||||
'object_replication_time': total,
|
||||
'object_replication_last': replication_last},
|
||||
self.rcache, self.logger)
|
||||
|
||||
def run_forever(self, *args, **kwargs):
|
||||
self.logger.info(_("Starting object replicator in daemon mode."))
|
||||
# Run the replicator continually
|
||||
while True:
|
||||
start = time.time()
|
||||
self._zero_stats()
|
||||
self.logger.info(_("Starting object replication pass."))
|
||||
# Run the replicator
|
||||
self.replicate()
|
||||
total = (time.time() - start) / 60
|
||||
total = (time.time() - self.stats['start']) / 60
|
||||
self.logger.info(
|
||||
_("Object replication complete. (%.02f minutes)"), total)
|
||||
dump_recon_cache({'object_replication_time': total,
|
||||
'object_replication_last': time.time()},
|
||||
replication_last = time.time()
|
||||
dump_recon_cache({'replication_stats': self.stats,
|
||||
'replication_time': total,
|
||||
'replication_last': replication_last,
|
||||
'object_replication_time': total,
|
||||
'object_replication_last': replication_last},
|
||||
self.rcache, self.logger)
|
||||
self.logger.debug('Replication sleeping for %s seconds.',
|
||||
self.interval)
|
||||
|
|
|
@ -15,7 +15,7 @@
|
|||
|
||||
""" Object Server for Swift """
|
||||
|
||||
import cPickle as pickle
|
||||
import six.moves.cPickle as pickle
|
||||
import json
|
||||
import os
|
||||
import multiprocessing
|
||||
|
@ -28,6 +28,7 @@ from swift import gettext_ as _
|
|||
from hashlib import md5
|
||||
|
||||
from eventlet import sleep, wsgi, Timeout
|
||||
from eventlet.greenthread import spawn
|
||||
|
||||
from swift.common.utils import public, get_logger, \
|
||||
config_true_value, timing_stats, replication, \
|
||||
|
@ -108,7 +109,9 @@ class ObjectController(BaseStorageServer):
|
|||
"""
|
||||
super(ObjectController, self).__init__(conf)
|
||||
self.logger = logger or get_logger(conf, log_route='object-server')
|
||||
self.node_timeout = int(conf.get('node_timeout', 3))
|
||||
self.node_timeout = float(conf.get('node_timeout', 3))
|
||||
self.container_update_timeout = float(
|
||||
conf.get('container_update_timeout', 1))
|
||||
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
|
||||
self.client_timeout = int(conf.get('client_timeout', 60))
|
||||
self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536))
|
||||
|
@ -198,7 +201,8 @@ class ObjectController(BaseStorageServer):
|
|||
device, partition, account, container, obj, policy, **kwargs)
|
||||
|
||||
def async_update(self, op, account, container, obj, host, partition,
|
||||
contdevice, headers_out, objdevice, policy):
|
||||
contdevice, headers_out, objdevice, policy,
|
||||
logger_thread_locals=None):
|
||||
"""
|
||||
Sends or saves an async update.
|
||||
|
||||
|
@ -213,7 +217,12 @@ class ObjectController(BaseStorageServer):
|
|||
request
|
||||
:param objdevice: device name that the object is in
|
||||
:param policy: the associated BaseStoragePolicy instance
|
||||
:param logger_thread_locals: The thread local values to be set on the
|
||||
self.logger to retain transaction
|
||||
logging information.
|
||||
"""
|
||||
if logger_thread_locals:
|
||||
self.logger.thread_locals = logger_thread_locals
|
||||
headers_out['user-agent'] = 'object-server %s' % os.getpid()
|
||||
full_path = '/%s/%s/%s' % (account, container, obj)
|
||||
if all([host, partition, contdevice]):
|
||||
|
@ -285,10 +294,28 @@ class ObjectController(BaseStorageServer):
|
|||
headers_out['x-trans-id'] = headers_in.get('x-trans-id', '-')
|
||||
headers_out['referer'] = request.as_referer()
|
||||
headers_out['X-Backend-Storage-Policy-Index'] = int(policy)
|
||||
update_greenthreads = []
|
||||
for conthost, contdevice in updates:
|
||||
self.async_update(op, account, container, obj, conthost,
|
||||
contpartition, contdevice, headers_out,
|
||||
objdevice, policy)
|
||||
gt = spawn(self.async_update, op, account, container, obj,
|
||||
conthost, contpartition, contdevice, headers_out,
|
||||
objdevice, policy,
|
||||
logger_thread_locals=self.logger.thread_locals)
|
||||
update_greenthreads.append(gt)
|
||||
# Wait a little bit to see if the container updates are successful.
|
||||
# If we immediately return after firing off the greenthread above, then
|
||||
# we're more likely to confuse the end-user who does a listing right
|
||||
# after getting a successful response to the object create. The
|
||||
# `container_update_timeout` bounds the length of time we wait so that
|
||||
# one slow container server doesn't make the entire request lag.
|
||||
try:
|
||||
with Timeout(self.container_update_timeout):
|
||||
for gt in update_greenthreads:
|
||||
gt.wait()
|
||||
except Timeout:
|
||||
# updates didn't go through, log it and return
|
||||
self.logger.debug(
|
||||
'Container update timeout (%.4fs) waiting for %s',
|
||||
self.container_update_timeout, updates)
|
||||
|
||||
def delete_at_update(self, op, delete_at, account, container, obj,
|
||||
request, objdevice, policy):
|
||||
|
@ -417,6 +444,11 @@ class ObjectController(BaseStorageServer):
|
|||
override = key.lower().replace(override_prefix, 'x-')
|
||||
update_headers[override] = val
|
||||
|
||||
def _preserve_slo_manifest(self, update_metadata, orig_metadata):
|
||||
if 'X-Static-Large-Object' in orig_metadata:
|
||||
update_metadata['X-Static-Large-Object'] = \
|
||||
orig_metadata['X-Static-Large-Object']
|
||||
|
||||
@public
|
||||
@timing_stats()
|
||||
def POST(self, request):
|
||||
|
@ -446,6 +478,7 @@ class ObjectController(BaseStorageServer):
|
|||
request=request,
|
||||
headers={'X-Backend-Timestamp': orig_timestamp.internal})
|
||||
metadata = {'X-Timestamp': req_timestamp.internal}
|
||||
self._preserve_slo_manifest(metadata, orig_metadata)
|
||||
metadata.update(val for val in request.headers.items()
|
||||
if is_user_meta('object', val[0]))
|
||||
for header_key in self.allowed_headers:
|
||||
|
@ -685,9 +718,6 @@ class ObjectController(BaseStorageServer):
|
|||
"""Handle HTTP GET requests for the Swift Object Server."""
|
||||
device, partition, account, container, obj, policy = \
|
||||
get_name_and_placement(request, 5, 5, True)
|
||||
keep_cache = self.keep_cache_private or (
|
||||
'X-Auth-Token' not in request.headers and
|
||||
'X-Storage-Token' not in request.headers)
|
||||
try:
|
||||
disk_file = self.get_diskfile(
|
||||
device, partition, account, container, obj,
|
||||
|
|
|
@ -164,7 +164,7 @@ class Receiver(object):
|
|||
self.node_index = int(
|
||||
self.request.headers['X-Backend-Ssync-Node-Index'])
|
||||
if self.node_index != self.frag_index:
|
||||
# a primary node should only recieve it's own fragments
|
||||
# a primary node should only receive it's own fragments
|
||||
raise swob.HTTPBadRequest(
|
||||
'Frag-Index (%s) != Node-Index (%s)' % (
|
||||
self.frag_index, self.node_index))
|
||||
|
@ -319,7 +319,11 @@ class Receiver(object):
|
|||
header = header.strip().lower()
|
||||
value = value.strip()
|
||||
subreq.headers[header] = value
|
||||
replication_headers.append(header)
|
||||
if header != 'etag':
|
||||
# make sure ssync doesn't cause 'Etag' to be added to
|
||||
# obj metadata in addition to 'ETag' which object server
|
||||
# sets (note capitalization)
|
||||
replication_headers.append(header)
|
||||
if header == 'content-length':
|
||||
content_length = int(value)
|
||||
# Establish subrequest body, if needed.
|
||||
|
|
|
@ -82,7 +82,6 @@ class Sender(object):
|
|||
set(self.send_list))
|
||||
can_delete_obj = dict((hash_, self.available_map[hash_])
|
||||
for hash_ in in_sync_hashes)
|
||||
self.disconnect()
|
||||
if not self.failures:
|
||||
return True, can_delete_obj
|
||||
else:
|
||||
|
@ -103,6 +102,8 @@ class Sender(object):
|
|||
self.node.get('replication_ip'),
|
||||
self.node.get('replication_port'),
|
||||
self.node.get('device'), self.job.get('partition'))
|
||||
finally:
|
||||
self.disconnect()
|
||||
except Exception:
|
||||
# We don't want any exceptions to escape our code and possibly
|
||||
# mess up the original replicator code that called us since it
|
||||
|
@ -211,8 +212,10 @@ class Sender(object):
|
|||
self.job['policy'], self.suffixes,
|
||||
frag_index=self.job.get('frag_index'))
|
||||
if self.remote_check_objs is not None:
|
||||
hash_gen = ifilter(lambda (path, object_hash, timestamp):
|
||||
object_hash in self.remote_check_objs, hash_gen)
|
||||
hash_gen = ifilter(
|
||||
lambda path_objhash_timestamp:
|
||||
path_objhash_timestamp[1] in
|
||||
self.remote_check_objs, hash_gen)
|
||||
for path, object_hash, timestamp in hash_gen:
|
||||
self.available_map[object_hash] = timestamp
|
||||
with exceptions.MessageTimeout(
|
||||
|
@ -349,6 +352,8 @@ class Sender(object):
|
|||
Closes down the connection to the object server once done
|
||||
with the SSYNC request.
|
||||
"""
|
||||
if not self.connection:
|
||||
return
|
||||
try:
|
||||
with exceptions.MessageTimeout(
|
||||
self.daemon.node_timeout, 'disconnect'):
|
||||
|
|
|
@ -13,7 +13,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import cPickle as pickle
|
||||
import six.moves.cPickle as pickle
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
|
@ -256,7 +256,7 @@ class ObjectUpdater(Daemon):
|
|||
|
||||
:param node: node dictionary from the container ring
|
||||
:param part: partition that holds the container
|
||||
:param op: operation performed (ex: 'POST' or 'DELETE')
|
||||
:param op: operation performed (ex: 'PUT' or 'DELETE')
|
||||
:param obj: object name being updated
|
||||
:param headers_out: headers to send with the update
|
||||
"""
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue