Merge remote-tracking branch 'remotes/origin/master' into merge-master
Include a trivial whitespace change in .zuul.yaml to make sure Zuul realizes it changed on master. Change-Id: I24069547265db9ebd39e565f966f4d0b82c0aea2
This commit is contained in:
commit
6afc1130fd
2
.mailmap
2
.mailmap
|
@ -79,7 +79,7 @@ Minwoo Bae <minwoob@us.ibm.com> Minwoo B
|
||||||
Jaivish Kothari <jaivish.kothari@nectechnologies.in> <janonymous.codevulture@gmail.com>
|
Jaivish Kothari <jaivish.kothari@nectechnologies.in> <janonymous.codevulture@gmail.com>
|
||||||
Michael Matur <michael.matur@gmail.com>
|
Michael Matur <michael.matur@gmail.com>
|
||||||
Kazuhiro Miyahara <miyahara.kazuhiro@lab.ntt.co.jp>
|
Kazuhiro Miyahara <miyahara.kazuhiro@lab.ntt.co.jp>
|
||||||
Alexandra Settle <alexandra.settle@rackspace.com>
|
Alexandra Settle <asettle@suse.com> <alexandra.settle@rackspace.com>
|
||||||
Kenichiro Matsuda <matsuda_kenichi@jp.fujitsu.com>
|
Kenichiro Matsuda <matsuda_kenichi@jp.fujitsu.com>
|
||||||
Atsushi Sakai <sakaia@jp.fujitsu.com>
|
Atsushi Sakai <sakaia@jp.fujitsu.com>
|
||||||
Takashi Natsume <natsume.takashi@lab.ntt.co.jp>
|
Takashi Natsume <natsume.takashi@lab.ntt.co.jp>
|
||||||
|
|
31
.zuul.yaml
31
.zuul.yaml
|
@ -35,6 +35,7 @@
|
||||||
- job:
|
- job:
|
||||||
name: swift-tox-py35
|
name: swift-tox-py35
|
||||||
parent: swift-tox-base
|
parent: swift-tox-base
|
||||||
|
nodeset: ubuntu-xenial
|
||||||
description: |
|
description: |
|
||||||
Run unit-tests for swift under cPython version 3.5.
|
Run unit-tests for swift under cPython version 3.5.
|
||||||
|
|
||||||
|
@ -67,6 +68,25 @@
|
||||||
NOSE_COVER_HTML_DIR: '{toxinidir}/cover'
|
NOSE_COVER_HTML_DIR: '{toxinidir}/cover'
|
||||||
post-run: tools/playbooks/common/cover-post.yaml
|
post-run: tools/playbooks/common/cover-post.yaml
|
||||||
|
|
||||||
|
- job:
|
||||||
|
name: swift-tox-py37
|
||||||
|
parent: swift-tox-base
|
||||||
|
nodeset: ubuntu-bionic
|
||||||
|
description: |
|
||||||
|
Run unit-tests for swift under cPython version 3.7.
|
||||||
|
|
||||||
|
Uses tox with the ``py37`` environment.
|
||||||
|
It sets TMPDIR to an XFS mount point created via
|
||||||
|
tools/test-setup.sh.
|
||||||
|
vars:
|
||||||
|
tox_envlist: py37
|
||||||
|
bindep_profile: test py37
|
||||||
|
python_version: 3.7
|
||||||
|
tox_environment:
|
||||||
|
NOSE_COVER_HTML: 1
|
||||||
|
NOSE_COVER_HTML_DIR: '{toxinidir}/cover'
|
||||||
|
post-run: tools/playbooks/common/cover-post.yaml
|
||||||
|
|
||||||
- job:
|
- job:
|
||||||
name: swift-tox-func
|
name: swift-tox-func
|
||||||
parent: swift-tox-base
|
parent: swift-tox-base
|
||||||
|
@ -316,6 +336,11 @@
|
||||||
- ^(api-ref|doc|releasenotes)/.*$
|
- ^(api-ref|doc|releasenotes)/.*$
|
||||||
- ^test/(functional|probe)/.*$
|
- ^test/(functional|probe)/.*$
|
||||||
voting: false
|
voting: false
|
||||||
|
- swift-tox-py37:
|
||||||
|
irrelevant-files:
|
||||||
|
- ^(api-ref|doc|releasenotes)/.*$
|
||||||
|
- ^test/(functional|probe)/.*$
|
||||||
|
voting: false
|
||||||
- swift-tox-func:
|
- swift-tox-func:
|
||||||
irrelevant-files:
|
irrelevant-files:
|
||||||
- ^(api-ref|doc|releasenotes)/.*$
|
- ^(api-ref|doc|releasenotes)/.*$
|
||||||
|
@ -344,7 +369,10 @@
|
||||||
- swift-tox-func-s3api-ceph-s3tests-tempauth:
|
- swift-tox-func-s3api-ceph-s3tests-tempauth:
|
||||||
irrelevant-files:
|
irrelevant-files:
|
||||||
- ^(api-ref|releasenotes)/.*$
|
- ^(api-ref|releasenotes)/.*$
|
||||||
- ^test/probe/.*$
|
# Keep doc/saio -- we use those sample configs in the saio playbooks
|
||||||
|
# Also keep doc/s3api -- it holds known failures for these tests
|
||||||
|
- ^doc/(requirements.txt|(manpages|source)/.*)$
|
||||||
|
- ^test/(unit|probe)/.*$
|
||||||
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
|
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG)$
|
||||||
- swift-probetests-centos-7:
|
- swift-probetests-centos-7:
|
||||||
irrelevant-files:
|
irrelevant-files:
|
||||||
|
@ -428,3 +456,4 @@
|
||||||
post:
|
post:
|
||||||
jobs:
|
jobs:
|
||||||
- publish-openstack-python-branch-tarball
|
- publish-openstack-python-branch-tarball
|
||||||
|
|
||||||
|
|
6
AUTHORS
6
AUTHORS
|
@ -45,7 +45,7 @@ Alex Holden (alex@alexjonasholden.com)
|
||||||
Alex Pecoraro (alex.pecoraro@emc.com)
|
Alex Pecoraro (alex.pecoraro@emc.com)
|
||||||
Alex Szarka (szarka@inf.u-szeged.hu)
|
Alex Szarka (szarka@inf.u-szeged.hu)
|
||||||
Alex Yang (alex890714@gmail.com)
|
Alex Yang (alex890714@gmail.com)
|
||||||
Alexandra Settle (alexandra.settle@rackspace.com)
|
Alexandra Settle (asettle@suse.com)
|
||||||
Alexandre Lécuyer (alexandre.lecuyer@corp.ovh.com)
|
Alexandre Lécuyer (alexandre.lecuyer@corp.ovh.com)
|
||||||
Alfredo Moralejo (amoralej@redhat.com)
|
Alfredo Moralejo (amoralej@redhat.com)
|
||||||
Alistair Coles (alistairncoles@gmail.com)
|
Alistair Coles (alistairncoles@gmail.com)
|
||||||
|
@ -166,6 +166,7 @@ Fujita Tomonori (fujita.tomonori@lab.ntt.co.jp)
|
||||||
Félix Cantournet (felix.cantournet@cloudwatt.com)
|
Félix Cantournet (felix.cantournet@cloudwatt.com)
|
||||||
Gage Hugo (gh159m@att.com)
|
Gage Hugo (gh159m@att.com)
|
||||||
Ganesh Maharaj Mahalingam (ganesh.mahalingam@intel.com)
|
Ganesh Maharaj Mahalingam (ganesh.mahalingam@intel.com)
|
||||||
|
gaobin (gaobin@inspur.com)
|
||||||
gaofei (gao.fei@inspur.com)
|
gaofei (gao.fei@inspur.com)
|
||||||
Gaurav B. Gangalwar (gaurav@gluster.com)
|
Gaurav B. Gangalwar (gaurav@gluster.com)
|
||||||
gecong1973 (ge.cong@zte.com.cn)
|
gecong1973 (ge.cong@zte.com.cn)
|
||||||
|
@ -173,6 +174,7 @@ gengchc2 (geng.changcai2@zte.com.cn)
|
||||||
Gerard Gine (ggine@swiftstack.com)
|
Gerard Gine (ggine@swiftstack.com)
|
||||||
Gerry Drudy (gerry.drudy@hpe.com)
|
Gerry Drudy (gerry.drudy@hpe.com)
|
||||||
Gil Vernik (gilv@il.ibm.com)
|
Gil Vernik (gilv@il.ibm.com)
|
||||||
|
Gleb Samsonov (sams-gleb@yandex.ru)
|
||||||
Gonéri Le Bouder (goneri.lebouder@enovance.com)
|
Gonéri Le Bouder (goneri.lebouder@enovance.com)
|
||||||
Graham Hayes (graham.hayes@hpe.com)
|
Graham Hayes (graham.hayes@hpe.com)
|
||||||
Gregory Haynes (greg@greghaynes.net)
|
Gregory Haynes (greg@greghaynes.net)
|
||||||
|
@ -276,6 +278,7 @@ Mehdi Abaakouk (sileht@redhat.com)
|
||||||
melissaml (ma.lei@99cloud.net)
|
melissaml (ma.lei@99cloud.net)
|
||||||
Michael Matur (michael.matur@gmail.com)
|
Michael Matur (michael.matur@gmail.com)
|
||||||
Michael Shuler (mshuler@gmail.com)
|
Michael Shuler (mshuler@gmail.com)
|
||||||
|
Michele Valsecchi (mvalsecc@redhat.com)
|
||||||
Mike Fedosin (mfedosin@mirantis.com)
|
Mike Fedosin (mfedosin@mirantis.com)
|
||||||
Mingyu Li (li.mingyu@99cloud.net)
|
Mingyu Li (li.mingyu@99cloud.net)
|
||||||
Minwoo Bae (minwoob@us.ibm.com)
|
Minwoo Bae (minwoob@us.ibm.com)
|
||||||
|
@ -286,6 +289,7 @@ Monty Taylor (mordred@inaugust.com)
|
||||||
Morgan Fainberg (morgan.fainberg@gmail.com)
|
Morgan Fainberg (morgan.fainberg@gmail.com)
|
||||||
Morita Kazutaka (morita.kazutaka@gmail.com)
|
Morita Kazutaka (morita.kazutaka@gmail.com)
|
||||||
Motonobu Ichimura (motonobu@gmail.com)
|
Motonobu Ichimura (motonobu@gmail.com)
|
||||||
|
Nadeem Syed (snadeem.hameed@gmail.com)
|
||||||
Nakagawa Masaaki (nakagawamsa@nttdata.co.jp)
|
Nakagawa Masaaki (nakagawamsa@nttdata.co.jp)
|
||||||
Nakul Dahiwade (nakul.dahiwade@intel.com)
|
Nakul Dahiwade (nakul.dahiwade@intel.com)
|
||||||
Nam Nguyen Hoai (namnh@vn.fujitsu.com)
|
Nam Nguyen Hoai (namnh@vn.fujitsu.com)
|
||||||
|
|
58
CHANGELOG
58
CHANGELOG
|
@ -1,3 +1,61 @@
|
||||||
|
swift (2.21.0, OpenStack Stein release)
|
||||||
|
|
||||||
|
* Change the behavior of the EC reconstructor to perform a
|
||||||
|
fragment rebuild to a handoff node when a primary peer responds
|
||||||
|
with 507 to the REPLICATE request. This changes EC to match the
|
||||||
|
existing behavior of replication when drives fail. After a
|
||||||
|
rebalance of EC rings (potentially removing unmounted/failed
|
||||||
|
devices), it's most IO efficient to run in handoffs_only mode to
|
||||||
|
avoid unnecessary rebuilds.
|
||||||
|
|
||||||
|
* O_TMPFILE support is now detected by attempting to use it
|
||||||
|
instead of looking at the kernel version. This allows older
|
||||||
|
kernels with backported patches to take advantage of the
|
||||||
|
O_TMPFILE functionality.
|
||||||
|
|
||||||
|
* Add slo_manifest_hook callback to allow other middlewares to
|
||||||
|
impose additional constraints on or make edits to SLO manifests
|
||||||
|
before being written. For example, a middleware could enforce
|
||||||
|
minimum segment size or insert data segments.
|
||||||
|
|
||||||
|
* Fixed an issue with multi-region EC policies that caused the EC
|
||||||
|
reconstructor to constantly attempt cross-region rebuild
|
||||||
|
traffic.
|
||||||
|
|
||||||
|
* Fixed an issue where S3 API v4 signatures would not be validated
|
||||||
|
against the body of the request, allowing a replay attack if
|
||||||
|
request headers were captured by a malicious third party.
|
||||||
|
|
||||||
|
* Display crypto data/metadata details in swift-object-info.
|
||||||
|
|
||||||
|
* formpost can now accept a content-encoding parameter.
|
||||||
|
|
||||||
|
* Fixed an issue where multipart uploads with the S3 API would
|
||||||
|
sometimes report an error despite all segments being upload
|
||||||
|
successfully.
|
||||||
|
|
||||||
|
* Multipart object segments are now actually deleted when the
|
||||||
|
multipart object is deleted via the S3 API.
|
||||||
|
|
||||||
|
* Swift now returns a 503 (instead of a 500) when an account
|
||||||
|
auto-create fails.
|
||||||
|
|
||||||
|
* Fixed a bug where encryption would store the incorrect key
|
||||||
|
metadata if the object name starts with a slash.
|
||||||
|
|
||||||
|
* Fixed an issue where an object server failure during a client
|
||||||
|
download could leave an open socket between the proxy and
|
||||||
|
client.
|
||||||
|
|
||||||
|
* Fixed an issue where deleted EC objects didn't have their
|
||||||
|
on-disk directories cleaned up. This would cause extra resource
|
||||||
|
usage on the object servers.
|
||||||
|
|
||||||
|
* Fixed issue where bulk requests using xml and expect
|
||||||
|
100-continue would return a malformed HTTP response.
|
||||||
|
|
||||||
|
* Various other minor bug fixes and improvements.
|
||||||
|
|
||||||
swift (2.20.0)
|
swift (2.20.0)
|
||||||
|
|
||||||
* S3 API compatibility updates
|
* S3 API compatibility updates
|
||||||
|
|
|
@ -27,7 +27,7 @@ log_level = INFO
|
||||||
interval = 300
|
interval = 300
|
||||||
# auto_create_account_prefix = .
|
# auto_create_account_prefix = .
|
||||||
# report_interval = 300
|
# report_interval = 300
|
||||||
# concurrency is the level of concurrency o use to do the work, this value
|
# concurrency is the level of concurrency to use to do the work, this value
|
||||||
# must be set to at least 1
|
# must be set to at least 1
|
||||||
# concurrency = 1
|
# concurrency = 1
|
||||||
# processes is how many parts to divide the work into, one part per process
|
# processes is how many parts to divide the work into, one part per process
|
||||||
|
|
|
@ -76,9 +76,10 @@ Installing dependencies
|
||||||
python2-netifaces python2-pip python2-dnspython \
|
python2-netifaces python2-pip python2-dnspython \
|
||||||
python2-mock
|
python2-mock
|
||||||
|
|
||||||
Note: This installs necessary system dependencies and *most* of the python
|
.. note::
|
||||||
dependencies. Later in the process setuptools/distribute or pip will install
|
This installs necessary system dependencies and *most* of the python
|
||||||
and/or upgrade packages.
|
dependencies. Later in the process setuptools/distribute or pip will install
|
||||||
|
and/or upgrade packages.
|
||||||
|
|
||||||
Next, choose either :ref:`partition-section` or :ref:`loopback-section`.
|
Next, choose either :ref:`partition-section` or :ref:`loopback-section`.
|
||||||
|
|
||||||
|
@ -90,50 +91,52 @@ Using a partition for storage
|
||||||
If you are going to use a separate partition for Swift data, be sure to add
|
If you are going to use a separate partition for Swift data, be sure to add
|
||||||
another device when creating the VM, and follow these instructions:
|
another device when creating the VM, and follow these instructions:
|
||||||
|
|
||||||
#. Set up a single partition::
|
#. Set up a single partition::
|
||||||
|
|
||||||
sudo fdisk /dev/sdb
|
sudo fdisk /dev/sdb
|
||||||
sudo mkfs.xfs /dev/sdb1
|
sudo mkfs.xfs /dev/sdb1
|
||||||
|
|
||||||
#. Edit ``/etc/fstab`` and add::
|
#. Edit ``/etc/fstab`` and add::
|
||||||
|
|
||||||
/dev/sdb1 /mnt/sdb1 xfs noatime,nodiratime,nobarrier,logbufs=8 0 0
|
/dev/sdb1 /mnt/sdb1 xfs noatime,nodiratime,nobarrier,logbufs=8 0 0
|
||||||
|
|
||||||
#. Create the mount point and the individualized links::
|
#. Create the mount point and the individualized links::
|
||||||
|
|
||||||
sudo mkdir /mnt/sdb1
|
sudo mkdir /mnt/sdb1
|
||||||
sudo mount /mnt/sdb1
|
sudo mount /mnt/sdb1
|
||||||
sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4
|
sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4
|
||||||
sudo chown ${USER}:${USER} /mnt/sdb1/*
|
sudo chown ${USER}:${USER} /mnt/sdb1/*
|
||||||
sudo mkdir /srv
|
sudo mkdir /srv
|
||||||
for x in {1..4}; do sudo ln -s /mnt/sdb1/$x /srv/$x; done
|
for x in {1..4}; do sudo ln -s /mnt/sdb1/$x /srv/$x; done
|
||||||
sudo mkdir -p /srv/1/node/sdb1 /srv/1/node/sdb5 \
|
sudo mkdir -p /srv/1/node/sdb1 /srv/1/node/sdb5 \
|
||||||
/srv/2/node/sdb2 /srv/2/node/sdb6 \
|
/srv/2/node/sdb2 /srv/2/node/sdb6 \
|
||||||
/srv/3/node/sdb3 /srv/3/node/sdb7 \
|
/srv/3/node/sdb3 /srv/3/node/sdb7 \
|
||||||
/srv/4/node/sdb4 /srv/4/node/sdb8 \
|
/srv/4/node/sdb4 /srv/4/node/sdb8 \
|
||||||
/var/run/swift
|
/var/run/swift
|
||||||
sudo chown -R ${USER}:${USER} /var/run/swift
|
sudo chown -R ${USER}:${USER} /var/run/swift
|
||||||
# **Make sure to include the trailing slash after /srv/$x/**
|
# **Make sure to include the trailing slash after /srv/$x/**
|
||||||
for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done
|
for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done
|
||||||
|
|
||||||
Note: For OpenSuse users, a user's primary group is `users`, so you have 2 options:
|
.. note::
|
||||||
|
For OpenSuse users, a user's primary group is ``users``, so you have 2 options:
|
||||||
|
|
||||||
* Change `${USER}:${USER}` to `${USER}:users` in all references of this guide; or
|
* Change ``${USER}:${USER}`` to ``${USER}:users`` in all references of this guide; or
|
||||||
* Create a group for your username and add yourself to it::
|
* Create a group for your username and add yourself to it::
|
||||||
|
|
||||||
sudo groupadd ${USER} && sudo gpasswd -a ${USER} ${USER}
|
sudo groupadd ${USER} && sudo gpasswd -a ${USER} ${USER}
|
||||||
|
|
||||||
Note: We create the mount points and mount the storage disk under
|
.. note::
|
||||||
/mnt/sdb1. This disk will contain one directory per simulated swift node,
|
We create the mount points and mount the storage disk under
|
||||||
each owned by the current swift user.
|
/mnt/sdb1. This disk will contain one directory per simulated swift node,
|
||||||
|
each owned by the current swift user.
|
||||||
|
|
||||||
We then create symlinks to these directories under /srv.
|
We then create symlinks to these directories under /srv.
|
||||||
If the disk sdb is unmounted, files will not be written under
|
If the disk sdb is unmounted, files will not be written under
|
||||||
/srv/\*, because the symbolic link destination /mnt/sdb1/* will not
|
/srv/\*, because the symbolic link destination /mnt/sdb1/* will not
|
||||||
exist. This prevents disk sync operations from writing to the root
|
exist. This prevents disk sync operations from writing to the root
|
||||||
partition in the event a drive is unmounted.
|
partition in the event a drive is unmounted.
|
||||||
|
|
||||||
#. Next, skip to :ref:`common-dev-section`.
|
#. Next, skip to :ref:`common-dev-section`.
|
||||||
|
|
||||||
|
|
||||||
.. _loopback-section:
|
.. _loopback-section:
|
||||||
|
@ -144,51 +147,53 @@ Using a loopback device for storage
|
||||||
If you want to use a loopback device instead of another partition, follow
|
If you want to use a loopback device instead of another partition, follow
|
||||||
these instructions:
|
these instructions:
|
||||||
|
|
||||||
#. Create the file for the loopback device::
|
#. Create the file for the loopback device::
|
||||||
|
|
||||||
sudo mkdir /srv
|
sudo mkdir /srv
|
||||||
sudo truncate -s 1GB /srv/swift-disk
|
sudo truncate -s 1GB /srv/swift-disk
|
||||||
sudo mkfs.xfs /srv/swift-disk
|
sudo mkfs.xfs /srv/swift-disk
|
||||||
|
|
||||||
Modify size specified in the ``truncate`` command to make a larger or
|
Modify size specified in the ``truncate`` command to make a larger or
|
||||||
smaller partition as needed.
|
smaller partition as needed.
|
||||||
|
|
||||||
#. Edit `/etc/fstab` and add::
|
#. Edit `/etc/fstab` and add::
|
||||||
|
|
||||||
/srv/swift-disk /mnt/sdb1 xfs loop,noatime,nodiratime,nobarrier,logbufs=8 0 0
|
/srv/swift-disk /mnt/sdb1 xfs loop,noatime,nodiratime,nobarrier,logbufs=8 0 0
|
||||||
|
|
||||||
#. Create the mount point and the individualized links::
|
#. Create the mount point and the individualized links::
|
||||||
|
|
||||||
sudo mkdir /mnt/sdb1
|
sudo mkdir /mnt/sdb1
|
||||||
sudo mount /mnt/sdb1
|
sudo mount /mnt/sdb1
|
||||||
sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4
|
sudo mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4
|
||||||
sudo chown ${USER}:${USER} /mnt/sdb1/*
|
sudo chown ${USER}:${USER} /mnt/sdb1/*
|
||||||
for x in {1..4}; do sudo ln -s /mnt/sdb1/$x /srv/$x; done
|
for x in {1..4}; do sudo ln -s /mnt/sdb1/$x /srv/$x; done
|
||||||
sudo mkdir -p /srv/1/node/sdb1 /srv/1/node/sdb5 \
|
sudo mkdir -p /srv/1/node/sdb1 /srv/1/node/sdb5 \
|
||||||
/srv/2/node/sdb2 /srv/2/node/sdb6 \
|
/srv/2/node/sdb2 /srv/2/node/sdb6 \
|
||||||
/srv/3/node/sdb3 /srv/3/node/sdb7 \
|
/srv/3/node/sdb3 /srv/3/node/sdb7 \
|
||||||
/srv/4/node/sdb4 /srv/4/node/sdb8 \
|
/srv/4/node/sdb4 /srv/4/node/sdb8 \
|
||||||
/var/run/swift
|
/var/run/swift
|
||||||
sudo chown -R ${USER}:${USER} /var/run/swift
|
sudo chown -R ${USER}:${USER} /var/run/swift
|
||||||
# **Make sure to include the trailing slash after /srv/$x/**
|
# **Make sure to include the trailing slash after /srv/$x/**
|
||||||
for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done
|
for x in {1..4}; do sudo chown -R ${USER}:${USER} /srv/$x/; done
|
||||||
|
|
||||||
Note: For OpenSuse users, a user's primary group is `users`, so you have 2 options:
|
.. note::
|
||||||
|
For OpenSuse users, a user's primary group is ``users``, so you have 2 options:
|
||||||
|
|
||||||
* Change `${USER}:${USER}` to `${USER}:users` in all references of this guide; or
|
* Change ``${USER}:${USER}`` to ``${USER}:users`` in all references of this guide; or
|
||||||
* Create a group for your username and add yourself to it::
|
* Create a group for your username and add yourself to it::
|
||||||
|
|
||||||
sudo groupadd ${USER} && sudo gpasswd -a ${USER} ${USER}
|
sudo groupadd ${USER} && sudo gpasswd -a ${USER} ${USER}
|
||||||
|
|
||||||
Note: We create the mount points and mount the loopback file under
|
.. note::
|
||||||
/mnt/sdb1. This file will contain one directory per simulated swift node,
|
We create the mount points and mount the loopback file under
|
||||||
each owned by the current swift user.
|
/mnt/sdb1. This file will contain one directory per simulated swift node,
|
||||||
|
each owned by the current swift user.
|
||||||
|
|
||||||
We then create symlinks to these directories under /srv.
|
We then create symlinks to these directories under /srv.
|
||||||
If the loopback file is unmounted, files will not be written under
|
If the loopback file is unmounted, files will not be written under
|
||||||
/srv/\*, because the symbolic link destination /mnt/sdb1/* will not
|
/srv/\*, because the symbolic link destination /mnt/sdb1/* will not
|
||||||
exist. This prevents disk sync operations from writing to the root
|
exist. This prevents disk sync operations from writing to the root
|
||||||
partition in the event a drive is unmounted.
|
partition in the event a drive is unmounted.
|
||||||
|
|
||||||
.. _common-dev-section:
|
.. _common-dev-section:
|
||||||
|
|
||||||
|
@ -229,117 +234,120 @@ To persist this, edit and add the following to ``/etc/fstab``::
|
||||||
Getting the code
|
Getting the code
|
||||||
----------------
|
----------------
|
||||||
|
|
||||||
#. Check out the python-swiftclient repo::
|
#. Check out the python-swiftclient repo::
|
||||||
|
|
||||||
cd $HOME; git clone https://github.com/openstack/python-swiftclient.git
|
cd $HOME; git clone https://github.com/openstack/python-swiftclient.git
|
||||||
|
|
||||||
#. Build a development installation of python-swiftclient::
|
#. Build a development installation of python-swiftclient::
|
||||||
|
|
||||||
cd $HOME/python-swiftclient; sudo python setup.py develop; cd -
|
cd $HOME/python-swiftclient; sudo python setup.py develop; cd -
|
||||||
|
|
||||||
Ubuntu 12.04 users need to install python-swiftclient's dependencies before the installation of
|
Ubuntu 12.04 users need to install python-swiftclient's dependencies before the installation of
|
||||||
python-swiftclient. This is due to a bug in an older version of setup tools::
|
python-swiftclient. This is due to a bug in an older version of setup tools::
|
||||||
|
|
||||||
cd $HOME/python-swiftclient; sudo pip install -r requirements.txt; sudo python setup.py develop; cd -
|
cd $HOME/python-swiftclient; sudo pip install -r requirements.txt; sudo python setup.py develop; cd -
|
||||||
|
|
||||||
#. Check out the swift repo::
|
#. Check out the swift repo::
|
||||||
|
|
||||||
git clone https://github.com/openstack/swift.git
|
git clone https://github.com/openstack/swift.git
|
||||||
|
|
||||||
#. Build a development installation of swift::
|
#. Build a development installation of swift::
|
||||||
|
|
||||||
cd $HOME/swift; sudo pip install --no-binary cryptography -r requirements.txt; sudo python setup.py develop; cd -
|
cd $HOME/swift; sudo pip install --no-binary cryptography -r requirements.txt; sudo python setup.py develop; cd -
|
||||||
|
|
||||||
Note: Due to a difference in libssl.so naming in OpenSuse to other Linux distros the wheel/binary wont work so the
|
.. note::
|
||||||
cryptography must be built, thus the ``--no-binary cryptography``.
|
Due to a difference in how ``libssl.so`` is named in OpenSuse vs. other Linux distros the
|
||||||
|
wheel/binary won't work; thus we use ``--no-binary cryptography`` to build ``cryptography``
|
||||||
|
locally.
|
||||||
|
|
||||||
Fedora 19 or later users might have to perform the following if development
|
Fedora 19 or later users might have to perform the following if development
|
||||||
installation of swift fails::
|
installation of swift fails::
|
||||||
|
|
||||||
sudo pip install -U xattr
|
sudo pip install -U xattr
|
||||||
|
|
||||||
#. Install swift's test dependencies::
|
#. Install swift's test dependencies::
|
||||||
|
|
||||||
cd $HOME/swift; sudo pip install -r test-requirements.txt
|
cd $HOME/swift; sudo pip install -r test-requirements.txt
|
||||||
|
|
||||||
----------------
|
----------------
|
||||||
Setting up rsync
|
Setting up rsync
|
||||||
----------------
|
----------------
|
||||||
|
|
||||||
#. Create ``/etc/rsyncd.conf``::
|
#. Create ``/etc/rsyncd.conf``::
|
||||||
|
|
||||||
sudo cp $HOME/swift/doc/saio/rsyncd.conf /etc/
|
sudo cp $HOME/swift/doc/saio/rsyncd.conf /etc/
|
||||||
sudo sed -i "s/<your-user-name>/${USER}/" /etc/rsyncd.conf
|
sudo sed -i "s/<your-user-name>/${USER}/" /etc/rsyncd.conf
|
||||||
|
|
||||||
Here is the default ``rsyncd.conf`` file contents maintained in the repo
|
Here is the default ``rsyncd.conf`` file contents maintained in the repo
|
||||||
that is copied and fixed up above:
|
that is copied and fixed up above:
|
||||||
|
|
||||||
.. literalinclude:: /../saio/rsyncd.conf
|
.. literalinclude:: /../saio/rsyncd.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. On Ubuntu, edit the following line in ``/etc/default/rsync``::
|
#. On Ubuntu, edit the following line in ``/etc/default/rsync``::
|
||||||
|
|
||||||
RSYNC_ENABLE=true
|
RSYNC_ENABLE=true
|
||||||
|
|
||||||
On Fedora, edit the following line in ``/etc/xinetd.d/rsync``::
|
On Fedora, edit the following line in ``/etc/xinetd.d/rsync``::
|
||||||
|
|
||||||
disable = no
|
disable = no
|
||||||
|
|
||||||
One might have to create the above files to perform the edits.
|
One might have to create the above files to perform the edits.
|
||||||
|
|
||||||
On OpenSuse, nothing needs to happen here.
|
On OpenSuse, nothing needs to happen here.
|
||||||
|
|
||||||
#. On platforms with SELinux in ``Enforcing`` mode, either set to ``Permissive``::
|
#. On platforms with SELinux in ``Enforcing`` mode, either set to ``Permissive``::
|
||||||
|
|
||||||
sudo setenforce Permissive
|
sudo setenforce Permissive
|
||||||
|
|
||||||
Or just allow rsync full access::
|
Or just allow rsync full access::
|
||||||
|
|
||||||
sudo setsebool -P rsync_full_access 1
|
sudo setsebool -P rsync_full_access 1
|
||||||
|
|
||||||
#. Start the rsync daemon
|
#. Start the rsync daemon
|
||||||
|
|
||||||
* On Ubuntu 14.04, run::
|
* On Ubuntu 14.04, run::
|
||||||
|
|
||||||
sudo service rsync restart
|
sudo service rsync restart
|
||||||
|
|
||||||
* On Ubuntu 16.04, run::
|
* On Ubuntu 16.04, run::
|
||||||
|
|
||||||
sudo systemctl enable rsync
|
sudo systemctl enable rsync
|
||||||
sudo systemctl start rsync
|
sudo systemctl start rsync
|
||||||
|
|
||||||
* On Fedora, run::
|
* On Fedora, run::
|
||||||
|
|
||||||
sudo systemctl restart xinetd.service
|
sudo systemctl restart xinetd.service
|
||||||
sudo systemctl enable rsyncd.service
|
sudo systemctl enable rsyncd.service
|
||||||
sudo systemctl start rsyncd.service
|
sudo systemctl start rsyncd.service
|
||||||
|
|
||||||
* On OpenSuse, run::
|
* On OpenSuse, run::
|
||||||
|
|
||||||
sudo systemctl enable rsyncd.service
|
sudo systemctl enable rsyncd.service
|
||||||
sudo systemctl start rsyncd.service
|
sudo systemctl start rsyncd.service
|
||||||
|
|
||||||
* On other xinetd based systems simply run::
|
* On other xinetd based systems simply run::
|
||||||
|
|
||||||
sudo service xinetd restart
|
sudo service xinetd restart
|
||||||
|
|
||||||
#. Verify rsync is accepting connections for all servers::
|
#. Verify rsync is accepting connections for all servers::
|
||||||
|
|
||||||
rsync rsync://pub@localhost/
|
rsync rsync://pub@localhost/
|
||||||
|
|
||||||
You should see the following output from the above command::
|
You should see the following output from the above command::
|
||||||
|
|
||||||
account6012
|
account6012
|
||||||
account6022
|
account6022
|
||||||
account6032
|
account6032
|
||||||
account6042
|
account6042
|
||||||
container6011
|
container6011
|
||||||
container6021
|
container6021
|
||||||
container6031
|
container6031
|
||||||
container6041
|
container6041
|
||||||
object6010
|
object6010
|
||||||
object6020
|
object6020
|
||||||
object6030
|
object6030
|
||||||
object6040
|
object6040
|
||||||
|
|
||||||
------------------
|
------------------
|
||||||
Starting memcached
|
Starting memcached
|
||||||
|
@ -362,50 +370,51 @@ running, tokens cannot be validated, and accessing Swift becomes impossible.
|
||||||
Optional: Setting up rsyslog for individual logging
|
Optional: Setting up rsyslog for individual logging
|
||||||
---------------------------------------------------
|
---------------------------------------------------
|
||||||
|
|
||||||
#. Install the swift rsyslogd configuration::
|
#. Install the swift rsyslogd configuration::
|
||||||
|
|
||||||
sudo cp $HOME/swift/doc/saio/rsyslog.d/10-swift.conf /etc/rsyslog.d/
|
sudo cp $HOME/swift/doc/saio/rsyslog.d/10-swift.conf /etc/rsyslog.d/
|
||||||
|
|
||||||
Note: OpenSuse may have the systemd logger installed, so if you want this
|
Note: OpenSuse may have the systemd logger installed, so if you want this
|
||||||
to work, you need to install rsyslog::
|
to work, you need to install rsyslog::
|
||||||
|
|
||||||
sudo zypper install rsyslog
|
sudo zypper install rsyslog
|
||||||
sudo systemctl start rsyslog.service
|
sudo systemctl start rsyslog.service
|
||||||
sudo systemctl enable rsyslog.service
|
sudo systemctl enable rsyslog.service
|
||||||
|
|
||||||
Be sure to review that conf file to determine if you want all the logs
|
Be sure to review that conf file to determine if you want all the logs
|
||||||
in one file vs. all the logs separated out, and if you want hourly logs
|
in one file vs. all the logs separated out, and if you want hourly logs
|
||||||
for stats processing. For convenience, we provide its default contents
|
for stats processing. For convenience, we provide its default contents
|
||||||
below:
|
below:
|
||||||
|
|
||||||
.. literalinclude:: /../saio/rsyslog.d/10-swift.conf
|
.. literalinclude:: /../saio/rsyslog.d/10-swift.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. Edit ``/etc/rsyslog.conf`` and make the following change (usually in the
|
#. Edit ``/etc/rsyslog.conf`` and make the following change (usually in the
|
||||||
"GLOBAL DIRECTIVES" section)::
|
"GLOBAL DIRECTIVES" section)::
|
||||||
|
|
||||||
$PrivDropToGroup adm
|
$PrivDropToGroup adm
|
||||||
|
|
||||||
#. If using hourly logs (see above) perform::
|
#. If using hourly logs (see above) perform::
|
||||||
|
|
||||||
sudo mkdir -p /var/log/swift/hourly
|
sudo mkdir -p /var/log/swift/hourly
|
||||||
|
|
||||||
Otherwise perform::
|
Otherwise perform::
|
||||||
|
|
||||||
sudo mkdir -p /var/log/swift
|
sudo mkdir -p /var/log/swift
|
||||||
|
|
||||||
#. Setup the logging directory and start syslog:
|
#. Setup the logging directory and start syslog:
|
||||||
|
|
||||||
* On Ubuntu::
|
* On Ubuntu::
|
||||||
|
|
||||||
sudo chown -R syslog.adm /var/log/swift
|
sudo chown -R syslog.adm /var/log/swift
|
||||||
sudo chmod -R g+w /var/log/swift
|
sudo chmod -R g+w /var/log/swift
|
||||||
sudo service rsyslog restart
|
sudo service rsyslog restart
|
||||||
|
|
||||||
* On Fedora and OpenSuse::
|
* On Fedora and OpenSuse::
|
||||||
|
|
||||||
sudo chown -R root:adm /var/log/swift
|
sudo chown -R root:adm /var/log/swift
|
||||||
sudo chmod -R g+w /var/log/swift
|
sudo chmod -R g+w /var/log/swift
|
||||||
sudo systemctl restart rsyslog.service
|
sudo systemctl restart rsyslog.service
|
||||||
|
|
||||||
---------------------
|
---------------------
|
||||||
Configuring each node
|
Configuring each node
|
||||||
|
@ -415,89 +424,106 @@ After performing the following steps, be sure to verify that Swift has access
|
||||||
to resulting configuration files (sample configuration files are provided with
|
to resulting configuration files (sample configuration files are provided with
|
||||||
all defaults in line-by-line comments).
|
all defaults in line-by-line comments).
|
||||||
|
|
||||||
#. Optionally remove an existing swift directory::
|
#. Optionally remove an existing swift directory::
|
||||||
|
|
||||||
sudo rm -rf /etc/swift
|
sudo rm -rf /etc/swift
|
||||||
|
|
||||||
#. Populate the ``/etc/swift`` directory itself::
|
#. Populate the ``/etc/swift`` directory itself::
|
||||||
|
|
||||||
cd $HOME/swift/doc; sudo cp -r saio/swift /etc/swift; cd -
|
cd $HOME/swift/doc; sudo cp -r saio/swift /etc/swift; cd -
|
||||||
sudo chown -R ${USER}:${USER} /etc/swift
|
sudo chown -R ${USER}:${USER} /etc/swift
|
||||||
|
|
||||||
#. Update ``<your-user-name>`` references in the Swift config files::
|
#. Update ``<your-user-name>`` references in the Swift config files::
|
||||||
|
|
||||||
find /etc/swift/ -name \*.conf | xargs sudo sed -i "s/<your-user-name>/${USER}/"
|
find /etc/swift/ -name \*.conf | xargs sudo sed -i "s/<your-user-name>/${USER}/"
|
||||||
|
|
||||||
The contents of the configuration files provided by executing the above
|
The contents of the configuration files provided by executing the above
|
||||||
commands are as follows:
|
commands are as follows:
|
||||||
|
|
||||||
#. ``/etc/swift/swift.conf``
|
#. ``/etc/swift/swift.conf``
|
||||||
|
|
||||||
.. literalinclude:: /../saio/swift/swift.conf
|
.. literalinclude:: /../saio/swift/swift.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. ``/etc/swift/proxy-server.conf``
|
#. ``/etc/swift/proxy-server.conf``
|
||||||
|
|
||||||
.. literalinclude:: /../saio/swift/proxy-server.conf
|
.. literalinclude:: /../saio/swift/proxy-server.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. ``/etc/swift/object-expirer.conf``
|
#. ``/etc/swift/object-expirer.conf``
|
||||||
|
|
||||||
.. literalinclude:: /../saio/swift/object-expirer.conf
|
.. literalinclude:: /../saio/swift/object-expirer.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. ``/etc/swift/container-reconciler.conf``
|
#. ``/etc/swift/container-reconciler.conf``
|
||||||
|
|
||||||
.. literalinclude:: /../saio/swift/container-reconciler.conf
|
.. literalinclude:: /../saio/swift/container-reconciler.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. ``/etc/swift/container-sync-realms.conf``
|
#. ``/etc/swift/container-sync-realms.conf``
|
||||||
|
|
||||||
.. literalinclude:: /../saio/swift/container-sync-realms.conf
|
.. literalinclude:: /../saio/swift/container-sync-realms.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. ``/etc/swift/account-server/1.conf``
|
#. ``/etc/swift/account-server/1.conf``
|
||||||
|
|
||||||
.. literalinclude:: /../saio/swift/account-server/1.conf
|
.. literalinclude:: /../saio/swift/account-server/1.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. ``/etc/swift/container-server/1.conf``
|
#. ``/etc/swift/container-server/1.conf``
|
||||||
|
|
||||||
.. literalinclude:: /../saio/swift/container-server/1.conf
|
.. literalinclude:: /../saio/swift/container-server/1.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. ``/etc/swift/object-server/1.conf``
|
#. ``/etc/swift/object-server/1.conf``
|
||||||
|
|
||||||
.. literalinclude:: /../saio/swift/object-server/1.conf
|
.. literalinclude:: /../saio/swift/object-server/1.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. ``/etc/swift/account-server/2.conf``
|
#. ``/etc/swift/account-server/2.conf``
|
||||||
|
|
||||||
.. literalinclude:: /../saio/swift/account-server/2.conf
|
.. literalinclude:: /../saio/swift/account-server/2.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. ``/etc/swift/container-server/2.conf``
|
#. ``/etc/swift/container-server/2.conf``
|
||||||
|
|
||||||
.. literalinclude:: /../saio/swift/container-server/2.conf
|
.. literalinclude:: /../saio/swift/container-server/2.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. ``/etc/swift/object-server/2.conf``
|
#. ``/etc/swift/object-server/2.conf``
|
||||||
|
|
||||||
.. literalinclude:: /../saio/swift/object-server/2.conf
|
.. literalinclude:: /../saio/swift/object-server/2.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. ``/etc/swift/account-server/3.conf``
|
#. ``/etc/swift/account-server/3.conf``
|
||||||
|
|
||||||
.. literalinclude:: /../saio/swift/account-server/3.conf
|
.. literalinclude:: /../saio/swift/account-server/3.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. ``/etc/swift/container-server/3.conf``
|
#. ``/etc/swift/container-server/3.conf``
|
||||||
|
|
||||||
.. literalinclude:: /../saio/swift/container-server/3.conf
|
.. literalinclude:: /../saio/swift/container-server/3.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. ``/etc/swift/object-server/3.conf``
|
#. ``/etc/swift/object-server/3.conf``
|
||||||
|
|
||||||
.. literalinclude:: /../saio/swift/object-server/3.conf
|
.. literalinclude:: /../saio/swift/object-server/3.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. ``/etc/swift/account-server/4.conf``
|
#. ``/etc/swift/account-server/4.conf``
|
||||||
|
|
||||||
.. literalinclude:: /../saio/swift/account-server/4.conf
|
.. literalinclude:: /../saio/swift/account-server/4.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. ``/etc/swift/container-server/4.conf``
|
#. ``/etc/swift/container-server/4.conf``
|
||||||
|
|
||||||
.. literalinclude:: /../saio/swift/container-server/4.conf
|
.. literalinclude:: /../saio/swift/container-server/4.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. ``/etc/swift/object-server/4.conf``
|
#. ``/etc/swift/object-server/4.conf``
|
||||||
|
|
||||||
.. literalinclude:: /../saio/swift/object-server/4.conf
|
.. literalinclude:: /../saio/swift/object-server/4.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
.. _setup_scripts:
|
.. _setup_scripts:
|
||||||
|
|
||||||
|
@ -505,139 +531,146 @@ commands are as follows:
|
||||||
Setting up scripts for running Swift
|
Setting up scripts for running Swift
|
||||||
------------------------------------
|
------------------------------------
|
||||||
|
|
||||||
#. Copy the SAIO scripts for resetting the environment::
|
#. Copy the SAIO scripts for resetting the environment::
|
||||||
|
|
||||||
mkdir -p $HOME/bin
|
mkdir -p $HOME/bin
|
||||||
cd $HOME/swift/doc; cp saio/bin/* $HOME/bin; cd -
|
cd $HOME/swift/doc; cp saio/bin/* $HOME/bin; cd -
|
||||||
chmod +x $HOME/bin/*
|
chmod +x $HOME/bin/*
|
||||||
|
|
||||||
#. Edit the ``$HOME/bin/resetswift`` script
|
#. Edit the ``$HOME/bin/resetswift`` script
|
||||||
|
|
||||||
The template ``resetswift`` script looks like the following:
|
The template ``resetswift`` script looks like the following:
|
||||||
|
|
||||||
.. literalinclude:: /../saio/bin/resetswift
|
.. literalinclude:: /../saio/bin/resetswift
|
||||||
|
:language: bash
|
||||||
|
|
||||||
If you are using a loopback device add an environment var to
|
If you are using a loopback device add an environment var to
|
||||||
substitute ``/dev/sdb1`` with ``/srv/swift-disk``::
|
substitute ``/dev/sdb1`` with ``/srv/swift-disk``::
|
||||||
|
|
||||||
echo "export SAIO_BLOCK_DEVICE=/srv/swift-disk" >> $HOME/.bashrc
|
echo "export SAIO_BLOCK_DEVICE=/srv/swift-disk" >> $HOME/.bashrc
|
||||||
|
|
||||||
If you did not set up rsyslog for individual logging, remove the ``find
|
If you did not set up rsyslog for individual logging, remove the ``find
|
||||||
/var/log/swift...`` line::
|
/var/log/swift...`` line::
|
||||||
|
|
||||||
sed -i "/find \/var\/log\/swift/d" $HOME/bin/resetswift
|
sed -i "/find \/var\/log\/swift/d" $HOME/bin/resetswift
|
||||||
|
|
||||||
|
|
||||||
#. Install the sample configuration file for running tests::
|
#. Install the sample configuration file for running tests::
|
||||||
|
|
||||||
cp $HOME/swift/test/sample.conf /etc/swift/test.conf
|
cp $HOME/swift/test/sample.conf /etc/swift/test.conf
|
||||||
|
|
||||||
The template ``test.conf`` looks like the following:
|
The template ``test.conf`` looks like the following:
|
||||||
|
|
||||||
.. literalinclude:: /../../test/sample.conf
|
.. literalinclude:: /../../test/sample.conf
|
||||||
|
:language: ini
|
||||||
|
|
||||||
#. Add an environment variable for running tests below::
|
#. Add an environment variable for running tests below::
|
||||||
|
|
||||||
echo "export SWIFT_TEST_CONFIG_FILE=/etc/swift/test.conf" >> $HOME/.bashrc
|
echo "export SWIFT_TEST_CONFIG_FILE=/etc/swift/test.conf" >> $HOME/.bashrc
|
||||||
|
|
||||||
#. Be sure that your ``PATH`` includes the ``bin`` directory::
|
#. Be sure that your ``PATH`` includes the ``bin`` directory::
|
||||||
|
|
||||||
echo "export PATH=${PATH}:$HOME/bin" >> $HOME/.bashrc
|
echo "export PATH=${PATH}:$HOME/bin" >> $HOME/.bashrc
|
||||||
|
|
||||||
#. Source the above environment variables into your current environment::
|
#. Source the above environment variables into your current environment::
|
||||||
|
|
||||||
. $HOME/.bashrc
|
. $HOME/.bashrc
|
||||||
|
|
||||||
#. Construct the initial rings using the provided script::
|
#. Construct the initial rings using the provided script::
|
||||||
|
|
||||||
remakerings
|
remakerings
|
||||||
|
|
||||||
The ``remakerings`` script looks like the following:
|
The ``remakerings`` script looks like the following:
|
||||||
|
|
||||||
.. literalinclude:: /../saio/bin/remakerings
|
.. literalinclude:: /../saio/bin/remakerings
|
||||||
|
:language: bash
|
||||||
|
|
||||||
You can expect the output from this command to produce the following. Note
|
You can expect the output from this command to produce the following. Note
|
||||||
that 3 object rings are created in order to test storage policies and EC in
|
that 3 object rings are created in order to test storage policies and EC in
|
||||||
the SAIO environment. The EC ring is the only one with all 8 devices.
|
the SAIO environment. The EC ring is the only one with all 8 devices.
|
||||||
There are also two replication rings, one for 3x replication and another
|
There are also two replication rings, one for 3x replication and another
|
||||||
for 2x replication, but those rings only use 4 devices::
|
for 2x replication, but those rings only use 4 devices:
|
||||||
|
|
||||||
Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
|
|
||||||
Device d1r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 1
|
|
||||||
Device d2r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 2
|
|
||||||
Device d3r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 3
|
|
||||||
Reassigned 3072 (300.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
|
|
||||||
Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
|
|
||||||
Device d1r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 1
|
|
||||||
Device d2r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 2
|
|
||||||
Device d3r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 3
|
|
||||||
Reassigned 2048 (200.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
|
|
||||||
Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
|
|
||||||
Device d1r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb5_"" with 1.0 weight got id 1
|
|
||||||
Device d2r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 2
|
|
||||||
Device d3r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb6_"" with 1.0 weight got id 3
|
|
||||||
Device d4r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 4
|
|
||||||
Device d5r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb7_"" with 1.0 weight got id 5
|
|
||||||
Device d6r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 6
|
|
||||||
Device d7r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb8_"" with 1.0 weight got id 7
|
|
||||||
Reassigned 6144 (600.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
|
|
||||||
Device d0r1z1-127.0.0.1:6011R127.0.0.1:6011/sdb1_"" with 1.0 weight got id 0
|
|
||||||
Device d1r1z2-127.0.0.2:6021R127.0.0.2:6021/sdb2_"" with 1.0 weight got id 1
|
|
||||||
Device d2r1z3-127.0.0.3:6031R127.0.0.3:6031/sdb3_"" with 1.0 weight got id 2
|
|
||||||
Device d3r1z4-127.0.0.4:6041R127.0.0.4:6041/sdb4_"" with 1.0 weight got id 3
|
|
||||||
Reassigned 3072 (300.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
|
|
||||||
Device d0r1z1-127.0.0.1:6012R127.0.0.1:6012/sdb1_"" with 1.0 weight got id 0
|
|
||||||
Device d1r1z2-127.0.0.2:6022R127.0.0.2:6022/sdb2_"" with 1.0 weight got id 1
|
|
||||||
Device d2r1z3-127.0.0.3:6032R127.0.0.3:6032/sdb3_"" with 1.0 weight got id 2
|
|
||||||
Device d3r1z4-127.0.0.4:6042R127.0.0.4:6042/sdb4_"" with 1.0 weight got id 3
|
|
||||||
Reassigned 3072 (300.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
|
|
||||||
|
|
||||||
|
|
||||||
#. Read more about Storage Policies and your SAIO :doc:`policies_saio`
|
.. code-block:: console
|
||||||
|
|
||||||
#. Verify the unit tests run::
|
Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
|
||||||
|
Device d1r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 1
|
||||||
|
Device d2r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 2
|
||||||
|
Device d3r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 3
|
||||||
|
Reassigned 3072 (300.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
|
||||||
|
Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
|
||||||
|
Device d1r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 1
|
||||||
|
Device d2r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 2
|
||||||
|
Device d3r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 3
|
||||||
|
Reassigned 2048 (200.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
|
||||||
|
Device d0r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb1_"" with 1.0 weight got id 0
|
||||||
|
Device d1r1z1-127.0.0.1:6010R127.0.0.1:6010/sdb5_"" with 1.0 weight got id 1
|
||||||
|
Device d2r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb2_"" with 1.0 weight got id 2
|
||||||
|
Device d3r1z2-127.0.0.2:6020R127.0.0.2:6020/sdb6_"" with 1.0 weight got id 3
|
||||||
|
Device d4r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb3_"" with 1.0 weight got id 4
|
||||||
|
Device d5r1z3-127.0.0.3:6030R127.0.0.3:6030/sdb7_"" with 1.0 weight got id 5
|
||||||
|
Device d6r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb4_"" with 1.0 weight got id 6
|
||||||
|
Device d7r1z4-127.0.0.4:6040R127.0.0.4:6040/sdb8_"" with 1.0 weight got id 7
|
||||||
|
Reassigned 6144 (600.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
|
||||||
|
Device d0r1z1-127.0.0.1:6011R127.0.0.1:6011/sdb1_"" with 1.0 weight got id 0
|
||||||
|
Device d1r1z2-127.0.0.2:6021R127.0.0.2:6021/sdb2_"" with 1.0 weight got id 1
|
||||||
|
Device d2r1z3-127.0.0.3:6031R127.0.0.3:6031/sdb3_"" with 1.0 weight got id 2
|
||||||
|
Device d3r1z4-127.0.0.4:6041R127.0.0.4:6041/sdb4_"" with 1.0 weight got id 3
|
||||||
|
Reassigned 3072 (300.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
|
||||||
|
Device d0r1z1-127.0.0.1:6012R127.0.0.1:6012/sdb1_"" with 1.0 weight got id 0
|
||||||
|
Device d1r1z2-127.0.0.2:6022R127.0.0.2:6022/sdb2_"" with 1.0 weight got id 1
|
||||||
|
Device d2r1z3-127.0.0.3:6032R127.0.0.3:6032/sdb3_"" with 1.0 weight got id 2
|
||||||
|
Device d3r1z4-127.0.0.4:6042R127.0.0.4:6042/sdb4_"" with 1.0 weight got id 3
|
||||||
|
Reassigned 3072 (300.00%) partitions. Balance is now 0.00. Dispersion is now 0.00
|
||||||
|
|
||||||
$HOME/swift/.unittests
|
|
||||||
|
|
||||||
Note that the unit tests do not require any swift daemons running.
|
#. Read more about Storage Policies and your SAIO :doc:`policies_saio`
|
||||||
|
|
||||||
#. Start the "main" Swift daemon processes (proxy, account, container, and
|
#. Verify the unit tests run::
|
||||||
object)::
|
|
||||||
|
|
||||||
startmain
|
$HOME/swift/.unittests
|
||||||
|
|
||||||
(The "``Unable to increase file descriptor limit. Running as non-root?``"
|
Note that the unit tests do not require any swift daemons running.
|
||||||
warnings are expected and ok.)
|
|
||||||
|
|
||||||
The ``startmain`` script looks like the following:
|
#. Start the "main" Swift daemon processes (proxy, account, container, and
|
||||||
|
object)::
|
||||||
|
|
||||||
.. literalinclude:: /../saio/bin/startmain
|
startmain
|
||||||
|
|
||||||
#. Get an ``X-Storage-Url`` and ``X-Auth-Token``::
|
(The "``Unable to increase file descriptor limit. Running as non-root?``"
|
||||||
|
warnings are expected and ok.)
|
||||||
|
|
||||||
curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0
|
The ``startmain`` script looks like the following:
|
||||||
|
|
||||||
#. Check that you can ``GET`` account::
|
.. literalinclude:: /../saio/bin/startmain
|
||||||
|
:language: bash
|
||||||
|
|
||||||
curl -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>
|
#. Get an ``X-Storage-Url`` and ``X-Auth-Token``::
|
||||||
|
|
||||||
#. Check that ``swift`` command provided by the python-swiftclient package works::
|
curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0
|
||||||
|
|
||||||
swift -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat
|
#. Check that you can ``GET`` account::
|
||||||
|
|
||||||
#. Verify the functional tests run::
|
curl -v -H 'X-Auth-Token: <token-from-x-auth-token-above>' <url-from-x-storage-url-above>
|
||||||
|
|
||||||
$HOME/swift/.functests
|
#. Check that ``swift`` command provided by the python-swiftclient package works::
|
||||||
|
|
||||||
(Note: functional tests will first delete everything in the configured
|
swift -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat
|
||||||
accounts.)
|
|
||||||
|
|
||||||
#. Verify the probe tests run::
|
#. Verify the functional tests run::
|
||||||
|
|
||||||
$HOME/swift/.probetests
|
$HOME/swift/.functests
|
||||||
|
|
||||||
(Note: probe tests will reset your environment as they call ``resetswift``
|
(Note: functional tests will first delete everything in the configured
|
||||||
for each test.)
|
accounts.)
|
||||||
|
|
||||||
|
#. Verify the probe tests run::
|
||||||
|
|
||||||
|
$HOME/swift/.probetests
|
||||||
|
|
||||||
|
(Note: probe tests will reset your environment as they call ``resetswift``
|
||||||
|
for each test.)
|
||||||
|
|
||||||
----------------
|
----------------
|
||||||
Debugging Issues
|
Debugging Issues
|
||||||
|
|
|
@ -589,7 +589,8 @@ reseller_prefix = AUTH_
|
||||||
# useful if there are multiple auth systems in the proxy pipeline.
|
# useful if there are multiple auth systems in the proxy pipeline.
|
||||||
delay_auth_decision = False
|
delay_auth_decision = False
|
||||||
|
|
||||||
# Keystone server details
|
# Keystone server details. Note that this differs from how swift3 was
|
||||||
|
# configured: in particular, the Keystone API version must be included.
|
||||||
auth_uri = http://keystonehost:35357/v3
|
auth_uri = http://keystonehost:35357/v3
|
||||||
|
|
||||||
# Connect/read timeout to use when communicating with Keystone
|
# Connect/read timeout to use when communicating with Keystone
|
||||||
|
|
|
@ -0,0 +1,69 @@
|
||||||
|
---
|
||||||
|
features:
|
||||||
|
- |
|
||||||
|
Change the behavior of the EC reconstructor to perform a
|
||||||
|
fragment rebuild to a handoff node when a primary peer responds
|
||||||
|
with 507 to the REPLICATE request. This changes EC to match the
|
||||||
|
existing behavior of replication when drives fail. After a
|
||||||
|
rebalance of EC rings (potentially removing unmounted/failed
|
||||||
|
devices), it's most IO efficient to run in handoffs_only mode to
|
||||||
|
avoid unnecessary rebuilds.
|
||||||
|
|
||||||
|
- |
|
||||||
|
O_TMPFILE support is now detected by attempting to use it
|
||||||
|
instead of looking at the kernel version. This allows older
|
||||||
|
kernels with backported patches to take advantage of the
|
||||||
|
O_TMPFILE functionality.
|
||||||
|
|
||||||
|
- |
|
||||||
|
Add slo_manifest_hook callback to allow other middlewares to
|
||||||
|
impose additional constraints on or make edits to SLO manifests
|
||||||
|
before being written. For example, a middleware could enforce
|
||||||
|
minimum segment size or insert data segments.
|
||||||
|
|
||||||
|
- |
|
||||||
|
Fixed an issue with multi-region EC policies that caused the EC
|
||||||
|
reconstructor to constantly attempt cross-region rebuild
|
||||||
|
traffic.
|
||||||
|
|
||||||
|
- |
|
||||||
|
Fixed an issue where S3 API v4 signatures would not be validated
|
||||||
|
against the body of the request, allowing a replay attack if
|
||||||
|
request headers were captured by a malicious third party.
|
||||||
|
|
||||||
|
- Display crypto data/metadata details in swift-object-info.
|
||||||
|
|
||||||
|
- formpost can now accept a content-encoding parameter.
|
||||||
|
|
||||||
|
- |
|
||||||
|
Fixed an issue where multipart uploads with the S3 API would
|
||||||
|
sometimes report an error despite all segments being upload
|
||||||
|
successfully.
|
||||||
|
|
||||||
|
- |
|
||||||
|
Multipart object segments are now actually deleted when the
|
||||||
|
multipart object is deleted via the S3 API.
|
||||||
|
|
||||||
|
- |
|
||||||
|
Swift now returns a 503 (instead of a 500) when an account
|
||||||
|
auto-create fails.
|
||||||
|
|
||||||
|
- |
|
||||||
|
Fixed a bug where encryption would store the incorrect key
|
||||||
|
metadata if the object name starts with a slash.
|
||||||
|
|
||||||
|
- |
|
||||||
|
Fixed an issue where an object server failure during a client
|
||||||
|
download could leave an open socket between the proxy and
|
||||||
|
client.
|
||||||
|
|
||||||
|
- |
|
||||||
|
Fixed an issue where deleted EC objects didn't have their
|
||||||
|
on-disk directories cleaned up. This would cause extra resource
|
||||||
|
usage on the object servers.
|
||||||
|
|
||||||
|
- |
|
||||||
|
Fixed issue where bulk requests using xml and expect
|
||||||
|
100-continue would return a malformed HTTP response.
|
||||||
|
|
||||||
|
- Various other minor bug fixes and improvements.
|
|
@ -7,6 +7,8 @@
|
||||||
|
|
||||||
current
|
current
|
||||||
|
|
||||||
|
stein
|
||||||
|
|
||||||
rocky
|
rocky
|
||||||
|
|
||||||
queens
|
queens
|
||||||
|
|
|
@ -0,0 +1,6 @@
|
||||||
|
===================================
|
||||||
|
Stein Series Release Notes
|
||||||
|
===================================
|
||||||
|
|
||||||
|
.. release-notes::
|
||||||
|
:branch: stable/stein
|
|
@ -16,7 +16,6 @@
|
||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
import socket
|
import socket
|
||||||
from swift import gettext_ as _
|
|
||||||
from logging import DEBUG
|
from logging import DEBUG
|
||||||
from math import sqrt
|
from math import sqrt
|
||||||
from time import time
|
from time import time
|
||||||
|
@ -142,10 +141,10 @@ class AccountReaper(Daemon):
|
||||||
continue
|
continue
|
||||||
self.reap_device(device)
|
self.reap_device(device)
|
||||||
except (Exception, Timeout):
|
except (Exception, Timeout):
|
||||||
self.logger.exception(_("Exception in top-level account reaper "
|
self.logger.exception("Exception in top-level account reaper "
|
||||||
"loop"))
|
"loop")
|
||||||
elapsed = time() - begin
|
elapsed = time() - begin
|
||||||
self.logger.info(_('Devices pass completed: %.02fs'), elapsed)
|
self.logger.info('Devices pass completed: %.02fs', elapsed)
|
||||||
|
|
||||||
def reap_device(self, device):
|
def reap_device(self, device):
|
||||||
"""
|
"""
|
||||||
|
@ -255,19 +254,15 @@ class AccountReaper(Daemon):
|
||||||
self.delay_reaping:
|
self.delay_reaping:
|
||||||
return False
|
return False
|
||||||
account = info['account']
|
account = info['account']
|
||||||
self.logger.info(_('Beginning pass on account %s'), account)
|
self.logger.info('Beginning pass on account %s', account)
|
||||||
self.reset_stats()
|
self.reset_stats()
|
||||||
container_limit = 1000
|
container_limit = 1000
|
||||||
if container_shard is not None:
|
if container_shard is not None:
|
||||||
container_limit *= len(nodes)
|
container_limit *= len(nodes)
|
||||||
try:
|
try:
|
||||||
marker = ''
|
containers = list(broker.list_containers_iter(
|
||||||
while True:
|
container_limit, '', None, None, None))
|
||||||
containers = \
|
while containers:
|
||||||
list(broker.list_containers_iter(container_limit, marker,
|
|
||||||
None, None, None))
|
|
||||||
if not containers:
|
|
||||||
break
|
|
||||||
try:
|
try:
|
||||||
for (container, _junk, _junk, _junk, _junk) in containers:
|
for (container, _junk, _junk, _junk, _junk) in containers:
|
||||||
if six.PY3:
|
if six.PY3:
|
||||||
|
@ -284,43 +279,44 @@ class AccountReaper(Daemon):
|
||||||
self.container_pool.waitall()
|
self.container_pool.waitall()
|
||||||
except (Exception, Timeout):
|
except (Exception, Timeout):
|
||||||
self.logger.exception(
|
self.logger.exception(
|
||||||
_('Exception with containers for account %s'), account)
|
'Exception with containers for account %s', account)
|
||||||
marker = containers[-1][0]
|
containers = list(broker.list_containers_iter(
|
||||||
if marker == '':
|
container_limit, containers[-1][0], None, None, None))
|
||||||
break
|
log_buf = ['Completed pass on account %s' % account]
|
||||||
log = 'Completed pass on account %s' % account
|
|
||||||
except (Exception, Timeout):
|
except (Exception, Timeout):
|
||||||
self.logger.exception(
|
self.logger.exception('Exception with account %s', account)
|
||||||
_('Exception with account %s'), account)
|
log_buf = ['Incomplete pass on account %s' % account]
|
||||||
log = _('Incomplete pass on account %s') % account
|
|
||||||
if self.stats_containers_deleted:
|
if self.stats_containers_deleted:
|
||||||
log += _(', %s containers deleted') % self.stats_containers_deleted
|
log_buf.append(', %s containers deleted' %
|
||||||
|
self.stats_containers_deleted)
|
||||||
if self.stats_objects_deleted:
|
if self.stats_objects_deleted:
|
||||||
log += _(', %s objects deleted') % self.stats_objects_deleted
|
log_buf.append(', %s objects deleted' % self.stats_objects_deleted)
|
||||||
if self.stats_containers_remaining:
|
if self.stats_containers_remaining:
|
||||||
log += _(', %s containers remaining') % \
|
log_buf.append(', %s containers remaining' %
|
||||||
self.stats_containers_remaining
|
self.stats_containers_remaining)
|
||||||
if self.stats_objects_remaining:
|
if self.stats_objects_remaining:
|
||||||
log += _(', %s objects remaining') % self.stats_objects_remaining
|
log_buf.append(', %s objects remaining' %
|
||||||
|
self.stats_objects_remaining)
|
||||||
if self.stats_containers_possibly_remaining:
|
if self.stats_containers_possibly_remaining:
|
||||||
log += _(', %s containers possibly remaining') % \
|
log_buf.append(', %s containers possibly remaining' %
|
||||||
self.stats_containers_possibly_remaining
|
self.stats_containers_possibly_remaining)
|
||||||
if self.stats_objects_possibly_remaining:
|
if self.stats_objects_possibly_remaining:
|
||||||
log += _(', %s objects possibly remaining') % \
|
log_buf.append(', %s objects possibly remaining' %
|
||||||
self.stats_objects_possibly_remaining
|
self.stats_objects_possibly_remaining)
|
||||||
if self.stats_return_codes:
|
if self.stats_return_codes:
|
||||||
log += _(', return codes: ')
|
log_buf.append(', return codes: ')
|
||||||
for code in sorted(self.stats_return_codes):
|
for code in sorted(self.stats_return_codes):
|
||||||
log += '%s %sxxs, ' % (self.stats_return_codes[code], code)
|
log_buf.append('%s %sxxs, ' % (self.stats_return_codes[code],
|
||||||
log = log[:-2]
|
code))
|
||||||
log += _(', elapsed: %.02fs') % (time() - begin)
|
log_buf[-1] = log_buf[-1][:-2]
|
||||||
self.logger.info(log)
|
log_buf.append(', elapsed: %.02fs' % (time() - begin))
|
||||||
|
self.logger.info(''.join(log_buf))
|
||||||
self.logger.timing_since('timing', self.start_time)
|
self.logger.timing_since('timing', self.start_time)
|
||||||
delete_timestamp = Timestamp(info['delete_timestamp'])
|
delete_timestamp = Timestamp(info['delete_timestamp'])
|
||||||
if self.stats_containers_remaining and \
|
if self.stats_containers_remaining and \
|
||||||
begin - float(delete_timestamp) >= self.reap_not_done_after:
|
begin - float(delete_timestamp) >= self.reap_not_done_after:
|
||||||
self.logger.warning(
|
self.logger.warning(
|
||||||
_('Account %(account)s has not been reaped since %(time)s') %
|
'Account %(account)s has not been reaped since %(time)s' %
|
||||||
{'account': account, 'time': delete_timestamp.isoformat})
|
{'account': account, 'time': delete_timestamp.isoformat})
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
@ -379,14 +375,14 @@ class AccountReaper(Daemon):
|
||||||
except ClientException as err:
|
except ClientException as err:
|
||||||
if self.logger.getEffectiveLevel() <= DEBUG:
|
if self.logger.getEffectiveLevel() <= DEBUG:
|
||||||
self.logger.exception(
|
self.logger.exception(
|
||||||
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
|
'Exception with %(ip)s:%(port)s/%(device)s', node)
|
||||||
self.stats_return_codes[err.http_status // 100] = \
|
self.stats_return_codes[err.http_status // 100] = \
|
||||||
self.stats_return_codes.get(err.http_status // 100, 0) + 1
|
self.stats_return_codes.get(err.http_status // 100, 0) + 1
|
||||||
self.logger.increment(
|
self.logger.increment(
|
||||||
'return_codes.%d' % (err.http_status // 100,))
|
'return_codes.%d' % (err.http_status // 100,))
|
||||||
except (Timeout, socket.error) as err:
|
except (Timeout, socket.error) as err:
|
||||||
self.logger.error(
|
self.logger.error(
|
||||||
_('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
|
'Timeout Exception with %(ip)s:%(port)s/%(device)s',
|
||||||
node)
|
node)
|
||||||
if not objects:
|
if not objects:
|
||||||
break
|
break
|
||||||
|
@ -397,21 +393,15 @@ class AccountReaper(Daemon):
|
||||||
self.logger.error('ERROR: invalid storage policy index: %r'
|
self.logger.error('ERROR: invalid storage policy index: %r'
|
||||||
% policy_index)
|
% policy_index)
|
||||||
for obj in objects:
|
for obj in objects:
|
||||||
obj_name = obj['name']
|
|
||||||
if isinstance(obj_name, six.text_type):
|
|
||||||
obj_name = obj_name.encode('utf8')
|
|
||||||
pool.spawn(self.reap_object, account, container, part,
|
pool.spawn(self.reap_object, account, container, part,
|
||||||
nodes, obj_name, policy_index)
|
nodes, obj['name'], policy_index)
|
||||||
pool.waitall()
|
pool.waitall()
|
||||||
except (Exception, Timeout):
|
except (Exception, Timeout):
|
||||||
self.logger.exception(_('Exception with objects for container '
|
self.logger.exception('Exception with objects for container '
|
||||||
'%(container)s for account %(account)s'
|
'%(container)s for account %(account)s',
|
||||||
),
|
|
||||||
{'container': container,
|
{'container': container,
|
||||||
'account': account})
|
'account': account})
|
||||||
marker = objects[-1]['name']
|
marker = objects[-1]['name']
|
||||||
if marker == '':
|
|
||||||
break
|
|
||||||
successes = 0
|
successes = 0
|
||||||
failures = 0
|
failures = 0
|
||||||
timestamp = Timestamp.now()
|
timestamp = Timestamp.now()
|
||||||
|
@ -434,7 +424,7 @@ class AccountReaper(Daemon):
|
||||||
except ClientException as err:
|
except ClientException as err:
|
||||||
if self.logger.getEffectiveLevel() <= DEBUG:
|
if self.logger.getEffectiveLevel() <= DEBUG:
|
||||||
self.logger.exception(
|
self.logger.exception(
|
||||||
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
|
'Exception with %(ip)s:%(port)s/%(device)s', node)
|
||||||
failures += 1
|
failures += 1
|
||||||
self.logger.increment('containers_failures')
|
self.logger.increment('containers_failures')
|
||||||
self.stats_return_codes[err.http_status // 100] = \
|
self.stats_return_codes[err.http_status // 100] = \
|
||||||
|
@ -443,7 +433,7 @@ class AccountReaper(Daemon):
|
||||||
'return_codes.%d' % (err.http_status // 100,))
|
'return_codes.%d' % (err.http_status // 100,))
|
||||||
except (Timeout, socket.error) as err:
|
except (Timeout, socket.error) as err:
|
||||||
self.logger.error(
|
self.logger.error(
|
||||||
_('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
|
'Timeout Exception with %(ip)s:%(port)s/%(device)s',
|
||||||
node)
|
node)
|
||||||
failures += 1
|
failures += 1
|
||||||
self.logger.increment('containers_failures')
|
self.logger.increment('containers_failures')
|
||||||
|
@ -510,7 +500,7 @@ class AccountReaper(Daemon):
|
||||||
except ClientException as err:
|
except ClientException as err:
|
||||||
if self.logger.getEffectiveLevel() <= DEBUG:
|
if self.logger.getEffectiveLevel() <= DEBUG:
|
||||||
self.logger.exception(
|
self.logger.exception(
|
||||||
_('Exception with %(ip)s:%(port)s/%(device)s'), node)
|
'Exception with %(ip)s:%(port)s/%(device)s', node)
|
||||||
failures += 1
|
failures += 1
|
||||||
self.logger.increment('objects_failures')
|
self.logger.increment('objects_failures')
|
||||||
self.stats_return_codes[err.http_status // 100] = \
|
self.stats_return_codes[err.http_status // 100] = \
|
||||||
|
@ -521,7 +511,7 @@ class AccountReaper(Daemon):
|
||||||
failures += 1
|
failures += 1
|
||||||
self.logger.increment('objects_failures')
|
self.logger.increment('objects_failures')
|
||||||
self.logger.error(
|
self.logger.error(
|
||||||
_('Timeout Exception with %(ip)s:%(port)s/%(device)s'),
|
'Timeout Exception with %(ip)s:%(port)s/%(device)s',
|
||||||
node)
|
node)
|
||||||
if successes > failures:
|
if successes > failures:
|
||||||
self.stats_objects_deleted += 1
|
self.stats_objects_deleted += 1
|
||||||
|
|
|
@ -515,7 +515,12 @@ def main(args=None):
|
||||||
logger = get_logger({}, name='ContainerBroker', log_to_console=True)
|
logger = get_logger({}, name='ContainerBroker', log_to_console=True)
|
||||||
broker = ContainerBroker(args.container_db, logger=logger,
|
broker = ContainerBroker(args.container_db, logger=logger,
|
||||||
skip_commits=True)
|
skip_commits=True)
|
||||||
broker.get_info()
|
try:
|
||||||
|
broker.get_info()
|
||||||
|
except Exception as exc:
|
||||||
|
print('Error opening container DB %s: %s' % (args.container_db, exc),
|
||||||
|
file=sys.stderr)
|
||||||
|
return 2
|
||||||
print('Loaded db broker for %s.' % broker.path, file=sys.stderr)
|
print('Loaded db broker for %s.' % broker.path, file=sys.stderr)
|
||||||
return args.func(broker, args)
|
return args.func(broker, args)
|
||||||
|
|
||||||
|
|
|
@ -178,6 +178,14 @@ class BufferedHTTPConnection(HTTPConnection):
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
|
def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
|
||||||
|
'''Send a request to the server.
|
||||||
|
|
||||||
|
:param method: specifies an HTTP request method, e.g. 'GET'.
|
||||||
|
:param url: specifies the object being requested, e.g. '/index.html'.
|
||||||
|
:param skip_host: if True does not add automatically a 'Host:' header
|
||||||
|
:param skip_accept_encoding: if True does not add automatically an
|
||||||
|
'Accept-Encoding:' header
|
||||||
|
'''
|
||||||
self._method = method
|
self._method = method
|
||||||
self._path = url
|
self._path = url
|
||||||
return HTTPConnection.putrequest(self, method, url, skip_host,
|
return HTTPConnection.putrequest(self, method, url, skip_host,
|
||||||
|
|
|
@ -380,6 +380,10 @@ class Bulk(object):
|
||||||
query request.
|
query request.
|
||||||
"""
|
"""
|
||||||
last_yield = time()
|
last_yield = time()
|
||||||
|
if out_content_type and out_content_type.endswith('/xml'):
|
||||||
|
to_yield = '<?xml version="1.0" encoding="UTF-8"?>\n'
|
||||||
|
else:
|
||||||
|
to_yield = ' '
|
||||||
separator = ''
|
separator = ''
|
||||||
failed_files = []
|
failed_files = []
|
||||||
resp_dict = {'Response Status': HTTPOk().status,
|
resp_dict = {'Response Status': HTTPOk().status,
|
||||||
|
@ -390,8 +394,6 @@ class Bulk(object):
|
||||||
try:
|
try:
|
||||||
if not out_content_type:
|
if not out_content_type:
|
||||||
raise HTTPNotAcceptable(request=req)
|
raise HTTPNotAcceptable(request=req)
|
||||||
if out_content_type.endswith('/xml'):
|
|
||||||
yield '<?xml version="1.0" encoding="UTF-8"?>\n'
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
vrs, account, _junk = req.split_path(2, 3, True)
|
vrs, account, _junk = req.split_path(2, 3, True)
|
||||||
|
@ -452,9 +454,9 @@ class Bulk(object):
|
||||||
for resp, obj_name, retry in pile.asyncstarmap(
|
for resp, obj_name, retry in pile.asyncstarmap(
|
||||||
do_delete, names_to_delete):
|
do_delete, names_to_delete):
|
||||||
if last_yield + self.yield_frequency < time():
|
if last_yield + self.yield_frequency < time():
|
||||||
separator = '\r\n\r\n'
|
|
||||||
last_yield = time()
|
last_yield = time()
|
||||||
yield ' '
|
yield to_yield
|
||||||
|
to_yield, separator = ' ', '\r\n\r\n'
|
||||||
self._process_delete(resp, pile, obj_name,
|
self._process_delete(resp, pile, obj_name,
|
||||||
resp_dict, failed_files,
|
resp_dict, failed_files,
|
||||||
failed_file_response, retry)
|
failed_file_response, retry)
|
||||||
|
@ -462,9 +464,9 @@ class Bulk(object):
|
||||||
# Abort, but drain off the in-progress deletes
|
# Abort, but drain off the in-progress deletes
|
||||||
for resp, obj_name, retry in pile:
|
for resp, obj_name, retry in pile:
|
||||||
if last_yield + self.yield_frequency < time():
|
if last_yield + self.yield_frequency < time():
|
||||||
separator = '\r\n\r\n'
|
|
||||||
last_yield = time()
|
last_yield = time()
|
||||||
yield ' '
|
yield to_yield
|
||||||
|
to_yield, separator = ' ', '\r\n\r\n'
|
||||||
# Don't pass in the pile, as we shouldn't retry
|
# Don't pass in the pile, as we shouldn't retry
|
||||||
self._process_delete(
|
self._process_delete(
|
||||||
resp, None, obj_name, resp_dict,
|
resp, None, obj_name, resp_dict,
|
||||||
|
@ -508,14 +510,16 @@ class Bulk(object):
|
||||||
'Response Body': '', 'Number Files Created': 0}
|
'Response Body': '', 'Number Files Created': 0}
|
||||||
failed_files = []
|
failed_files = []
|
||||||
last_yield = time()
|
last_yield = time()
|
||||||
|
if out_content_type and out_content_type.endswith('/xml'):
|
||||||
|
to_yield = '<?xml version="1.0" encoding="UTF-8"?>\n'
|
||||||
|
else:
|
||||||
|
to_yield = ' '
|
||||||
separator = ''
|
separator = ''
|
||||||
containers_accessed = set()
|
containers_accessed = set()
|
||||||
req.environ['eventlet.minimum_write_chunk_size'] = 0
|
req.environ['eventlet.minimum_write_chunk_size'] = 0
|
||||||
try:
|
try:
|
||||||
if not out_content_type:
|
if not out_content_type:
|
||||||
raise HTTPNotAcceptable(request=req)
|
raise HTTPNotAcceptable(request=req)
|
||||||
if out_content_type.endswith('/xml'):
|
|
||||||
yield '<?xml version="1.0" encoding="UTF-8"?>\n'
|
|
||||||
|
|
||||||
if req.content_length is None and \
|
if req.content_length is None and \
|
||||||
req.headers.get('transfer-encoding',
|
req.headers.get('transfer-encoding',
|
||||||
|
@ -533,9 +537,9 @@ class Bulk(object):
|
||||||
containers_created = 0
|
containers_created = 0
|
||||||
while True:
|
while True:
|
||||||
if last_yield + self.yield_frequency < time():
|
if last_yield + self.yield_frequency < time():
|
||||||
separator = '\r\n\r\n'
|
|
||||||
last_yield = time()
|
last_yield = time()
|
||||||
yield ' '
|
yield to_yield
|
||||||
|
to_yield, separator = ' ', '\r\n\r\n'
|
||||||
tar_info = next(tar)
|
tar_info = next(tar)
|
||||||
if tar_info is None or \
|
if tar_info is None or \
|
||||||
len(failed_files) >= self.max_failed_extractions:
|
len(failed_files) >= self.max_failed_extractions:
|
||||||
|
|
|
@ -114,12 +114,11 @@ greater than 5GB.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from six.moves.urllib.parse import quote, unquote
|
|
||||||
|
|
||||||
from swift.common.utils import get_logger, config_true_value, FileLikeIter, \
|
from swift.common.utils import get_logger, config_true_value, FileLikeIter, \
|
||||||
close_if_possible
|
close_if_possible
|
||||||
from swift.common.swob import Request, HTTPPreconditionFailed, \
|
from swift.common.swob import Request, HTTPPreconditionFailed, \
|
||||||
HTTPRequestEntityTooLarge, HTTPBadRequest, HTTPException
|
HTTPRequestEntityTooLarge, HTTPBadRequest, HTTPException, \
|
||||||
|
wsgi_quote, wsgi_unquote
|
||||||
from swift.common.http import HTTP_MULTIPLE_CHOICES, is_success, HTTP_OK
|
from swift.common.http import HTTP_MULTIPLE_CHOICES, is_success, HTTP_OK
|
||||||
from swift.common.constraints import check_account_format, MAX_FILE_SIZE
|
from swift.common.constraints import check_account_format, MAX_FILE_SIZE
|
||||||
from swift.common.request_helpers import copy_header_subset, remove_items, \
|
from swift.common.request_helpers import copy_header_subset, remove_items, \
|
||||||
|
@ -183,7 +182,7 @@ class ServerSideCopyWebContext(WSGIContext):
|
||||||
|
|
||||||
def get_source_resp(self, req):
|
def get_source_resp(self, req):
|
||||||
sub_req = make_subrequest(
|
sub_req = make_subrequest(
|
||||||
req.environ, path=quote(req.path_info), headers=req.headers,
|
req.environ, path=wsgi_quote(req.path_info), headers=req.headers,
|
||||||
swift_source='SSC')
|
swift_source='SSC')
|
||||||
return sub_req.get_response(self.app)
|
return sub_req.get_response(self.app)
|
||||||
|
|
||||||
|
@ -257,9 +256,9 @@ class ServerSideCopyMiddleware(object):
|
||||||
)(req.environ, start_response)
|
)(req.environ, start_response)
|
||||||
dest_account = account
|
dest_account = account
|
||||||
if 'Destination-Account' in req.headers:
|
if 'Destination-Account' in req.headers:
|
||||||
dest_account = unquote(req.headers.get('Destination-Account'))
|
dest_account = wsgi_unquote(req.headers.get('Destination-Account'))
|
||||||
dest_account = check_account_format(req, dest_account)
|
dest_account = check_account_format(req, dest_account)
|
||||||
req.headers['X-Copy-From-Account'] = quote(account)
|
req.headers['X-Copy-From-Account'] = wsgi_quote(account)
|
||||||
account = dest_account
|
account = dest_account
|
||||||
del req.headers['Destination-Account']
|
del req.headers['Destination-Account']
|
||||||
dest_container, dest_object = _check_destination_header(req)
|
dest_container, dest_object = _check_destination_header(req)
|
||||||
|
@ -275,7 +274,7 @@ class ServerSideCopyMiddleware(object):
|
||||||
req.path_info = '/%s/%s/%s/%s' % (
|
req.path_info = '/%s/%s/%s/%s' % (
|
||||||
ver, dest_account, dest_container, dest_object)
|
ver, dest_account, dest_container, dest_object)
|
||||||
req.headers['Content-Length'] = 0
|
req.headers['Content-Length'] = 0
|
||||||
req.headers['X-Copy-From'] = quote(source)
|
req.headers['X-Copy-From'] = wsgi_quote(source)
|
||||||
del req.headers['Destination']
|
del req.headers['Destination']
|
||||||
return self.handle_PUT(req, start_response)
|
return self.handle_PUT(req, start_response)
|
||||||
|
|
||||||
|
@ -312,8 +311,8 @@ class ServerSideCopyMiddleware(object):
|
||||||
def _create_response_headers(self, source_path, source_resp, sink_req):
|
def _create_response_headers(self, source_path, source_resp, sink_req):
|
||||||
resp_headers = dict()
|
resp_headers = dict()
|
||||||
acct, path = source_path.split('/', 3)[2:4]
|
acct, path = source_path.split('/', 3)[2:4]
|
||||||
resp_headers['X-Copied-From-Account'] = quote(acct)
|
resp_headers['X-Copied-From-Account'] = wsgi_quote(acct)
|
||||||
resp_headers['X-Copied-From'] = quote(path)
|
resp_headers['X-Copied-From'] = wsgi_quote(path)
|
||||||
if 'last-modified' in source_resp.headers:
|
if 'last-modified' in source_resp.headers:
|
||||||
resp_headers['X-Copied-From-Last-Modified'] = \
|
resp_headers['X-Copied-From-Last-Modified'] = \
|
||||||
source_resp.headers['last-modified']
|
source_resp.headers['last-modified']
|
||||||
|
@ -334,7 +333,7 @@ class ServerSideCopyMiddleware(object):
|
||||||
src_account_name = req.headers.get('X-Copy-From-Account')
|
src_account_name = req.headers.get('X-Copy-From-Account')
|
||||||
if src_account_name:
|
if src_account_name:
|
||||||
src_account_name = check_account_format(
|
src_account_name = check_account_format(
|
||||||
req, unquote(src_account_name))
|
req, wsgi_unquote(src_account_name))
|
||||||
else:
|
else:
|
||||||
src_account_name = acct
|
src_account_name = acct
|
||||||
src_container_name, src_obj_name = _check_copy_from_header(req)
|
src_container_name, src_obj_name = _check_copy_from_header(req)
|
||||||
|
|
|
@ -18,7 +18,8 @@ import hmac
|
||||||
from swift.common.exceptions import UnknownSecretIdError
|
from swift.common.exceptions import UnknownSecretIdError
|
||||||
from swift.common.middleware.crypto.crypto_utils import CRYPTO_KEY_CALLBACK
|
from swift.common.middleware.crypto.crypto_utils import CRYPTO_KEY_CALLBACK
|
||||||
from swift.common.swob import Request, HTTPException, wsgi_to_bytes
|
from swift.common.swob import Request, HTTPException, wsgi_to_bytes
|
||||||
from swift.common.utils import readconf, strict_b64decode, get_logger
|
from swift.common.utils import readconf, strict_b64decode, get_logger, \
|
||||||
|
split_path
|
||||||
from swift.common.wsgi import WSGIContext
|
from swift.common.wsgi import WSGIContext
|
||||||
|
|
||||||
|
|
||||||
|
@ -77,29 +78,54 @@ class KeyMasterContext(WSGIContext):
|
||||||
version = key_id['v']
|
version = key_id['v']
|
||||||
if version not in ('1', '2'):
|
if version not in ('1', '2'):
|
||||||
raise ValueError('Unknown key_id version: %s' % version)
|
raise ValueError('Unknown key_id version: %s' % version)
|
||||||
|
if version == '1' and not key_id['path'].startswith(
|
||||||
|
'/' + self.account + '/'):
|
||||||
|
# Well shoot. This was the bug that made us notice we needed
|
||||||
|
# a v2! Hope the current account/container was the original!
|
||||||
|
key_acct, key_cont, key_obj = (
|
||||||
|
self.account, self.container, key_id['path'])
|
||||||
|
else:
|
||||||
|
key_acct, key_cont, key_obj = split_path(
|
||||||
|
key_id['path'], 1, 3, True)
|
||||||
|
|
||||||
|
check_path = (
|
||||||
|
self.account, self.container or key_cont, self.obj or key_obj)
|
||||||
|
if (key_acct, key_cont, key_obj) != check_path:
|
||||||
|
self.keymaster.logger.info(
|
||||||
|
"Path stored in meta (%r) does not match path from "
|
||||||
|
"request (%r)! Using path from meta.",
|
||||||
|
key_id['path'],
|
||||||
|
'/' + '/'.join(x for x in [
|
||||||
|
self.account, self.container, self.obj] if x))
|
||||||
else:
|
else:
|
||||||
secret_id = self.keymaster.active_secret_id
|
secret_id = self.keymaster.active_secret_id
|
||||||
# v1 had a bug where we would claim the path was just the object
|
# v1 had a bug where we would claim the path was just the object
|
||||||
# name if the object started with a slash. Bump versions to
|
# name if the object started with a slash. Bump versions to
|
||||||
# establish that we can trust the path.
|
# establish that we can trust the path.
|
||||||
version = '2'
|
version = '2'
|
||||||
|
key_acct, key_cont, key_obj = (
|
||||||
|
self.account, self.container, self.obj)
|
||||||
|
|
||||||
if (secret_id, version) in self._keys:
|
if (secret_id, version) in self._keys:
|
||||||
return self._keys[(secret_id, version)]
|
return self._keys[(secret_id, version)]
|
||||||
|
|
||||||
keys = {}
|
keys = {}
|
||||||
account_path = '/' + self.account
|
account_path = '/' + key_acct
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
# self.account/container/obj reflect the level of the *request*,
|
||||||
|
# which may be different from the level of the key_id-path. Only
|
||||||
|
# fetch the keys that the request needs.
|
||||||
if self.container:
|
if self.container:
|
||||||
path = account_path + '/' + self.container
|
path = account_path + '/' + key_cont
|
||||||
keys['container'] = self.keymaster.create_key(
|
keys['container'] = self.keymaster.create_key(
|
||||||
path, secret_id=secret_id)
|
path, secret_id=secret_id)
|
||||||
|
|
||||||
if self.obj:
|
if self.obj:
|
||||||
if self.obj.startswith('/') and version == '1':
|
if key_obj.startswith('/') and version == '1':
|
||||||
path = self.obj
|
path = key_obj
|
||||||
else:
|
else:
|
||||||
path = path + '/' + self.obj
|
path = path + '/' + key_obj
|
||||||
keys['object'] = self.keymaster.create_key(
|
keys['object'] = self.keymaster.create_key(
|
||||||
path, secret_id=secret_id)
|
path, secret_id=secret_id)
|
||||||
|
|
||||||
|
|
|
@ -144,7 +144,8 @@ class GetContext(WSGIContext):
|
||||||
def _get_container_listing(self, req, version, account, container,
|
def _get_container_listing(self, req, version, account, container,
|
||||||
prefix, marker=''):
|
prefix, marker=''):
|
||||||
con_req = make_subrequest(
|
con_req = make_subrequest(
|
||||||
req.environ, path='/'.join(['', version, account, container]),
|
req.environ,
|
||||||
|
path=quote('/'.join(['', version, account, container])),
|
||||||
method='GET',
|
method='GET',
|
||||||
headers={'x-auth-token': req.headers.get('x-auth-token')},
|
headers={'x-auth-token': req.headers.get('x-auth-token')},
|
||||||
agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
|
agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
|
||||||
|
|
|
@ -126,7 +126,9 @@ import hmac
|
||||||
from hashlib import sha1
|
from hashlib import sha1
|
||||||
from time import time
|
from time import time
|
||||||
|
|
||||||
|
import six
|
||||||
from six.moves.urllib.parse import quote
|
from six.moves.urllib.parse import quote
|
||||||
|
|
||||||
from swift.common.exceptions import MimeInvalid
|
from swift.common.exceptions import MimeInvalid
|
||||||
from swift.common.middleware.tempurl import get_tempurl_keys_from_metadata
|
from swift.common.middleware.tempurl import get_tempurl_keys_from_metadata
|
||||||
from swift.common.utils import streq_const_time, register_swift_info, \
|
from swift.common.utils import streq_const_time, register_swift_info, \
|
||||||
|
@ -229,7 +231,7 @@ class FormPost(object):
|
||||||
start_response(status, headers)
|
start_response(status, headers)
|
||||||
return [body]
|
return [body]
|
||||||
except MimeInvalid:
|
except MimeInvalid:
|
||||||
body = 'FormPost: invalid starting boundary'
|
body = b'FormPost: invalid starting boundary'
|
||||||
start_response(
|
start_response(
|
||||||
'400 Bad Request',
|
'400 Bad Request',
|
||||||
(('Content-Type', 'text/plain'),
|
(('Content-Type', 'text/plain'),
|
||||||
|
@ -237,6 +239,8 @@ class FormPost(object):
|
||||||
return [body]
|
return [body]
|
||||||
except (FormInvalid, EOFError) as err:
|
except (FormInvalid, EOFError) as err:
|
||||||
body = 'FormPost: %s' % err
|
body = 'FormPost: %s' % err
|
||||||
|
if six.PY3:
|
||||||
|
body = body.encode('utf-8')
|
||||||
start_response(
|
start_response(
|
||||||
'400 Bad Request',
|
'400 Bad Request',
|
||||||
(('Content-Type', 'text/plain'),
|
(('Content-Type', 'text/plain'),
|
||||||
|
@ -258,6 +262,8 @@ class FormPost(object):
|
||||||
:returns: status_line, headers_list, body
|
:returns: status_line, headers_list, body
|
||||||
"""
|
"""
|
||||||
keys = self._get_keys(env)
|
keys = self._get_keys(env)
|
||||||
|
if six.PY3:
|
||||||
|
boundary = boundary.encode('utf-8')
|
||||||
status = message = ''
|
status = message = ''
|
||||||
attributes = {}
|
attributes = {}
|
||||||
subheaders = []
|
subheaders = []
|
||||||
|
@ -282,14 +288,13 @@ class FormPost(object):
|
||||||
hdrs['Content-Type'] or 'application/octet-stream'
|
hdrs['Content-Type'] or 'application/octet-stream'
|
||||||
if 'content-encoding' not in attributes and \
|
if 'content-encoding' not in attributes and \
|
||||||
'content-encoding' in hdrs:
|
'content-encoding' in hdrs:
|
||||||
attributes['content-encoding'] = \
|
attributes['content-encoding'] = hdrs['Content-Encoding']
|
||||||
hdrs['Content-Encoding']
|
|
||||||
status, subheaders = \
|
status, subheaders = \
|
||||||
self._perform_subrequest(env, attributes, fp, keys)
|
self._perform_subrequest(env, attributes, fp, keys)
|
||||||
if not status.startswith('2'):
|
if not status.startswith('2'):
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
data = ''
|
data = b''
|
||||||
mxln = MAX_VALUE_LENGTH
|
mxln = MAX_VALUE_LENGTH
|
||||||
while mxln:
|
while mxln:
|
||||||
chunk = fp.read(mxln)
|
chunk = fp.read(mxln)
|
||||||
|
@ -299,6 +304,8 @@ class FormPost(object):
|
||||||
data += chunk
|
data += chunk
|
||||||
while fp.read(READ_CHUNK_SIZE):
|
while fp.read(READ_CHUNK_SIZE):
|
||||||
pass
|
pass
|
||||||
|
if six.PY3:
|
||||||
|
data = data.decode('utf-8')
|
||||||
if 'name' in attrs:
|
if 'name' in attrs:
|
||||||
attributes[attrs['name'].lower()] = data.rstrip('\r\n--')
|
attributes[attrs['name'].lower()] = data.rstrip('\r\n--')
|
||||||
if not status:
|
if not status:
|
||||||
|
@ -315,6 +322,8 @@ class FormPost(object):
|
||||||
body = status + '\r\nFormPost: ' + message.title()
|
body = status + '\r\nFormPost: ' + message.title()
|
||||||
headers.extend([('Content-Type', 'text/plain'),
|
headers.extend([('Content-Type', 'text/plain'),
|
||||||
('Content-Length', len(body))])
|
('Content-Length', len(body))])
|
||||||
|
if six.PY3:
|
||||||
|
body = body.encode('utf-8')
|
||||||
return status, headers, body
|
return status, headers, body
|
||||||
status = status.split(' ', 1)[0]
|
status = status.split(' ', 1)[0]
|
||||||
if '?' in redirect:
|
if '?' in redirect:
|
||||||
|
@ -324,6 +333,8 @@ class FormPost(object):
|
||||||
redirect += 'status=%s&message=%s' % (quote(status), quote(message))
|
redirect += 'status=%s&message=%s' % (quote(status), quote(message))
|
||||||
body = '<html><body><p><a href="%s">' \
|
body = '<html><body><p><a href="%s">' \
|
||||||
'Click to continue...</a></p></body></html>' % redirect
|
'Click to continue...</a></p></body></html>' % redirect
|
||||||
|
if six.PY3:
|
||||||
|
body = body.encode('utf-8')
|
||||||
headers.extend(
|
headers.extend(
|
||||||
[('Location', redirect), ('Content-Length', str(len(body)))])
|
[('Location', redirect), ('Content-Length', str(len(body)))])
|
||||||
return '303 See Other', headers, body
|
return '303 See Other', headers, body
|
||||||
|
@ -385,6 +396,8 @@ class FormPost(object):
|
||||||
attributes.get('max_file_size') or '0',
|
attributes.get('max_file_size') or '0',
|
||||||
attributes.get('max_file_count') or '0',
|
attributes.get('max_file_count') or '0',
|
||||||
attributes.get('expires') or '0')
|
attributes.get('expires') or '0')
|
||||||
|
if six.PY3:
|
||||||
|
hmac_body = hmac_body.encode('utf-8')
|
||||||
|
|
||||||
has_valid_sig = False
|
has_valid_sig = False
|
||||||
for key in keys:
|
for key in keys:
|
||||||
|
|
|
@ -13,7 +13,7 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
from urllib import quote
|
from six.moves.urllib.parse import quote
|
||||||
from swift.common.utils import public
|
from swift.common.utils import public
|
||||||
|
|
||||||
from swift.common.middleware.s3api.controllers.base import Controller
|
from swift.common.middleware.s3api.controllers.base import Controller
|
||||||
|
|
|
@ -24,7 +24,8 @@ import six
|
||||||
from six.moves.urllib.parse import quote, unquote, parse_qsl
|
from six.moves.urllib.parse import quote, unquote, parse_qsl
|
||||||
import string
|
import string
|
||||||
|
|
||||||
from swift.common.utils import split_path, json, get_swift_info
|
from swift.common.utils import split_path, json, get_swift_info, \
|
||||||
|
close_if_possible
|
||||||
from swift.common import swob
|
from swift.common import swob
|
||||||
from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \
|
from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \
|
||||||
HTTP_NO_CONTENT, HTTP_UNAUTHORIZED, HTTP_FORBIDDEN, HTTP_NOT_FOUND, \
|
HTTP_NO_CONTENT, HTTP_UNAUTHORIZED, HTTP_FORBIDDEN, HTTP_NOT_FOUND, \
|
||||||
|
@ -110,6 +111,34 @@ def _header_acl_property(resource):
|
||||||
doc='Get and set the %s acl property' % resource)
|
doc='Get and set the %s acl property' % resource)
|
||||||
|
|
||||||
|
|
||||||
|
class HashingInput(object):
|
||||||
|
"""
|
||||||
|
wsgi.input wrapper to verify the hash of the input as it's read.
|
||||||
|
"""
|
||||||
|
def __init__(self, reader, content_length, hasher, expected_hex_hash):
|
||||||
|
self._input = reader
|
||||||
|
self._to_read = content_length
|
||||||
|
self._hasher = hasher()
|
||||||
|
self._expected = expected_hex_hash
|
||||||
|
|
||||||
|
def read(self, size=None):
|
||||||
|
chunk = self._input.read(size)
|
||||||
|
self._hasher.update(chunk)
|
||||||
|
self._to_read -= len(chunk)
|
||||||
|
if self._to_read < 0 or (size > len(chunk) and self._to_read) or (
|
||||||
|
self._to_read == 0 and
|
||||||
|
self._hasher.hexdigest() != self._expected):
|
||||||
|
self.close()
|
||||||
|
# Since we don't return the last chunk, the PUT never completes
|
||||||
|
raise swob.HTTPUnprocessableEntity(
|
||||||
|
'The X-Amz-Content-SHA56 you specified did not match '
|
||||||
|
'what we received.')
|
||||||
|
return chunk
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
close_if_possible(self._input)
|
||||||
|
|
||||||
|
|
||||||
class SigV4Mixin(object):
|
class SigV4Mixin(object):
|
||||||
"""
|
"""
|
||||||
A request class mixin to provide S3 signature v4 functionality
|
A request class mixin to provide S3 signature v4 functionality
|
||||||
|
@ -401,6 +430,20 @@ class SigV4Mixin(object):
|
||||||
raise InvalidRequest(msg)
|
raise InvalidRequest(msg)
|
||||||
else:
|
else:
|
||||||
hashed_payload = self.headers['X-Amz-Content-SHA256']
|
hashed_payload = self.headers['X-Amz-Content-SHA256']
|
||||||
|
if self.content_length == 0:
|
||||||
|
if hashed_payload != sha256().hexdigest():
|
||||||
|
raise BadDigest(
|
||||||
|
'The X-Amz-Content-SHA56 you specified did not match '
|
||||||
|
'what we received.')
|
||||||
|
elif self.content_length:
|
||||||
|
self.environ['wsgi.input'] = HashingInput(
|
||||||
|
self.environ['wsgi.input'],
|
||||||
|
self.content_length,
|
||||||
|
sha256,
|
||||||
|
hashed_payload)
|
||||||
|
# else, not provided -- Swift will kick out a 411 Length Required
|
||||||
|
# which will get translated back to a S3-style response in
|
||||||
|
# S3Request._swift_error_codes
|
||||||
cr.append(hashed_payload)
|
cr.append(hashed_payload)
|
||||||
return '\n'.join(cr).encode('utf-8')
|
return '\n'.join(cr).encode('utf-8')
|
||||||
|
|
||||||
|
@ -1267,12 +1310,15 @@ class S3Request(swob.Request):
|
||||||
sw_req = self.to_swift_req(method, container, obj, headers=headers,
|
sw_req = self.to_swift_req(method, container, obj, headers=headers,
|
||||||
body=body, query=query)
|
body=body, query=query)
|
||||||
|
|
||||||
sw_resp = sw_req.get_response(app)
|
try:
|
||||||
|
sw_resp = sw_req.get_response(app)
|
||||||
# reuse account and tokens
|
except swob.HTTPException as err:
|
||||||
_, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
|
sw_resp = err
|
||||||
2, 3, True)
|
else:
|
||||||
self.account = utf8encode(self.account)
|
# reuse account and tokens
|
||||||
|
_, self.account, _ = split_path(sw_resp.environ['PATH_INFO'],
|
||||||
|
2, 3, True)
|
||||||
|
self.account = utf8encode(self.account)
|
||||||
|
|
||||||
resp = S3Response.from_swift_resp(sw_resp)
|
resp = S3Response.from_swift_resp(sw_resp)
|
||||||
status = resp.status_int # pylint: disable-msg=E1101
|
status = resp.status_int # pylint: disable-msg=E1101
|
||||||
|
|
|
@ -33,6 +33,25 @@ This middleware:
|
||||||
* Optionally can retrieve and cache secret from keystone
|
* Optionally can retrieve and cache secret from keystone
|
||||||
to validate signature locally
|
to validate signature locally
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
If upgrading from swift3, the ``auth_version`` config option has been
|
||||||
|
removed, and the ``auth_uri`` option now includes the Keystone API
|
||||||
|
version. If you previously had a configuration like
|
||||||
|
|
||||||
|
.. code-block:: ini
|
||||||
|
|
||||||
|
[filter:s3token]
|
||||||
|
use = egg:swift3#s3token
|
||||||
|
auth_uri = https://keystonehost:35357
|
||||||
|
auth_version = 3
|
||||||
|
|
||||||
|
you should now use
|
||||||
|
|
||||||
|
.. code-block:: ini
|
||||||
|
|
||||||
|
[filter:s3token]
|
||||||
|
use = egg:swift#s3token
|
||||||
|
auth_uri = https://keystonehost:35357/v3
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
|
|
|
@ -457,8 +457,8 @@ def parse_and_validate_input(req_body, req_path):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
obj_path = '/'.join(['', vrs, account,
|
obj_path = '/'.join(['', vrs, account,
|
||||||
seg_dict['path'].lstrip('/')])
|
quote(seg_dict['path'].lstrip('/'))])
|
||||||
if req_path == quote(obj_path):
|
if req_path == obj_path:
|
||||||
errors.append(
|
errors.append(
|
||||||
b"Index %d: manifest must not include itself as a segment"
|
b"Index %d: manifest must not include itself as a segment"
|
||||||
% seg_index)
|
% seg_index)
|
||||||
|
@ -526,7 +526,7 @@ class SloGetContext(WSGIContext):
|
||||||
Raise exception on failures.
|
Raise exception on failures.
|
||||||
"""
|
"""
|
||||||
sub_req = make_subrequest(
|
sub_req = make_subrequest(
|
||||||
req.environ, path='/'.join(['', version, acc, con, obj]),
|
req.environ, path=quote('/'.join(['', version, acc, con, obj])),
|
||||||
method='GET',
|
method='GET',
|
||||||
headers={'x-auth-token': req.headers.get('x-auth-token')},
|
headers={'x-auth-token': req.headers.get('x-auth-token')},
|
||||||
agent='%(orig)s SLO MultipartGET', swift_source='SLO')
|
agent='%(orig)s SLO MultipartGET', swift_source='SLO')
|
||||||
|
@ -1109,8 +1109,8 @@ class StaticLargeObject(object):
|
||||||
path2indices[seg_dict['path']].append(index)
|
path2indices[seg_dict['path']].append(index)
|
||||||
|
|
||||||
def do_head(obj_name):
|
def do_head(obj_name):
|
||||||
obj_path = '/'.join(['', vrs, account,
|
obj_path = quote('/'.join([
|
||||||
get_valid_utf8_str(obj_name).lstrip('/')])
|
'', vrs, account, get_valid_utf8_str(obj_name).lstrip('/')]))
|
||||||
|
|
||||||
sub_req = make_subrequest(
|
sub_req = make_subrequest(
|
||||||
req.environ, path=obj_path + '?', # kill the query string
|
req.environ, path=obj_path + '?', # kill the query string
|
||||||
|
|
|
@ -184,7 +184,7 @@ from eventlet import Timeout
|
||||||
import six
|
import six
|
||||||
from swift.common.swob import Response, Request, wsgi_to_str
|
from swift.common.swob import Response, Request, wsgi_to_str
|
||||||
from swift.common.swob import HTTPBadRequest, HTTPForbidden, HTTPNotFound, \
|
from swift.common.swob import HTTPBadRequest, HTTPForbidden, HTTPNotFound, \
|
||||||
HTTPUnauthorized
|
HTTPUnauthorized, HTTPMethodNotAllowed
|
||||||
|
|
||||||
from swift.common.request_helpers import get_sys_meta_prefix
|
from swift.common.request_helpers import get_sys_meta_prefix
|
||||||
from swift.common.middleware.acl import (
|
from swift.common.middleware.acl import (
|
||||||
|
@ -688,6 +688,9 @@ class TempAuth(object):
|
||||||
"""
|
"""
|
||||||
req.start_time = time()
|
req.start_time = time()
|
||||||
handler = None
|
handler = None
|
||||||
|
if req.method != 'GET':
|
||||||
|
req.response = HTTPMethodNotAllowed(request=req)
|
||||||
|
return req.response
|
||||||
try:
|
try:
|
||||||
version, account, user, _junk = split_path(req.path_info,
|
version, account, user, _junk = split_path(req.path_info,
|
||||||
1, 4, True)
|
1, 4, True)
|
||||||
|
|
|
@ -26,7 +26,6 @@ import sys
|
||||||
import time
|
import time
|
||||||
|
|
||||||
import six
|
import six
|
||||||
from six.moves.urllib.parse import unquote
|
|
||||||
from swift.common.header_key_dict import HeaderKeyDict
|
from swift.common.header_key_dict import HeaderKeyDict
|
||||||
|
|
||||||
from swift import gettext_ as _
|
from swift import gettext_ as _
|
||||||
|
@ -35,11 +34,11 @@ from swift.common.exceptions import ListingIterError, SegmentError
|
||||||
from swift.common.http import is_success
|
from swift.common.http import is_success
|
||||||
from swift.common.swob import HTTPBadRequest, \
|
from swift.common.swob import HTTPBadRequest, \
|
||||||
HTTPServiceUnavailable, Range, is_chunked, multi_range_iterator, \
|
HTTPServiceUnavailable, Range, is_chunked, multi_range_iterator, \
|
||||||
HTTPPreconditionFailed, wsgi_to_bytes
|
HTTPPreconditionFailed, wsgi_to_bytes, wsgi_unquote, wsgi_to_str
|
||||||
from swift.common.utils import split_path, validate_device_partition, \
|
from swift.common.utils import split_path, validate_device_partition, \
|
||||||
close_if_possible, maybe_multipart_byteranges_to_document_iters, \
|
close_if_possible, maybe_multipart_byteranges_to_document_iters, \
|
||||||
multipart_byteranges_to_document_iters, parse_content_type, \
|
multipart_byteranges_to_document_iters, parse_content_type, \
|
||||||
parse_content_range, csv_append, list_from_csv, Spliterator
|
parse_content_range, csv_append, list_from_csv, Spliterator, quote
|
||||||
|
|
||||||
from swift.common.wsgi import make_subrequest
|
from swift.common.wsgi import make_subrequest
|
||||||
|
|
||||||
|
@ -115,14 +114,13 @@ def split_and_validate_path(request, minsegs=1, maxsegs=None,
|
||||||
Utility function to split and validate the request path.
|
Utility function to split and validate the request path.
|
||||||
|
|
||||||
:returns: result of :meth:`~swift.common.utils.split_path` if
|
:returns: result of :meth:`~swift.common.utils.split_path` if
|
||||||
everything's okay
|
everything's okay, as native strings
|
||||||
:raises HTTPBadRequest: if something's not okay
|
:raises HTTPBadRequest: if something's not okay
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
segs = split_path(unquote(request.path),
|
segs = request.split_path(minsegs, maxsegs, rest_with_last)
|
||||||
minsegs, maxsegs, rest_with_last)
|
|
||||||
validate_device_partition(segs[0], segs[1])
|
validate_device_partition(segs[0], segs[1])
|
||||||
return segs
|
return [wsgi_to_str(seg) for seg in segs]
|
||||||
except ValueError as err:
|
except ValueError as err:
|
||||||
raise HTTPBadRequest(body=str(err), request=request,
|
raise HTTPBadRequest(body=str(err), request=request,
|
||||||
content_type='text/plain')
|
content_type='text/plain')
|
||||||
|
@ -308,7 +306,7 @@ def check_path_header(req, name, length, error_msg):
|
||||||
:raise: HTTPPreconditionFailed if header value
|
:raise: HTTPPreconditionFailed if header value
|
||||||
is not well formatted.
|
is not well formatted.
|
||||||
"""
|
"""
|
||||||
hdr = unquote(req.headers.get(name))
|
hdr = wsgi_unquote(req.headers.get(name))
|
||||||
if not hdr.startswith('/'):
|
if not hdr.startswith('/'):
|
||||||
hdr = '/' + hdr
|
hdr = '/' + hdr
|
||||||
try:
|
try:
|
||||||
|
@ -391,7 +389,7 @@ class SegmentedIterable(object):
|
||||||
# segment is a plain old object, not some flavor of large
|
# segment is a plain old object, not some flavor of large
|
||||||
# object; therefore, its etag is its MD5sum and hence we can
|
# object; therefore, its etag is its MD5sum and hence we can
|
||||||
# check it.
|
# check it.
|
||||||
path = seg_path + '?multipart-manifest=get'
|
path = quote(seg_path) + '?multipart-manifest=get'
|
||||||
seg_req = make_subrequest(
|
seg_req = make_subrequest(
|
||||||
self.req.environ, path=path, method='GET',
|
self.req.environ, path=path, method='GET',
|
||||||
headers={'x-auth-token': self.req.headers.get(
|
headers={'x-auth-token': self.req.headers.get(
|
||||||
|
|
|
@ -309,6 +309,50 @@ def str_to_wsgi(native_str):
|
||||||
return bytes_to_wsgi(native_str.encode('utf8', errors='surrogateescape'))
|
return bytes_to_wsgi(native_str.encode('utf8', errors='surrogateescape'))
|
||||||
|
|
||||||
|
|
||||||
|
def wsgi_quote(wsgi_str):
|
||||||
|
if six.PY2:
|
||||||
|
if not isinstance(wsgi_str, bytes):
|
||||||
|
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
|
||||||
|
return urllib.parse.quote(wsgi_str)
|
||||||
|
|
||||||
|
if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
|
||||||
|
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
|
||||||
|
return urllib.parse.quote(wsgi_str, encoding='latin-1')
|
||||||
|
|
||||||
|
|
||||||
|
def wsgi_unquote(wsgi_str):
|
||||||
|
if six.PY2:
|
||||||
|
if not isinstance(wsgi_str, bytes):
|
||||||
|
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
|
||||||
|
return urllib.parse.unquote(wsgi_str)
|
||||||
|
|
||||||
|
if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
|
||||||
|
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
|
||||||
|
return urllib.parse.unquote(wsgi_str, encoding='latin-1')
|
||||||
|
|
||||||
|
|
||||||
|
def wsgi_quote_plus(wsgi_str):
|
||||||
|
if six.PY2:
|
||||||
|
if not isinstance(wsgi_str, bytes):
|
||||||
|
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
|
||||||
|
return urllib.parse.quote_plus(wsgi_str)
|
||||||
|
|
||||||
|
if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
|
||||||
|
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
|
||||||
|
return urllib.parse.quote_plus(wsgi_str, encoding='latin-1')
|
||||||
|
|
||||||
|
|
||||||
|
def wsgi_unquote_plus(wsgi_str):
|
||||||
|
if six.PY2:
|
||||||
|
if not isinstance(wsgi_str, bytes):
|
||||||
|
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
|
||||||
|
return urllib.parse.unquote_plus(wsgi_str)
|
||||||
|
|
||||||
|
if not isinstance(wsgi_str, str) or any(ord(x) > 255 for x in wsgi_str):
|
||||||
|
raise TypeError('Expected a WSGI string; got %r' % wsgi_str)
|
||||||
|
return urllib.parse.unquote_plus(wsgi_str, encoding='latin-1')
|
||||||
|
|
||||||
|
|
||||||
def _resp_status_property():
|
def _resp_status_property():
|
||||||
"""
|
"""
|
||||||
Set and retrieve the value of Response.status
|
Set and retrieve the value of Response.status
|
||||||
|
|
|
@ -4273,8 +4273,7 @@ def iter_multipart_mime_documents(wsgi_input, boundary, read_chunk_size=4096):
|
||||||
for doing that if necessary.
|
for doing that if necessary.
|
||||||
|
|
||||||
:param wsgi_input: The file-like object to read from.
|
:param wsgi_input: The file-like object to read from.
|
||||||
:param boundary: The mime boundary to separate new file-like
|
:param boundary: The mime boundary to separate new file-like objects on.
|
||||||
objects on.
|
|
||||||
:returns: A generator of file-like objects for each part.
|
:returns: A generator of file-like objects for each part.
|
||||||
:raises MimeInvalid: if the document is malformed
|
:raises MimeInvalid: if the document is malformed
|
||||||
"""
|
"""
|
||||||
|
|
|
@ -32,13 +32,10 @@ from eventlet.green import socket, ssl, os as green_os
|
||||||
import six
|
import six
|
||||||
from six import BytesIO
|
from six import BytesIO
|
||||||
from six import StringIO
|
from six import StringIO
|
||||||
from six.moves.urllib.parse import unquote
|
|
||||||
if six.PY2:
|
|
||||||
import mimetools
|
|
||||||
|
|
||||||
from swift.common import utils, constraints
|
from swift.common import utils, constraints
|
||||||
from swift.common.storage_policy import BindPortsCache
|
from swift.common.storage_policy import BindPortsCache
|
||||||
from swift.common.swob import Request
|
from swift.common.swob import Request, wsgi_unquote
|
||||||
from swift.common.utils import capture_stdio, disable_fallocate, \
|
from swift.common.utils import capture_stdio, disable_fallocate, \
|
||||||
drop_privileges, get_logger, NullLogger, config_true_value, \
|
drop_privileges, get_logger, NullLogger, config_true_value, \
|
||||||
validate_configuration, get_hub, config_auto_int_value, \
|
validate_configuration, get_hub, config_auto_int_value, \
|
||||||
|
@ -148,31 +145,6 @@ def wrap_conf_type(f):
|
||||||
appconfig = wrap_conf_type(loadwsgi.appconfig)
|
appconfig = wrap_conf_type(loadwsgi.appconfig)
|
||||||
|
|
||||||
|
|
||||||
def monkey_patch_mimetools():
|
|
||||||
"""
|
|
||||||
mimetools.Message defaults content-type to "text/plain"
|
|
||||||
This changes it to default to None, so we can detect missing headers.
|
|
||||||
"""
|
|
||||||
if six.PY3:
|
|
||||||
# The mimetools has been removed from Python 3
|
|
||||||
return
|
|
||||||
|
|
||||||
orig_parsetype = mimetools.Message.parsetype
|
|
||||||
|
|
||||||
def parsetype(self):
|
|
||||||
if not self.typeheader:
|
|
||||||
self.type = None
|
|
||||||
self.maintype = None
|
|
||||||
self.subtype = None
|
|
||||||
self.plisttext = ''
|
|
||||||
else:
|
|
||||||
orig_parsetype(self)
|
|
||||||
parsetype.patched = True
|
|
||||||
|
|
||||||
if not getattr(mimetools.Message.parsetype, 'patched', None):
|
|
||||||
mimetools.Message.parsetype = parsetype
|
|
||||||
|
|
||||||
|
|
||||||
def get_socket(conf):
|
def get_socket(conf):
|
||||||
"""Bind socket to bind ip:port in conf
|
"""Bind socket to bind ip:port in conf
|
||||||
|
|
||||||
|
@ -448,6 +420,18 @@ class SwiftHttpProtocol(wsgi.HttpProtocol):
|
||||||
# versions the output from error is same as info anyway
|
# versions the output from error is same as info anyway
|
||||||
self.server.log.info('ERROR WSGI: ' + f, *a)
|
self.server.log.info('ERROR WSGI: ' + f, *a)
|
||||||
|
|
||||||
|
class MessageClass(wsgi.HttpProtocol.MessageClass):
|
||||||
|
'''Subclass to see when the client didn't provide a Content-Type'''
|
||||||
|
# for py2:
|
||||||
|
def parsetype(self):
|
||||||
|
if self.typeheader is None:
|
||||||
|
self.typeheader = ''
|
||||||
|
wsgi.HttpProtocol.MessageClass.parsetype(self)
|
||||||
|
|
||||||
|
# for py3:
|
||||||
|
def get_default_type(self):
|
||||||
|
return ''
|
||||||
|
|
||||||
|
|
||||||
class SwiftHttpProxiedProtocol(SwiftHttpProtocol):
|
class SwiftHttpProxiedProtocol(SwiftHttpProtocol):
|
||||||
"""
|
"""
|
||||||
|
@ -1156,7 +1140,6 @@ def _initrp(conf_path, app_section, *args, **kwargs):
|
||||||
if config_true_value(conf.get('disable_fallocate', 'no')):
|
if config_true_value(conf.get('disable_fallocate', 'no')):
|
||||||
disable_fallocate()
|
disable_fallocate()
|
||||||
|
|
||||||
monkey_patch_mimetools()
|
|
||||||
return (conf, logger, log_name)
|
return (conf, logger, log_name)
|
||||||
|
|
||||||
|
|
||||||
|
@ -1319,7 +1302,7 @@ def make_subrequest(env, method=None, path=None, body=None, headers=None,
|
||||||
path = path or ''
|
path = path or ''
|
||||||
if path and '?' in path:
|
if path and '?' in path:
|
||||||
path, query_string = path.split('?', 1)
|
path, query_string = path.split('?', 1)
|
||||||
newenv = make_env(env, method, path=unquote(path), agent=agent,
|
newenv = make_env(env, method, path=wsgi_unquote(path), agent=agent,
|
||||||
query_string=query_string, swift_source=swift_source)
|
query_string=query_string, swift_source=swift_source)
|
||||||
if not headers:
|
if not headers:
|
||||||
headers = {}
|
headers = {}
|
||||||
|
|
|
@ -338,7 +338,7 @@ class ContainerBroker(DatabaseBroker):
|
||||||
self._db_files = None
|
self._db_files = None
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def create_broker(self, device_path, part, account, container, logger=None,
|
def create_broker(cls, device_path, part, account, container, logger=None,
|
||||||
epoch=None, put_timestamp=None,
|
epoch=None, put_timestamp=None,
|
||||||
storage_policy_index=None):
|
storage_policy_index=None):
|
||||||
"""
|
"""
|
||||||
|
|
|
@ -131,14 +131,6 @@ def get_tmp_dir(policy_or_index):
|
||||||
return get_policy_string(TMP_BASE, policy_or_index)
|
return get_policy_string(TMP_BASE, policy_or_index)
|
||||||
|
|
||||||
|
|
||||||
def _unlink_if_present(filename):
|
|
||||||
try:
|
|
||||||
os.unlink(filename)
|
|
||||||
except OSError as err:
|
|
||||||
if err.errno != errno.ENOENT:
|
|
||||||
raise
|
|
||||||
|
|
||||||
|
|
||||||
def _get_filename(fd):
|
def _get_filename(fd):
|
||||||
"""
|
"""
|
||||||
Helper function to get to file name from a file descriptor or filename.
|
Helper function to get to file name from a file descriptor or filename.
|
||||||
|
@ -1623,12 +1615,10 @@ class BaseDiskFileManager(object):
|
||||||
partition_path = get_part_path(dev_path, policy, partition)
|
partition_path = get_part_path(dev_path, policy, partition)
|
||||||
if suffixes is None:
|
if suffixes is None:
|
||||||
suffixes = self.yield_suffixes(device, partition, policy)
|
suffixes = self.yield_suffixes(device, partition, policy)
|
||||||
considering_all_suffixes = True
|
|
||||||
else:
|
else:
|
||||||
suffixes = (
|
suffixes = (
|
||||||
(os.path.join(partition_path, suffix), suffix)
|
(os.path.join(partition_path, suffix), suffix)
|
||||||
for suffix in suffixes)
|
for suffix in suffixes)
|
||||||
considering_all_suffixes = False
|
|
||||||
|
|
||||||
key_preference = (
|
key_preference = (
|
||||||
('ts_meta', 'meta_info', 'timestamp'),
|
('ts_meta', 'meta_info', 'timestamp'),
|
||||||
|
@ -1637,19 +1627,18 @@ class BaseDiskFileManager(object):
|
||||||
('ts_ctype', 'ctype_info', 'ctype_timestamp'),
|
('ts_ctype', 'ctype_info', 'ctype_timestamp'),
|
||||||
)
|
)
|
||||||
|
|
||||||
# We delete as many empty directories as we can.
|
# cleanup_ondisk_files() will remove empty hash dirs, and we'll
|
||||||
# cleanup_ondisk_files() takes care of the hash dirs, and we take
|
# invalidate any empty suffix dirs so they'll get cleaned up on
|
||||||
# care of the suffix dirs and possibly even the partition dir.
|
# the next rehash
|
||||||
have_nonempty_suffix = False
|
|
||||||
for suffix_path, suffix in suffixes:
|
for suffix_path, suffix in suffixes:
|
||||||
have_nonempty_hashdir = False
|
found_files = False
|
||||||
for object_hash in self._listdir(suffix_path):
|
for object_hash in self._listdir(suffix_path):
|
||||||
object_path = os.path.join(suffix_path, object_hash)
|
object_path = os.path.join(suffix_path, object_hash)
|
||||||
try:
|
try:
|
||||||
results = self.cleanup_ondisk_files(
|
results = self.cleanup_ondisk_files(
|
||||||
object_path, **kwargs)
|
object_path, **kwargs)
|
||||||
if results['files']:
|
if results['files']:
|
||||||
have_nonempty_hashdir = True
|
found_files = True
|
||||||
timestamps = {}
|
timestamps = {}
|
||||||
for ts_key, info_key, info_ts_key in key_preference:
|
for ts_key, info_key, info_ts_key in key_preference:
|
||||||
if info_key not in results:
|
if info_key not in results:
|
||||||
|
@ -1669,38 +1658,8 @@ class BaseDiskFileManager(object):
|
||||||
'Invalid diskfile filename in %r (%s)' % (
|
'Invalid diskfile filename in %r (%s)' % (
|
||||||
object_path, err))
|
object_path, err))
|
||||||
|
|
||||||
if have_nonempty_hashdir:
|
if not found_files:
|
||||||
have_nonempty_suffix = True
|
self.invalidate_hash(suffix_path)
|
||||||
else:
|
|
||||||
try:
|
|
||||||
os.rmdir(suffix_path)
|
|
||||||
except OSError as err:
|
|
||||||
if err.errno not in (errno.ENOENT, errno.ENOTEMPTY):
|
|
||||||
self.logger.debug(
|
|
||||||
'Error cleaning up empty suffix directory %s: %s',
|
|
||||||
suffix_path, err)
|
|
||||||
# cleanup_ondisk_files tries to remove empty hash dirs,
|
|
||||||
# but if it fails, so will we. An empty directory
|
|
||||||
# structure won't cause errors (just slowdown), so we
|
|
||||||
# ignore the exception.
|
|
||||||
if considering_all_suffixes and not have_nonempty_suffix:
|
|
||||||
# There's nothing of interest in the partition, so delete it
|
|
||||||
try:
|
|
||||||
# Remove hashes.pkl *then* hashes.invalid; otherwise, if we
|
|
||||||
# remove hashes.invalid but leave hashes.pkl, that makes it
|
|
||||||
# look as though the invalidations in hashes.invalid never
|
|
||||||
# occurred.
|
|
||||||
_unlink_if_present(os.path.join(partition_path, HASH_FILE))
|
|
||||||
_unlink_if_present(os.path.join(partition_path,
|
|
||||||
HASH_INVALIDATIONS_FILE))
|
|
||||||
# This lock is only held by people dealing with the hashes
|
|
||||||
# or the hash invalidations, and we've just removed those.
|
|
||||||
_unlink_if_present(os.path.join(partition_path, ".lock"))
|
|
||||||
_unlink_if_present(os.path.join(partition_path,
|
|
||||||
".lock-replication"))
|
|
||||||
os.rmdir(partition_path)
|
|
||||||
except OSError as err:
|
|
||||||
self.logger.debug("Error cleaning up empty partition: %s", err)
|
|
||||||
|
|
||||||
|
|
||||||
class BaseDiskFileWriter(object):
|
class BaseDiskFileWriter(object):
|
||||||
|
|
|
@ -412,6 +412,9 @@ class DiskFile(object):
|
||||||
raise DiskFileNotOpen()
|
raise DiskFileNotOpen()
|
||||||
return self._metadata
|
return self._metadata
|
||||||
|
|
||||||
|
get_datafile_metadata = get_metadata
|
||||||
|
get_metafile_metadata = get_metadata
|
||||||
|
|
||||||
def read_metadata(self, current_time=None):
|
def read_metadata(self, current_time=None):
|
||||||
"""
|
"""
|
||||||
Return the metadata for an object.
|
Return the metadata for an object.
|
||||||
|
|
|
@ -25,6 +25,7 @@
|
||||||
# collected. We've seen objects hang around forever otherwise.
|
# collected. We've seen objects hang around forever otherwise.
|
||||||
|
|
||||||
from six.moves.urllib.parse import unquote
|
from six.moves.urllib.parse import unquote
|
||||||
|
from six.moves import zip
|
||||||
|
|
||||||
import collections
|
import collections
|
||||||
import itertools
|
import itertools
|
||||||
|
@ -697,7 +698,8 @@ class BaseObjectController(Controller):
|
||||||
"""
|
"""
|
||||||
raise NotImplementedError()
|
raise NotImplementedError()
|
||||||
|
|
||||||
def _delete_object(self, req, obj_ring, partition, headers):
|
def _delete_object(self, req, obj_ring, partition, headers,
|
||||||
|
node_count=None, node_iterator=None):
|
||||||
"""Delete object considering write-affinity.
|
"""Delete object considering write-affinity.
|
||||||
|
|
||||||
When deleting object in write affinity deployment, also take configured
|
When deleting object in write affinity deployment, also take configured
|
||||||
|
@ -711,37 +713,12 @@ class BaseObjectController(Controller):
|
||||||
:param headers: system headers to storage nodes
|
:param headers: system headers to storage nodes
|
||||||
:return: Response object
|
:return: Response object
|
||||||
"""
|
"""
|
||||||
policy_index = req.headers.get('X-Backend-Storage-Policy-Index')
|
|
||||||
policy = POLICIES.get_by_index(policy_index)
|
|
||||||
|
|
||||||
node_count = None
|
|
||||||
node_iterator = None
|
|
||||||
|
|
||||||
policy_options = self.app.get_policy_options(policy)
|
|
||||||
is_local = policy_options.write_affinity_is_local_fn
|
|
||||||
if is_local is not None:
|
|
||||||
primaries = obj_ring.get_part_nodes(partition)
|
|
||||||
node_count = len(primaries)
|
|
||||||
|
|
||||||
local_handoffs = policy_options.write_affinity_handoff_delete_count
|
|
||||||
if local_handoffs is None:
|
|
||||||
local_primaries = [node for node in primaries
|
|
||||||
if is_local(node)]
|
|
||||||
local_handoffs = len(primaries) - len(local_primaries)
|
|
||||||
|
|
||||||
node_count += local_handoffs
|
|
||||||
|
|
||||||
node_iterator = self.iter_nodes_local_first(
|
|
||||||
obj_ring, partition, policy=policy, local_handoffs_first=True
|
|
||||||
)
|
|
||||||
|
|
||||||
status_overrides = {404: 204}
|
status_overrides = {404: 204}
|
||||||
resp = self.make_requests(req, obj_ring,
|
resp = self.make_requests(req, obj_ring,
|
||||||
partition, 'DELETE', req.swift_entity_path,
|
partition, 'DELETE', req.swift_entity_path,
|
||||||
headers, overrides=status_overrides,
|
headers, overrides=status_overrides,
|
||||||
node_count=node_count,
|
node_count=node_count,
|
||||||
node_iterator=node_iterator)
|
node_iterator=node_iterator)
|
||||||
|
|
||||||
return resp
|
return resp
|
||||||
|
|
||||||
def _post_object(self, req, obj_ring, partition, headers):
|
def _post_object(self, req, obj_ring, partition, headers):
|
||||||
|
@ -861,6 +838,7 @@ class BaseObjectController(Controller):
|
||||||
|
|
||||||
# Include local handoff nodes if write-affinity is enabled.
|
# Include local handoff nodes if write-affinity is enabled.
|
||||||
node_count = len(nodes)
|
node_count = len(nodes)
|
||||||
|
node_iterator = None
|
||||||
policy = POLICIES.get_by_index(policy_index)
|
policy = POLICIES.get_by_index(policy_index)
|
||||||
policy_options = self.app.get_policy_options(policy)
|
policy_options = self.app.get_policy_options(policy)
|
||||||
is_local = policy_options.write_affinity_is_local_fn
|
is_local = policy_options.write_affinity_is_local_fn
|
||||||
|
@ -870,11 +848,16 @@ class BaseObjectController(Controller):
|
||||||
local_primaries = [node for node in nodes if is_local(node)]
|
local_primaries = [node for node in nodes if is_local(node)]
|
||||||
local_handoffs = len(nodes) - len(local_primaries)
|
local_handoffs = len(nodes) - len(local_primaries)
|
||||||
node_count += local_handoffs
|
node_count += local_handoffs
|
||||||
|
node_iterator = self.iter_nodes_local_first(
|
||||||
|
obj_ring, partition, policy=policy, local_handoffs_first=True
|
||||||
|
)
|
||||||
|
|
||||||
headers = self._backend_requests(
|
headers = self._backend_requests(
|
||||||
req, node_count, container_partition, container_nodes,
|
req, node_count, container_partition, container_nodes,
|
||||||
container_path=container_path)
|
container_path=container_path)
|
||||||
return self._delete_object(req, obj_ring, partition, headers)
|
return self._delete_object(req, obj_ring, partition, headers,
|
||||||
|
node_count=node_count,
|
||||||
|
node_iterator=node_iterator)
|
||||||
|
|
||||||
|
|
||||||
@ObjectControllerRouter.register(REPL_POLICY)
|
@ObjectControllerRouter.register(REPL_POLICY)
|
||||||
|
@ -1112,18 +1095,14 @@ class ECAppIter(object):
|
||||||
resp.content_type = self.learned_content_type
|
resp.content_type = self.learned_content_type
|
||||||
resp.content_length = self.obj_length
|
resp.content_length = self.obj_length
|
||||||
|
|
||||||
def _next_range(self):
|
def _next_ranges(self):
|
||||||
# Each FA part should have approximately the same headers. We really
|
# Each FA part should have approximately the same headers. We really
|
||||||
# only care about Content-Range and Content-Type, and that'll be the
|
# only care about Content-Range and Content-Type, and that'll be the
|
||||||
# same for all the different FAs.
|
# same for all the different FAs.
|
||||||
frag_iters = []
|
for part_infos in zip(*self.internal_parts_iters):
|
||||||
headers = None
|
frag_iters = [pi['part_iter'] for pi in part_infos]
|
||||||
for parts_iter in self.internal_parts_iters:
|
headers = HeaderKeyDict(part_infos[0]['headers'])
|
||||||
part_info = next(parts_iter)
|
yield headers, frag_iters
|
||||||
frag_iters.append(part_info['part_iter'])
|
|
||||||
headers = part_info['headers']
|
|
||||||
headers = HeaderKeyDict(headers)
|
|
||||||
return headers, frag_iters
|
|
||||||
|
|
||||||
def _actual_range(self, req_start, req_end, entity_length):
|
def _actual_range(self, req_start, req_end, entity_length):
|
||||||
# Takes 3 args: (requested-first-byte, requested-last-byte,
|
# Takes 3 args: (requested-first-byte, requested-last-byte,
|
||||||
|
@ -1272,11 +1251,7 @@ class ECAppIter(object):
|
||||||
seen_first_headers = False
|
seen_first_headers = False
|
||||||
ranges_for_resp = {}
|
ranges_for_resp = {}
|
||||||
|
|
||||||
while True:
|
for headers, frag_iters in self._next_ranges():
|
||||||
# this'll raise StopIteration and exit the loop
|
|
||||||
next_range = self._next_range()
|
|
||||||
|
|
||||||
headers, frag_iters = next_range
|
|
||||||
content_type = headers['Content-Type']
|
content_type = headers['Content-Type']
|
||||||
|
|
||||||
content_range = headers.get('Content-Range')
|
content_range = headers.get('Content-Range')
|
||||||
|
|
|
@ -53,8 +53,7 @@ from test.unit import SkipTest
|
||||||
|
|
||||||
from swift.common import constraints, utils, ring, storage_policy
|
from swift.common import constraints, utils, ring, storage_policy
|
||||||
from swift.common.ring import Ring
|
from swift.common.ring import Ring
|
||||||
from swift.common.wsgi import (
|
from swift.common.wsgi import loadapp, SwiftHttpProtocol
|
||||||
monkey_patch_mimetools, loadapp, SwiftHttpProtocol)
|
|
||||||
from swift.common.utils import config_true_value, split_path
|
from swift.common.utils import config_true_value, split_path
|
||||||
from swift.account import server as account_server
|
from swift.account import server as account_server
|
||||||
from swift.container import server as container_server
|
from swift.container import server as container_server
|
||||||
|
@ -493,8 +492,6 @@ def in_process_setup(the_object_server=object_server):
|
||||||
swift_conf_src = _in_process_find_conf_file(conf_src_dir, 'swift.conf')
|
swift_conf_src = _in_process_find_conf_file(conf_src_dir, 'swift.conf')
|
||||||
_info('Using swift config from %s' % swift_conf_src)
|
_info('Using swift config from %s' % swift_conf_src)
|
||||||
|
|
||||||
monkey_patch_mimetools()
|
|
||||||
|
|
||||||
global _testdir
|
global _testdir
|
||||||
_testdir = os.path.join(mkdtemp(), 'tmp_functional')
|
_testdir = os.path.join(mkdtemp(), 'tmp_functional')
|
||||||
utils.mkdirs(_testdir)
|
utils.mkdirs(_testdir)
|
||||||
|
@ -1292,3 +1289,15 @@ def requires_policies(f):
|
||||||
return f(self, *args, **kwargs)
|
return f(self, *args, **kwargs)
|
||||||
|
|
||||||
return wrapper
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
|
def requires_bulk(f):
|
||||||
|
@functools.wraps(f)
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
if skip or not cluster_info:
|
||||||
|
raise SkipTest('Requires bulk middleware')
|
||||||
|
# Determine whether this cluster has bulk middleware; if not, skip test
|
||||||
|
if not cluster_info.get('bulk_upload', {}):
|
||||||
|
raise SkipTest('Requires bulk middleware')
|
||||||
|
return f(*args, **kwargs)
|
||||||
|
return wrapper
|
||||||
|
|
|
@ -49,7 +49,8 @@ class TestDloEnv(BaseEnv):
|
||||||
file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter))
|
file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter))
|
||||||
file_item.write(letter * 10)
|
file_item.write(letter * 10)
|
||||||
|
|
||||||
file_item = cls.container.file("%s/seg_upper%s" % (prefix, letter))
|
file_item = cls.container.file(
|
||||||
|
"%s/seg_upper_%%ff%s" % (prefix, letter))
|
||||||
file_item.write(letter.upper() * 10)
|
file_item.write(letter.upper() * 10)
|
||||||
|
|
||||||
for letter in ('f', 'g', 'h', 'i', 'j'):
|
for letter in ('f', 'g', 'h', 'i', 'j'):
|
||||||
|
@ -64,7 +65,7 @@ class TestDloEnv(BaseEnv):
|
||||||
|
|
||||||
man2 = cls.container.file("man2")
|
man2 = cls.container.file("man2")
|
||||||
man2.write('man2-contents',
|
man2.write('man2-contents',
|
||||||
hdrs={"X-Object-Manifest": "%s/%s/seg_upper" %
|
hdrs={"X-Object-Manifest": "%s/%s/seg_upper_%%25ff" %
|
||||||
(cls.container.name, prefix)})
|
(cls.container.name, prefix)})
|
||||||
|
|
||||||
manall = cls.container.file("manall")
|
manall = cls.container.file("manall")
|
||||||
|
|
|
@ -20,11 +20,12 @@ import json
|
||||||
import unittest2
|
import unittest2
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
import time
|
import time
|
||||||
|
from xml.dom import minidom
|
||||||
|
|
||||||
from six.moves import range
|
from six.moves import range
|
||||||
|
|
||||||
from test.functional import check_response, retry, requires_acls, \
|
from test.functional import check_response, retry, requires_acls, \
|
||||||
requires_policies, SkipTest
|
requires_policies, SkipTest, requires_bulk
|
||||||
import test.functional as tf
|
import test.functional as tf
|
||||||
|
|
||||||
|
|
||||||
|
@ -1646,6 +1647,37 @@ class TestObject(unittest2.TestCase):
|
||||||
self.assertEqual(resp.status, 200)
|
self.assertEqual(resp.status, 200)
|
||||||
self.assertEqual(body, resp.read())
|
self.assertEqual(body, resp.read())
|
||||||
|
|
||||||
|
@requires_bulk
|
||||||
|
def test_bulk_delete(self):
|
||||||
|
|
||||||
|
def bulk_delete(url, token, parsed, conn):
|
||||||
|
# try to bulk delete the object that was created during test setup
|
||||||
|
conn.request('DELETE', '%s/%s/%s?bulk-delete' % (
|
||||||
|
parsed.path, self.container, self.obj),
|
||||||
|
'%s/%s' % (self.container, self.obj),
|
||||||
|
{'X-Auth-Token': token,
|
||||||
|
'Accept': 'application/xml',
|
||||||
|
'Expect': '100-continue',
|
||||||
|
'Content-Type': 'text/plain'})
|
||||||
|
return check_response(conn)
|
||||||
|
resp = retry(bulk_delete)
|
||||||
|
self.assertEqual(resp.status, 200)
|
||||||
|
body = resp.read()
|
||||||
|
tree = minidom.parseString(body)
|
||||||
|
self.assertEqual(tree.documentElement.tagName, 'delete')
|
||||||
|
|
||||||
|
errors = tree.getElementsByTagName('errors')
|
||||||
|
self.assertEqual(len(errors), 1)
|
||||||
|
errors = [c.data if c.nodeType == c.TEXT_NODE else c.childNodes[0].data
|
||||||
|
for c in errors[0].childNodes
|
||||||
|
if c.nodeType != c.TEXT_NODE or c.data.strip()]
|
||||||
|
self.assertEqual(errors, [])
|
||||||
|
|
||||||
|
final_status = tree.getElementsByTagName('response_status')
|
||||||
|
self.assertEqual(len(final_status), 1)
|
||||||
|
self.assertEqual(len(final_status[0].childNodes), 1)
|
||||||
|
self.assertEqual(final_status[0].childNodes[0].data, '200 OK')
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest2.main()
|
unittest2.main()
|
||||||
|
|
|
@ -96,6 +96,8 @@ class TestSloEnv(BaseEnv):
|
||||||
seg_info['seg_e']]),
|
seg_info['seg_e']]),
|
||||||
parms={'multipart-manifest': 'put'})
|
parms={'multipart-manifest': 'put'})
|
||||||
|
|
||||||
|
cls.container.file('seg_with_%ff_funky_name').write('z' * 10)
|
||||||
|
|
||||||
# Put the same manifest in the container2
|
# Put the same manifest in the container2
|
||||||
file_item = cls.container2.file("manifest-abcde")
|
file_item = cls.container2.file("manifest-abcde")
|
||||||
file_item.write(
|
file_item.write(
|
||||||
|
@ -564,6 +566,17 @@ class TestSlo(Base):
|
||||||
parms={'multipart-manifest': 'put'})
|
parms={'multipart-manifest': 'put'})
|
||||||
self.assert_status(201)
|
self.assert_status(201)
|
||||||
|
|
||||||
|
def test_slo_funky_segment(self):
|
||||||
|
file_item = self.env.container.file("manifest-with-funky-segment")
|
||||||
|
file_item.write(
|
||||||
|
json.dumps([{
|
||||||
|
'path': '/%s/%s' % (self.env.container.name,
|
||||||
|
'seg_with_%ff_funky_name')}]),
|
||||||
|
parms={'multipart-manifest': 'put'})
|
||||||
|
self.assert_status(201)
|
||||||
|
|
||||||
|
self.assertEqual('z' * 10, file_item.read())
|
||||||
|
|
||||||
def test_slo_missing_etag(self):
|
def test_slo_missing_etag(self):
|
||||||
file_item = self.env.container.file("manifest-a-missing-etag")
|
file_item = self.env.container.file("manifest-a-missing-etag")
|
||||||
file_item.write(
|
file_item.write(
|
||||||
|
|
|
@ -128,11 +128,13 @@ class Base(unittest2.TestCase):
|
||||||
|
|
||||||
|
|
||||||
class Base2(object):
|
class Base2(object):
|
||||||
def setUp(self):
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
Utils.create_name = Utils.create_utf8_name
|
Utils.create_name = Utils.create_utf8_name
|
||||||
super(Base2, self).setUp()
|
super(Base2, cls).setUpClass()
|
||||||
|
|
||||||
def tearDown(self):
|
@classmethod
|
||||||
|
def tearDownClass(cls):
|
||||||
Utils.create_name = Utils.create_ascii_name
|
Utils.create_name = Utils.create_ascii_name
|
||||||
|
|
||||||
|
|
||||||
|
@ -353,8 +355,10 @@ class TestAccount(Base):
|
||||||
self.assertEqual(len(containers), len(self.env.containers))
|
self.assertEqual(len(containers), len(self.env.containers))
|
||||||
self.assert_status(200)
|
self.assert_status(200)
|
||||||
|
|
||||||
|
marker = (containers[-1] if format_type is None
|
||||||
|
else containers[-1]['name'])
|
||||||
containers = self.env.account.containers(
|
containers = self.env.account.containers(
|
||||||
parms={'format': format_type, 'marker': containers[-1]})
|
parms={'format': format_type, 'marker': marker})
|
||||||
self.assertEqual(len(containers), 0)
|
self.assertEqual(len(containers), 0)
|
||||||
if format_type is None:
|
if format_type is None:
|
||||||
self.assert_status(204)
|
self.assert_status(204)
|
||||||
|
@ -771,8 +775,9 @@ class TestContainer(Base):
|
||||||
self.assertEqual(len(files), len(self.env.files))
|
self.assertEqual(len(files), len(self.env.files))
|
||||||
self.assert_status(200)
|
self.assert_status(200)
|
||||||
|
|
||||||
|
marker = files[-1] if format_type is None else files[-1]['name']
|
||||||
files = self.env.container.files(
|
files = self.env.container.files(
|
||||||
parms={'format': format_type, 'marker': files[-1]})
|
parms={'format': format_type, 'marker': marker})
|
||||||
self.assertEqual(len(files), 0)
|
self.assertEqual(len(files), 0)
|
||||||
|
|
||||||
if format_type is None:
|
if format_type is None:
|
||||||
|
|
|
@ -142,8 +142,8 @@ class TestReplicatorFunctions(ReplProbeTest):
|
||||||
for files in test_node_files_list:
|
for files in test_node_files_list:
|
||||||
self.assertIn(files, new_files_list[0])
|
self.assertIn(files, new_files_list[0])
|
||||||
|
|
||||||
for dir in test_node_dir_list:
|
for directory in test_node_dir_list:
|
||||||
self.assertIn(dir, new_dir_list[0])
|
self.assertIn(directory, new_dir_list[0])
|
||||||
break
|
break
|
||||||
except Exception:
|
except Exception:
|
||||||
if time.time() - begin > 60:
|
if time.time() - begin > 60:
|
||||||
|
|
|
@ -22,7 +22,6 @@ import unittest
|
||||||
|
|
||||||
from logging import DEBUG
|
from logging import DEBUG
|
||||||
from mock import patch, call, DEFAULT
|
from mock import patch, call, DEFAULT
|
||||||
import six
|
|
||||||
import eventlet
|
import eventlet
|
||||||
|
|
||||||
from swift.account import reaper
|
from swift.account import reaper
|
||||||
|
@ -85,9 +84,13 @@ class FakeAccountBroker(object):
|
||||||
'delete_timestamp': time.time() - 10}
|
'delete_timestamp': time.time() - 10}
|
||||||
return info
|
return info
|
||||||
|
|
||||||
def list_containers_iter(self, *args):
|
def list_containers_iter(self, limit, marker, *args):
|
||||||
for cont in self.containers:
|
for cont in self.containers:
|
||||||
yield cont, None, None, None, None
|
if cont > marker:
|
||||||
|
yield cont, None, None, None, None
|
||||||
|
limit -= 1
|
||||||
|
if limit <= 0:
|
||||||
|
break
|
||||||
|
|
||||||
def is_status_deleted(self):
|
def is_status_deleted(self):
|
||||||
return True
|
return True
|
||||||
|
@ -196,11 +199,11 @@ class TestReaper(unittest.TestCase):
|
||||||
raise self.myexp
|
raise self.myexp
|
||||||
if self.timeout:
|
if self.timeout:
|
||||||
raise eventlet.Timeout()
|
raise eventlet.Timeout()
|
||||||
objects = [{'name': 'o1'},
|
objects = [{'name': u'o1'},
|
||||||
{'name': 'o2'},
|
{'name': u'o2'},
|
||||||
{'name': six.text_type('o3')},
|
{'name': u'o3'},
|
||||||
{'name': ''}]
|
{'name': u'o4'}]
|
||||||
return None, objects
|
return None, [o for o in objects if o['name'] > kwargs['marker']]
|
||||||
|
|
||||||
def fake_container_ring(self):
|
def fake_container_ring(self):
|
||||||
return FakeRing()
|
return FakeRing()
|
||||||
|
@ -589,7 +592,7 @@ class TestReaper(unittest.TestCase):
|
||||||
self.r.stats_return_codes.get(2, 0) + 1
|
self.r.stats_return_codes.get(2, 0) + 1
|
||||||
|
|
||||||
def test_reap_account(self):
|
def test_reap_account(self):
|
||||||
containers = ('c1', 'c2', 'c3', '')
|
containers = ('c1', 'c2', 'c3', 'c4')
|
||||||
broker = FakeAccountBroker(containers)
|
broker = FakeAccountBroker(containers)
|
||||||
self.called_amount = 0
|
self.called_amount = 0
|
||||||
self.r = r = self.init_reaper({}, fakelogger=True)
|
self.r = r = self.init_reaper({}, fakelogger=True)
|
||||||
|
|
|
@ -482,6 +482,78 @@ class TestKeymaster(unittest.TestCase):
|
||||||
self.assertEqual(expected_keys, keys)
|
self.assertEqual(expected_keys, keys)
|
||||||
self.assertEqual([('/a/c', None), ('/a/c//o', None)], calls)
|
self.assertEqual([('/a/c', None), ('/a/c//o', None)], calls)
|
||||||
|
|
||||||
|
def test_v1_keys_with_weird_paths(self):
|
||||||
|
secrets = {None: os.urandom(32),
|
||||||
|
'22': os.urandom(33)}
|
||||||
|
conf = {}
|
||||||
|
for secret_id, secret in secrets.items():
|
||||||
|
opt = ('encryption_root_secret%s' %
|
||||||
|
(('_%s' % secret_id) if secret_id else ''))
|
||||||
|
conf[opt] = base64.b64encode(secret)
|
||||||
|
conf['active_root_secret_id'] = '22'
|
||||||
|
self.app = keymaster.KeyMaster(self.swift, conf)
|
||||||
|
orig_create_key = self.app.create_key
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def mock_create_key(path, secret_id=None):
|
||||||
|
calls.append((path, secret_id))
|
||||||
|
return orig_create_key(path, secret_id)
|
||||||
|
|
||||||
|
# request path doesn't match stored path -- this could happen if you
|
||||||
|
# misconfigured your proxy to have copy right of encryption
|
||||||
|
context = keymaster.KeyMasterContext(self.app, 'a', 'not-c', 'not-o')
|
||||||
|
for version in ('1', '2'):
|
||||||
|
with mock.patch.object(self.app, 'create_key', mock_create_key):
|
||||||
|
keys = context.fetch_crypto_keys(key_id={
|
||||||
|
'v': version, 'path': '/a/c/o'})
|
||||||
|
expected_keys = {
|
||||||
|
'container': hmac.new(secrets[None], b'/a/c',
|
||||||
|
digestmod=hashlib.sha256).digest(),
|
||||||
|
'object': hmac.new(secrets[None], b'/a/c/o',
|
||||||
|
digestmod=hashlib.sha256).digest(),
|
||||||
|
'id': {'path': '/a/c/o', 'v': version},
|
||||||
|
'all_ids': [
|
||||||
|
{'path': '/a/c/o', 'v': version},
|
||||||
|
{'path': '/a/c/o', 'secret_id': '22', 'v': version}]}
|
||||||
|
self.assertEqual(expected_keys, keys)
|
||||||
|
self.assertEqual([('/a/c', None), ('/a/c/o', None)], calls)
|
||||||
|
del calls[:]
|
||||||
|
|
||||||
|
context = keymaster.KeyMasterContext(
|
||||||
|
self.app, 'not-a', 'not-c', '/not-o')
|
||||||
|
with mock.patch.object(self.app, 'create_key', mock_create_key):
|
||||||
|
keys = context.fetch_crypto_keys(key_id={
|
||||||
|
'v': '1', 'path': '/o'})
|
||||||
|
expected_keys = {
|
||||||
|
'container': hmac.new(secrets[None], b'/not-a/not-c',
|
||||||
|
digestmod=hashlib.sha256).digest(),
|
||||||
|
'object': hmac.new(secrets[None], b'/o',
|
||||||
|
digestmod=hashlib.sha256).digest(),
|
||||||
|
'id': {'path': '/o', 'v': '1'},
|
||||||
|
'all_ids': [
|
||||||
|
{'path': '/o', 'v': '1'},
|
||||||
|
{'path': '/o', 'secret_id': '22', 'v': '1'}]}
|
||||||
|
self.assertEqual(expected_keys, keys)
|
||||||
|
self.assertEqual([('/not-a/not-c', None), ('/o', None)], calls)
|
||||||
|
del calls[:]
|
||||||
|
|
||||||
|
context = keymaster.KeyMasterContext(
|
||||||
|
self.app, 'not-a', 'not-c', '/not-o')
|
||||||
|
with mock.patch.object(self.app, 'create_key', mock_create_key):
|
||||||
|
keys = context.fetch_crypto_keys(key_id={
|
||||||
|
'v': '2', 'path': '/a/c//o'})
|
||||||
|
expected_keys = {
|
||||||
|
'container': hmac.new(secrets[None], b'/a/c',
|
||||||
|
digestmod=hashlib.sha256).digest(),
|
||||||
|
'object': hmac.new(secrets[None], b'/a/c//o',
|
||||||
|
digestmod=hashlib.sha256).digest(),
|
||||||
|
'id': {'path': '/a/c//o', 'v': '2'},
|
||||||
|
'all_ids': [
|
||||||
|
{'path': '/a/c//o', 'v': '2'},
|
||||||
|
{'path': '/a/c//o', 'secret_id': '22', 'v': '2'}]}
|
||||||
|
self.assertEqual(expected_keys, keys)
|
||||||
|
self.assertEqual([('/a/c', None), ('/a/c//o', None)], calls)
|
||||||
|
|
||||||
@mock.patch('swift.common.middleware.crypto.keymaster.readconf')
|
@mock.patch('swift.common.middleware.crypto.keymaster.readconf')
|
||||||
def test_keymaster_config_path(self, mock_readconf):
|
def test_keymaster_config_path(self, mock_readconf):
|
||||||
for secret in (os.urandom(32), os.urandom(33), os.urandom(50)):
|
for secret in (os.urandom(32), os.urandom(33), os.urandom(50)):
|
||||||
|
|
|
@ -717,7 +717,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
|
||||||
'Response Status': '201 Created',
|
'Response Status': '201 Created',
|
||||||
'Errors': [],
|
'Errors': [],
|
||||||
})])
|
})])
|
||||||
mock_time.return_value.time.side_effect = (
|
mock_time.time.side_effect = (
|
||||||
1, # start_time
|
1, # start_time
|
||||||
12, # first whitespace
|
12, # first whitespace
|
||||||
13, # second...
|
13, # second...
|
||||||
|
@ -769,7 +769,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
|
||||||
'Response Status': '400 Bad Request',
|
'Response Status': '400 Bad Request',
|
||||||
'Errors': [['some/object', '403 Forbidden']],
|
'Errors': [['some/object', '403 Forbidden']],
|
||||||
})])
|
})])
|
||||||
mock_time.return_value.time.side_effect = (
|
mock_time.time.side_effect = (
|
||||||
1, # start_time
|
1, # start_time
|
||||||
12, # first whitespace
|
12, # first whitespace
|
||||||
13, # second...
|
13, # second...
|
||||||
|
@ -818,7 +818,7 @@ class TestS3ApiMultiUpload(S3ApiTestCase):
|
||||||
'Response Status': '400 Bad Request',
|
'Response Status': '400 Bad Request',
|
||||||
'Errors': [['some/object', '404 Not Found']],
|
'Errors': [['some/object', '404 Not Found']],
|
||||||
})])
|
})])
|
||||||
mock_time.return_value.time.side_effect = (
|
mock_time.time.side_effect = (
|
||||||
1, # start_time
|
1, # start_time
|
||||||
12, # first whitespace
|
12, # first whitespace
|
||||||
13, # second...
|
13, # second...
|
||||||
|
|
|
@ -469,6 +469,60 @@ class TestS3ApiObj(S3ApiTestCase):
|
||||||
# Check that s3api converts a Content-MD5 header into an etag.
|
# Check that s3api converts a Content-MD5 header into an etag.
|
||||||
self.assertEqual(headers['etag'], etag)
|
self.assertEqual(headers['etag'], etag)
|
||||||
|
|
||||||
|
@s3acl
|
||||||
|
def test_object_PUT_v4(self):
|
||||||
|
body_sha = hashlib.sha256(self.object_body).hexdigest()
|
||||||
|
req = Request.blank(
|
||||||
|
'/bucket/object',
|
||||||
|
environ={'REQUEST_METHOD': 'PUT'},
|
||||||
|
headers={
|
||||||
|
'Authorization':
|
||||||
|
'AWS4-HMAC-SHA256 '
|
||||||
|
'Credential=test:tester/%s/us-east-1/s3/aws4_request, '
|
||||||
|
'SignedHeaders=host;x-amz-date, '
|
||||||
|
'Signature=hmac' % (
|
||||||
|
self.get_v4_amz_date_header().split('T', 1)[0]),
|
||||||
|
'x-amz-date': self.get_v4_amz_date_header(),
|
||||||
|
'x-amz-storage-class': 'STANDARD',
|
||||||
|
'x-amz-content-sha256': body_sha,
|
||||||
|
'Date': self.get_date_header()},
|
||||||
|
body=self.object_body)
|
||||||
|
req.date = datetime.now()
|
||||||
|
req.content_type = 'text/plain'
|
||||||
|
status, headers, body = self.call_s3api(req)
|
||||||
|
self.assertEqual(status.split()[0], '200')
|
||||||
|
# Check that s3api returns an etag header.
|
||||||
|
self.assertEqual(headers['etag'],
|
||||||
|
'"%s"' % self.response_headers['etag'])
|
||||||
|
|
||||||
|
_, _, headers = self.swift.calls_with_headers[-1]
|
||||||
|
# No way to determine ETag to send
|
||||||
|
self.assertNotIn('etag', headers)
|
||||||
|
|
||||||
|
@s3acl
|
||||||
|
def test_object_PUT_v4_bad_hash(self):
|
||||||
|
req = Request.blank(
|
||||||
|
'/bucket/object',
|
||||||
|
environ={'REQUEST_METHOD': 'PUT'},
|
||||||
|
headers={
|
||||||
|
'Authorization':
|
||||||
|
'AWS4-HMAC-SHA256 '
|
||||||
|
'Credential=test:tester/%s/us-east-1/s3/aws4_request, '
|
||||||
|
'SignedHeaders=host;x-amz-date, '
|
||||||
|
'Signature=hmac' % (
|
||||||
|
self.get_v4_amz_date_header().split('T', 1)[0]),
|
||||||
|
'x-amz-date': self.get_v4_amz_date_header(),
|
||||||
|
'x-amz-storage-class': 'STANDARD',
|
||||||
|
'x-amz-content-sha256': 'not the hash',
|
||||||
|
'Date': self.get_date_header()},
|
||||||
|
body=self.object_body)
|
||||||
|
req.date = datetime.now()
|
||||||
|
req.content_type = 'text/plain'
|
||||||
|
status, headers, body = self.call_s3api(req)
|
||||||
|
self.assertEqual(status.split()[0], '400')
|
||||||
|
print(body)
|
||||||
|
self.assertEqual(self._get_error_code(body), 'BadDigest')
|
||||||
|
|
||||||
def test_object_PUT_headers(self):
|
def test_object_PUT_headers(self):
|
||||||
content_md5 = self.etag.decode('hex').encode('base64').strip()
|
content_md5 = self.etag.decode('hex').encode('base64').strip()
|
||||||
|
|
||||||
|
|
|
@ -13,9 +13,12 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
import hashlib
|
||||||
from mock import patch, MagicMock
|
from mock import patch, MagicMock
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
|
from six import BytesIO
|
||||||
|
|
||||||
from swift.common import swob
|
from swift.common import swob
|
||||||
from swift.common.swob import Request, HTTPNoContent
|
from swift.common.swob import Request, HTTPNoContent
|
||||||
from swift.common.middleware.s3api.utils import mktime
|
from swift.common.middleware.s3api.utils import mktime
|
||||||
|
@ -24,7 +27,7 @@ from swift.common.middleware.s3api.subresource import ACL, User, Owner, \
|
||||||
Grant, encode_acl
|
Grant, encode_acl
|
||||||
from test.unit.common.middleware.s3api.test_s3api import S3ApiTestCase
|
from test.unit.common.middleware.s3api.test_s3api import S3ApiTestCase
|
||||||
from swift.common.middleware.s3api.s3request import S3Request, \
|
from swift.common.middleware.s3api.s3request import S3Request, \
|
||||||
S3AclRequest, SigV4Request, SIGV4_X_AMZ_DATE_FORMAT
|
S3AclRequest, SigV4Request, SIGV4_X_AMZ_DATE_FORMAT, HashingInput
|
||||||
from swift.common.middleware.s3api.s3response import InvalidArgument, \
|
from swift.common.middleware.s3api.s3response import InvalidArgument, \
|
||||||
NoSuchBucket, InternalError, \
|
NoSuchBucket, InternalError, \
|
||||||
AccessDenied, SignatureDoesNotMatch, RequestTimeTooSkewed
|
AccessDenied, SignatureDoesNotMatch, RequestTimeTooSkewed
|
||||||
|
@ -802,5 +805,76 @@ class TestRequest(S3ApiTestCase):
|
||||||
u'\u30c9\u30e9\u30b4\u30f3'))
|
u'\u30c9\u30e9\u30b4\u30f3'))
|
||||||
|
|
||||||
|
|
||||||
|
class TestHashingInput(S3ApiTestCase):
|
||||||
|
def test_good(self):
|
||||||
|
raw = b'123456789'
|
||||||
|
wrapped = HashingInput(BytesIO(raw), 9, hashlib.md5,
|
||||||
|
hashlib.md5(raw).hexdigest())
|
||||||
|
self.assertEqual(b'1234', wrapped.read(4))
|
||||||
|
self.assertEqual(b'56', wrapped.read(2))
|
||||||
|
# trying to read past the end gets us whatever's left
|
||||||
|
self.assertEqual(b'789', wrapped.read(4))
|
||||||
|
# can continue trying to read -- but it'll be empty
|
||||||
|
self.assertEqual(b'', wrapped.read(2))
|
||||||
|
|
||||||
|
self.assertFalse(wrapped._input.closed)
|
||||||
|
wrapped.close()
|
||||||
|
self.assertTrue(wrapped._input.closed)
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
wrapped = HashingInput(BytesIO(b''), 0, hashlib.sha256,
|
||||||
|
hashlib.sha256(b'').hexdigest())
|
||||||
|
self.assertEqual(b'', wrapped.read(4))
|
||||||
|
self.assertEqual(b'', wrapped.read(2))
|
||||||
|
|
||||||
|
self.assertFalse(wrapped._input.closed)
|
||||||
|
wrapped.close()
|
||||||
|
self.assertTrue(wrapped._input.closed)
|
||||||
|
|
||||||
|
def test_too_long(self):
|
||||||
|
raw = b'123456789'
|
||||||
|
wrapped = HashingInput(BytesIO(raw), 8, hashlib.md5,
|
||||||
|
hashlib.md5(raw).hexdigest())
|
||||||
|
self.assertEqual(b'1234', wrapped.read(4))
|
||||||
|
self.assertEqual(b'56', wrapped.read(2))
|
||||||
|
# even though the hash matches, there was more data than we expected
|
||||||
|
with self.assertRaises(swob.Response) as raised:
|
||||||
|
wrapped.read(3)
|
||||||
|
self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
|
||||||
|
# the error causes us to close the input
|
||||||
|
self.assertTrue(wrapped._input.closed)
|
||||||
|
|
||||||
|
def test_too_short(self):
|
||||||
|
raw = b'123456789'
|
||||||
|
wrapped = HashingInput(BytesIO(raw), 10, hashlib.md5,
|
||||||
|
hashlib.md5(raw).hexdigest())
|
||||||
|
self.assertEqual(b'1234', wrapped.read(4))
|
||||||
|
self.assertEqual(b'56', wrapped.read(2))
|
||||||
|
# even though the hash matches, there was more data than we expected
|
||||||
|
with self.assertRaises(swob.Response) as raised:
|
||||||
|
wrapped.read(4)
|
||||||
|
self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
|
||||||
|
self.assertTrue(wrapped._input.closed)
|
||||||
|
|
||||||
|
def test_bad_hash(self):
|
||||||
|
raw = b'123456789'
|
||||||
|
wrapped = HashingInput(BytesIO(raw), 9, hashlib.sha256,
|
||||||
|
hashlib.md5(raw).hexdigest())
|
||||||
|
self.assertEqual(b'1234', wrapped.read(4))
|
||||||
|
self.assertEqual(b'5678', wrapped.read(4))
|
||||||
|
with self.assertRaises(swob.Response) as raised:
|
||||||
|
wrapped.read(4)
|
||||||
|
self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
|
||||||
|
self.assertTrue(wrapped._input.closed)
|
||||||
|
|
||||||
|
def test_empty_bad_hash(self):
|
||||||
|
wrapped = HashingInput(BytesIO(b''), 0, hashlib.sha256, 'nope')
|
||||||
|
with self.assertRaises(swob.Response) as raised:
|
||||||
|
wrapped.read(3)
|
||||||
|
self.assertEqual(raised.exception.status, '422 Unprocessable Entity')
|
||||||
|
# the error causes us to close the input
|
||||||
|
self.assertTrue(wrapped._input.closed)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
|
@ -335,6 +335,29 @@ class TestServerSideCopyMiddleware(unittest.TestCase):
|
||||||
self.assertEqual('PUT', self.authorized[1].method)
|
self.assertEqual('PUT', self.authorized[1].method)
|
||||||
self.assertEqual('/v1/a/c/o', self.authorized[1].path)
|
self.assertEqual('/v1/a/c/o', self.authorized[1].path)
|
||||||
|
|
||||||
|
def test_copy_with_unicode(self):
|
||||||
|
self.app.register('GET', '/v1/a/c/\xF0\x9F\x8C\xB4', swob.HTTPOk,
|
||||||
|
{}, 'passed')
|
||||||
|
self.app.register('PUT', '/v1/a/c/\xE2\x98\x83', swob.HTTPCreated, {})
|
||||||
|
# Just for fun, let's have a mix of properly encoded and not
|
||||||
|
req = Request.blank('/v1/a/c/%F0\x9F%8C%B4',
|
||||||
|
environ={'REQUEST_METHOD': 'COPY'},
|
||||||
|
headers={'Content-Length': '0',
|
||||||
|
'Destination': 'c/%E2\x98%83'})
|
||||||
|
status, headers, body = self.call_ssc(req)
|
||||||
|
self.assertEqual(status, '201 Created')
|
||||||
|
calls = self.app.calls_with_headers
|
||||||
|
method, path, req_headers = calls[0]
|
||||||
|
self.assertEqual('GET', method)
|
||||||
|
self.assertEqual('/v1/a/c/\xF0\x9F\x8C\xB4', path)
|
||||||
|
self.assertIn(('X-Copied-From', 'c/%F0%9F%8C%B4'), headers)
|
||||||
|
|
||||||
|
self.assertEqual(len(self.authorized), 2)
|
||||||
|
self.assertEqual('GET', self.authorized[0].method)
|
||||||
|
self.assertEqual('/v1/a/c/%F0%9F%8C%B4', self.authorized[0].path)
|
||||||
|
self.assertEqual('PUT', self.authorized[1].method)
|
||||||
|
self.assertEqual('/v1/a/c/%E2%98%83', self.authorized[1].path)
|
||||||
|
|
||||||
def test_copy_with_spaces_in_x_copy_from_and_account(self):
|
def test_copy_with_spaces_in_x_copy_from_and_account(self):
|
||||||
self.app.register('GET', '/v1/a/c/o o2', swob.HTTPOk, {}, b'passed')
|
self.app.register('GET', '/v1/a/c/o o2', swob.HTTPOk, {}, b'passed')
|
||||||
self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {})
|
self.app.register('PUT', '/v1/a1/c1/o', swob.HTTPCreated, {})
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -1024,6 +1024,36 @@ class TestAuth(unittest.TestCase):
|
||||||
resp = req.get_response(ath)
|
resp = req.get_response(ath)
|
||||||
self.assertEqual(204, resp.status_int)
|
self.assertEqual(204, resp.status_int)
|
||||||
|
|
||||||
|
def test_request_method_not_allowed(self):
|
||||||
|
test_auth = auth.filter_factory({'user_ac_user': 'testing'})(FakeApp())
|
||||||
|
req = self._make_request(
|
||||||
|
'/auth/v1.0',
|
||||||
|
headers={'X-Auth-User': 'ac:user', 'X-Auth-Key': 'testing'},
|
||||||
|
environ={'REQUEST_METHOD': 'PUT'})
|
||||||
|
resp = req.get_response(test_auth)
|
||||||
|
self.assertEqual(resp.status_int, 405)
|
||||||
|
|
||||||
|
req = self._make_request(
|
||||||
|
'/auth/v1.0',
|
||||||
|
headers={'X-Auth-User': 'ac:user', 'X-Auth-Key': 'testing'},
|
||||||
|
environ={'REQUEST_METHOD': 'HEAD'})
|
||||||
|
resp = req.get_response(test_auth)
|
||||||
|
self.assertEqual(resp.status_int, 405)
|
||||||
|
|
||||||
|
req = self._make_request(
|
||||||
|
'/auth/v1.0',
|
||||||
|
headers={'X-Auth-User': 'ac:user', 'X-Auth-Key': 'testing'},
|
||||||
|
environ={'REQUEST_METHOD': 'POST'})
|
||||||
|
resp = req.get_response(test_auth)
|
||||||
|
self.assertEqual(resp.status_int, 405)
|
||||||
|
|
||||||
|
req = self._make_request(
|
||||||
|
'/auth/v1.0',
|
||||||
|
headers={'X-Auth-User': 'ac:user', 'X-Auth-Key': 'testing'},
|
||||||
|
environ={'REQUEST_METHOD': 'DELETE'})
|
||||||
|
resp = req.get_response(test_auth)
|
||||||
|
self.assertEqual(resp.status_int, 405)
|
||||||
|
|
||||||
|
|
||||||
class TestAuthWithMultiplePrefixes(TestAuth):
|
class TestAuthWithMultiplePrefixes(TestAuth):
|
||||||
"""
|
"""
|
||||||
|
|
|
@ -27,12 +27,8 @@ import types
|
||||||
|
|
||||||
import eventlet.wsgi
|
import eventlet.wsgi
|
||||||
|
|
||||||
import six
|
|
||||||
from six import BytesIO
|
from six import BytesIO
|
||||||
from six import StringIO
|
|
||||||
from six.moves.urllib.parse import quote
|
from six.moves.urllib.parse import quote
|
||||||
if six.PY2:
|
|
||||||
import mimetools
|
|
||||||
|
|
||||||
import mock
|
import mock
|
||||||
|
|
||||||
|
@ -69,53 +65,6 @@ def _fake_rings(tmpdir):
|
||||||
class TestWSGI(unittest.TestCase):
|
class TestWSGI(unittest.TestCase):
|
||||||
"""Tests for swift.common.wsgi"""
|
"""Tests for swift.common.wsgi"""
|
||||||
|
|
||||||
def setUp(self):
|
|
||||||
if six.PY2:
|
|
||||||
self._orig_parsetype = mimetools.Message.parsetype
|
|
||||||
|
|
||||||
def tearDown(self):
|
|
||||||
if six.PY2:
|
|
||||||
mimetools.Message.parsetype = self._orig_parsetype
|
|
||||||
|
|
||||||
@unittest.skipIf(six.PY3, "test specific to Python 2")
|
|
||||||
def test_monkey_patch_mimetools(self):
|
|
||||||
sio = StringIO('blah')
|
|
||||||
self.assertEqual(mimetools.Message(sio).type, 'text/plain')
|
|
||||||
sio = StringIO('blah')
|
|
||||||
self.assertEqual(mimetools.Message(sio).plisttext, '')
|
|
||||||
sio = StringIO('blah')
|
|
||||||
self.assertEqual(mimetools.Message(sio).maintype, 'text')
|
|
||||||
sio = StringIO('blah')
|
|
||||||
self.assertEqual(mimetools.Message(sio).subtype, 'plain')
|
|
||||||
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
|
|
||||||
self.assertEqual(mimetools.Message(sio).type, 'text/html')
|
|
||||||
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
|
|
||||||
self.assertEqual(mimetools.Message(sio).plisttext,
|
|
||||||
'; charset=ISO-8859-4')
|
|
||||||
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
|
|
||||||
self.assertEqual(mimetools.Message(sio).maintype, 'text')
|
|
||||||
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
|
|
||||||
self.assertEqual(mimetools.Message(sio).subtype, 'html')
|
|
||||||
|
|
||||||
wsgi.monkey_patch_mimetools()
|
|
||||||
sio = StringIO('blah')
|
|
||||||
self.assertIsNone(mimetools.Message(sio).type)
|
|
||||||
sio = StringIO('blah')
|
|
||||||
self.assertEqual(mimetools.Message(sio).plisttext, '')
|
|
||||||
sio = StringIO('blah')
|
|
||||||
self.assertIsNone(mimetools.Message(sio).maintype)
|
|
||||||
sio = StringIO('blah')
|
|
||||||
self.assertIsNone(mimetools.Message(sio).subtype)
|
|
||||||
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
|
|
||||||
self.assertEqual(mimetools.Message(sio).type, 'text/html')
|
|
||||||
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
|
|
||||||
self.assertEqual(mimetools.Message(sio).plisttext,
|
|
||||||
'; charset=ISO-8859-4')
|
|
||||||
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
|
|
||||||
self.assertEqual(mimetools.Message(sio).maintype, 'text')
|
|
||||||
sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
|
|
||||||
self.assertEqual(mimetools.Message(sio).subtype, 'html')
|
|
||||||
|
|
||||||
def test_init_request_processor(self):
|
def test_init_request_processor(self):
|
||||||
config = """
|
config = """
|
||||||
[DEFAULT]
|
[DEFAULT]
|
||||||
|
|
|
@ -40,6 +40,7 @@ from swift.common.storage_policy import StoragePolicy, ECStoragePolicy
|
||||||
from swift.common.middleware import listing_formats, proxy_logging
|
from swift.common.middleware import listing_formats, proxy_logging
|
||||||
from swift.common import utils
|
from swift.common import utils
|
||||||
from swift.common.utils import mkdirs, normalize_timestamp, NullLogger
|
from swift.common.utils import mkdirs, normalize_timestamp, NullLogger
|
||||||
|
from swift.common.wsgi import SwiftHttpProtocol
|
||||||
from swift.container import server as container_server
|
from swift.container import server as container_server
|
||||||
from swift.obj import server as object_server
|
from swift.obj import server as object_server
|
||||||
from swift.proxy import server as proxy_server
|
from swift.proxy import server as proxy_server
|
||||||
|
@ -212,17 +213,28 @@ def setup_servers(the_object_server=object_server, extra_conf=None):
|
||||||
nl = NullLogger()
|
nl = NullLogger()
|
||||||
logging_prosv = proxy_logging.ProxyLoggingMiddleware(
|
logging_prosv = proxy_logging.ProxyLoggingMiddleware(
|
||||||
listing_formats.ListingFilter(prosrv), conf, logger=prosrv.logger)
|
listing_formats.ListingFilter(prosrv), conf, logger=prosrv.logger)
|
||||||
prospa = spawn(wsgi.server, prolis, logging_prosv, nl)
|
prospa = spawn(wsgi.server, prolis, logging_prosv, nl,
|
||||||
acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl)
|
protocol=SwiftHttpProtocol)
|
||||||
acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl)
|
acc1spa = spawn(wsgi.server, acc1lis, acc1srv, nl,
|
||||||
con1spa = spawn(wsgi.server, con1lis, con1srv, nl)
|
protocol=SwiftHttpProtocol)
|
||||||
con2spa = spawn(wsgi.server, con2lis, con2srv, nl)
|
acc2spa = spawn(wsgi.server, acc2lis, acc2srv, nl,
|
||||||
obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl)
|
protocol=SwiftHttpProtocol)
|
||||||
obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl)
|
con1spa = spawn(wsgi.server, con1lis, con1srv, nl,
|
||||||
obj3spa = spawn(wsgi.server, obj3lis, obj3srv, nl)
|
protocol=SwiftHttpProtocol)
|
||||||
obj4spa = spawn(wsgi.server, obj4lis, obj4srv, nl)
|
con2spa = spawn(wsgi.server, con2lis, con2srv, nl,
|
||||||
obj5spa = spawn(wsgi.server, obj5lis, obj5srv, nl)
|
protocol=SwiftHttpProtocol)
|
||||||
obj6spa = spawn(wsgi.server, obj6lis, obj6srv, nl)
|
obj1spa = spawn(wsgi.server, obj1lis, obj1srv, nl,
|
||||||
|
protocol=SwiftHttpProtocol)
|
||||||
|
obj2spa = spawn(wsgi.server, obj2lis, obj2srv, nl,
|
||||||
|
protocol=SwiftHttpProtocol)
|
||||||
|
obj3spa = spawn(wsgi.server, obj3lis, obj3srv, nl,
|
||||||
|
protocol=SwiftHttpProtocol)
|
||||||
|
obj4spa = spawn(wsgi.server, obj4lis, obj4srv, nl,
|
||||||
|
protocol=SwiftHttpProtocol)
|
||||||
|
obj5spa = spawn(wsgi.server, obj5lis, obj5srv, nl,
|
||||||
|
protocol=SwiftHttpProtocol)
|
||||||
|
obj6spa = spawn(wsgi.server, obj6lis, obj6srv, nl,
|
||||||
|
protocol=SwiftHttpProtocol)
|
||||||
context["test_coros"] = \
|
context["test_coros"] = \
|
||||||
(prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa, obj3spa,
|
(prospa, acc1spa, acc2spa, con1spa, con2spa, obj1spa, obj2spa, obj3spa,
|
||||||
obj4spa, obj5spa, obj6spa)
|
obj4spa, obj5spa, obj6spa)
|
||||||
|
|
|
@ -1415,7 +1415,8 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
|
||||||
df2_suffix, df2_hash,
|
df2_suffix, df2_hash,
|
||||||
"1525354556.65758.ts")))
|
"1525354556.65758.ts")))
|
||||||
|
|
||||||
# Expire the tombstones
|
# Cache the hashes and expire the tombstones
|
||||||
|
self.df_mgr.get_hashes(self.existing_device, '0', [], POLICIES[0])
|
||||||
the_time[0] += 2 * self.df_mgr.reclaim_age
|
the_time[0] += 2 * self.df_mgr.reclaim_age
|
||||||
|
|
||||||
hashes = list(self.df_mgr.yield_hashes(
|
hashes = list(self.df_mgr.yield_hashes(
|
||||||
|
@ -1440,7 +1441,30 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
|
||||||
self.testdir, self.existing_device, 'objects', '0',
|
self.testdir, self.existing_device, 'objects', '0',
|
||||||
df2_suffix, df2_hash)))
|
df2_suffix, df2_hash)))
|
||||||
|
|
||||||
# The empty suffix dirs are gone
|
# The empty suffix dirs, and partition are still there
|
||||||
|
self.assertTrue(os.path.isdir(os.path.join(
|
||||||
|
self.testdir, self.existing_device, 'objects', '0',
|
||||||
|
df1_suffix)))
|
||||||
|
self.assertTrue(os.path.isdir(os.path.join(
|
||||||
|
self.testdir, self.existing_device, 'objects', '0',
|
||||||
|
df2_suffix)))
|
||||||
|
|
||||||
|
# but the suffixes is invalid
|
||||||
|
part_dir = os.path.join(
|
||||||
|
self.testdir, self.existing_device, 'objects', '0')
|
||||||
|
invalidations_file = os.path.join(
|
||||||
|
part_dir, diskfile.HASH_INVALIDATIONS_FILE)
|
||||||
|
with open(invalidations_file) as f:
|
||||||
|
self.assertEqual('%s\n%s' % (df1_suffix, df2_suffix),
|
||||||
|
f.read().strip('\n')) # sanity
|
||||||
|
|
||||||
|
# next time get hashes runs
|
||||||
|
with mock.patch('time.time', mock_time):
|
||||||
|
hashes = self.df_mgr.get_hashes(
|
||||||
|
self.existing_device, '0', [], POLICIES[0])
|
||||||
|
self.assertEqual(hashes, {})
|
||||||
|
|
||||||
|
# ... suffixes will get cleanup
|
||||||
self.assertFalse(os.path.exists(os.path.join(
|
self.assertFalse(os.path.exists(os.path.join(
|
||||||
self.testdir, self.existing_device, 'objects', '0',
|
self.testdir, self.existing_device, 'objects', '0',
|
||||||
df1_suffix)))
|
df1_suffix)))
|
||||||
|
@ -1448,8 +1472,9 @@ class DiskFileManagerMixin(BaseDiskFileTestMixin):
|
||||||
self.testdir, self.existing_device, 'objects', '0',
|
self.testdir, self.existing_device, 'objects', '0',
|
||||||
df2_suffix)))
|
df2_suffix)))
|
||||||
|
|
||||||
# The empty partition dir is gone
|
# but really it's not diskfile's jobs to decide if a partition belongs
|
||||||
self.assertFalse(os.path.exists(os.path.join(
|
# on a node or not
|
||||||
|
self.assertTrue(os.path.isdir(os.path.join(
|
||||||
self.testdir, self.existing_device, 'objects', '0')))
|
self.testdir, self.existing_device, 'objects', '0')))
|
||||||
|
|
||||||
def test_focused_yield_hashes_does_not_clean_up(self):
|
def test_focused_yield_hashes_does_not_clean_up(self):
|
||||||
|
|
|
@ -1327,12 +1327,31 @@ class TestGlobalSetupObjectReconstructor(unittest.TestCase):
|
||||||
for stat_key, expected in expected_stats.items():
|
for stat_key, expected in expected_stats.items():
|
||||||
stat_method, stat_prefix = stat_key
|
stat_method, stat_prefix = stat_key
|
||||||
self.assertStatCount(stat_method, stat_prefix, expected)
|
self.assertStatCount(stat_method, stat_prefix, expected)
|
||||||
|
|
||||||
|
stub_data = self.reconstructor._get_hashes(
|
||||||
|
'sda1', 2, self.policy, do_listdir=True)
|
||||||
|
stub_data.update({'7ca': {None: '8f19c38e1cf8e2390d4ca29051407ae3'}})
|
||||||
|
pickle_path = os.path.join(part_path, 'hashes.pkl')
|
||||||
|
with open(pickle_path, 'w') as f:
|
||||||
|
pickle.dump(stub_data, f)
|
||||||
|
|
||||||
# part 2 should be totally empty
|
# part 2 should be totally empty
|
||||||
hash_gen = self.reconstructor._df_router[self.policy].yield_hashes(
|
hash_gen = self.reconstructor._df_router[self.policy].yield_hashes(
|
||||||
'sda1', '2', self.policy)
|
'sda1', '2', self.policy, suffixes=stub_data.keys())
|
||||||
for path, hash_, ts in hash_gen:
|
for path, hash_, ts in hash_gen:
|
||||||
self.fail('found %s with %s in %s' % (hash_, ts, path))
|
self.fail('found %s with %s in %s' % (hash_, ts, path))
|
||||||
# even the partition directory is gone
|
|
||||||
|
new_hashes = self.reconstructor._get_hashes(
|
||||||
|
'sda1', 2, self.policy, do_listdir=True)
|
||||||
|
self.assertFalse(new_hashes)
|
||||||
|
|
||||||
|
# N.B. the partition directory is removed next pass
|
||||||
|
ssync_calls = []
|
||||||
|
with mocked_http_conn() as request_log:
|
||||||
|
with mock.patch('swift.obj.reconstructor.ssync_sender',
|
||||||
|
self._make_fake_ssync(ssync_calls)):
|
||||||
|
self.reconstructor.reconstruct(override_partitions=[2])
|
||||||
|
self.assertEqual([], ssync_calls)
|
||||||
self.assertFalse(os.access(part_path, os.F_OK))
|
self.assertFalse(os.access(part_path, os.F_OK))
|
||||||
|
|
||||||
def test_process_job_all_success(self):
|
def test_process_job_all_success(self):
|
||||||
|
|
|
@ -531,7 +531,7 @@ class CommonObjectControllerMixin(BaseObjectControllerMixin):
|
||||||
self.assertNotIn('X-Delete-At-Host', headers)
|
self.assertNotIn('X-Delete-At-Host', headers)
|
||||||
self.assertNotIn('X-Delete-At-Device', headers)
|
self.assertNotIn('X-Delete-At-Device', headers)
|
||||||
|
|
||||||
def test_DELETE_write_affinity_before_replication(self):
|
def test_DELETE_write_affinity_after_replication(self):
|
||||||
policy_conf = self.app.get_policy_options(self.policy)
|
policy_conf = self.app.get_policy_options(self.policy)
|
||||||
policy_conf.write_affinity_handoff_delete_count = self.replicas() // 2
|
policy_conf.write_affinity_handoff_delete_count = self.replicas() // 2
|
||||||
policy_conf.write_affinity_is_local_fn = (
|
policy_conf.write_affinity_is_local_fn = (
|
||||||
|
@ -545,7 +545,7 @@ class CommonObjectControllerMixin(BaseObjectControllerMixin):
|
||||||
|
|
||||||
self.assertEqual(resp.status_int, 204)
|
self.assertEqual(resp.status_int, 204)
|
||||||
|
|
||||||
def test_DELETE_write_affinity_after_replication(self):
|
def test_DELETE_write_affinity_before_replication(self):
|
||||||
policy_conf = self.app.get_policy_options(self.policy)
|
policy_conf = self.app.get_policy_options(self.policy)
|
||||||
policy_conf.write_affinity_handoff_delete_count = self.replicas() // 2
|
policy_conf.write_affinity_handoff_delete_count = self.replicas() // 2
|
||||||
policy_conf.write_affinity_is_local_fn = (
|
policy_conf.write_affinity_is_local_fn = (
|
||||||
|
@ -1416,6 +1416,36 @@ class TestReplicatedObjController(CommonObjectControllerMixin,
|
||||||
self.assertIn(req.swift_entity_path.decode('utf-8'), log_lines[0])
|
self.assertIn(req.swift_entity_path.decode('utf-8'), log_lines[0])
|
||||||
self.assertIn('ERROR with Object server', log_lines[0])
|
self.assertIn('ERROR with Object server', log_lines[0])
|
||||||
|
|
||||||
|
def test_DELETE_with_write_affinity(self):
|
||||||
|
policy_conf = self.app.get_policy_options(self.policy)
|
||||||
|
policy_conf.write_affinity_handoff_delete_count = self.replicas() // 2
|
||||||
|
policy_conf.write_affinity_is_local_fn = (
|
||||||
|
lambda node: node['region'] == 1)
|
||||||
|
|
||||||
|
req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE')
|
||||||
|
|
||||||
|
codes = [204, 204, 404, 204]
|
||||||
|
with mocked_http_conn(*codes):
|
||||||
|
resp = req.get_response(self.app)
|
||||||
|
self.assertEqual(resp.status_int, 204)
|
||||||
|
|
||||||
|
codes = [204, 404, 404, 204]
|
||||||
|
with mocked_http_conn(*codes):
|
||||||
|
resp = req.get_response(self.app)
|
||||||
|
self.assertEqual(resp.status_int, 204)
|
||||||
|
|
||||||
|
policy_conf.write_affinity_handoff_delete_count = 2
|
||||||
|
|
||||||
|
codes = [204, 204, 404, 204, 404]
|
||||||
|
with mocked_http_conn(*codes):
|
||||||
|
resp = req.get_response(self.app)
|
||||||
|
self.assertEqual(resp.status_int, 204)
|
||||||
|
|
||||||
|
codes = [204, 404, 404, 204, 204]
|
||||||
|
with mocked_http_conn(*codes):
|
||||||
|
resp = req.get_response(self.app)
|
||||||
|
self.assertEqual(resp.status_int, 204)
|
||||||
|
|
||||||
def test_PUT_error_during_transfer_data(self):
|
def test_PUT_error_during_transfer_data(self):
|
||||||
class FakeReader(object):
|
class FakeReader(object):
|
||||||
def read(self, size):
|
def read(self, size):
|
||||||
|
@ -1816,15 +1846,39 @@ class TestReplicatedObjController(CommonObjectControllerMixin,
|
||||||
|
|
||||||
@patch_policies(
|
@patch_policies(
|
||||||
[StoragePolicy(0, '1-replica', True),
|
[StoragePolicy(0, '1-replica', True),
|
||||||
StoragePolicy(1, '5-replica', False),
|
StoragePolicy(1, '4-replica', False),
|
||||||
StoragePolicy(2, '8-replica', False),
|
StoragePolicy(2, '8-replica', False),
|
||||||
StoragePolicy(3, '15-replica', False)],
|
StoragePolicy(3, '15-replica', False)],
|
||||||
fake_ring_args=[
|
fake_ring_args=[
|
||||||
{'replicas': 1}, {'replicas': 5}, {'replicas': 8}, {'replicas': 15}])
|
{'replicas': 1}, {'replicas': 4}, {'replicas': 8}, {'replicas': 15}])
|
||||||
class TestReplicatedObjControllerVariousReplicas(CommonObjectControllerMixin,
|
class TestReplicatedObjControllerVariousReplicas(CommonObjectControllerMixin,
|
||||||
unittest.TestCase):
|
unittest.TestCase):
|
||||||
controller_cls = obj.ReplicatedObjectController
|
controller_cls = obj.ReplicatedObjectController
|
||||||
|
|
||||||
|
def test_DELETE_with_write_affinity(self):
|
||||||
|
policy_index = 1
|
||||||
|
self.policy = POLICIES[policy_index]
|
||||||
|
policy_conf = self.app.get_policy_options(self.policy)
|
||||||
|
self.app.container_info['storage_policy'] = policy_index
|
||||||
|
policy_conf.write_affinity_handoff_delete_count = \
|
||||||
|
self.replicas(self.policy) // 2
|
||||||
|
policy_conf.write_affinity_is_local_fn = (
|
||||||
|
lambda node: node['region'] == 1)
|
||||||
|
|
||||||
|
req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE')
|
||||||
|
|
||||||
|
codes = [204, 204, 404, 404, 204, 204]
|
||||||
|
with mocked_http_conn(*codes):
|
||||||
|
resp = req.get_response(self.app)
|
||||||
|
self.assertEqual(resp.status_int, 204)
|
||||||
|
|
||||||
|
policy_conf.write_affinity_handoff_delete_count = 1
|
||||||
|
|
||||||
|
codes = [204, 204, 404, 404, 204]
|
||||||
|
with mocked_http_conn(*codes):
|
||||||
|
resp = req.get_response(self.app)
|
||||||
|
self.assertEqual(resp.status_int, 204)
|
||||||
|
|
||||||
|
|
||||||
@patch_policies()
|
@patch_policies()
|
||||||
class TestReplicatedObjControllerMimePutter(BaseObjectControllerMixin,
|
class TestReplicatedObjControllerMimePutter(BaseObjectControllerMixin,
|
||||||
|
|
|
@ -71,7 +71,7 @@ from swift.common import utils, constraints
|
||||||
from swift.common.utils import hash_path, storage_directory, \
|
from swift.common.utils import hash_path, storage_directory, \
|
||||||
parse_content_type, parse_mime_headers, \
|
parse_content_type, parse_mime_headers, \
|
||||||
iter_multipart_mime_documents, public, mkdirs, NullLogger
|
iter_multipart_mime_documents, public, mkdirs, NullLogger
|
||||||
from swift.common.wsgi import monkey_patch_mimetools, loadapp, ConfigString
|
from swift.common.wsgi import loadapp, ConfigString
|
||||||
from swift.proxy.controllers import base as proxy_base
|
from swift.proxy.controllers import base as proxy_base
|
||||||
from swift.proxy.controllers.base import get_cache_key, cors_validation, \
|
from swift.proxy.controllers.base import get_cache_key, cors_validation, \
|
||||||
get_account_info, get_container_info
|
get_account_info, get_container_info
|
||||||
|
@ -97,7 +97,6 @@ def do_setup(object_server):
|
||||||
# setup test context and break out some globals for convenience
|
# setup test context and break out some globals for convenience
|
||||||
global _test_context, _testdir, _test_servers, _test_sockets, \
|
global _test_context, _testdir, _test_servers, _test_sockets, \
|
||||||
_test_POLICIES
|
_test_POLICIES
|
||||||
monkey_patch_mimetools()
|
|
||||||
_test_context = setup_servers(object_server)
|
_test_context = setup_servers(object_server)
|
||||||
_testdir = _test_context["testdir"]
|
_testdir = _test_context["testdir"]
|
||||||
_test_servers = _test_context["test_servers"]
|
_test_servers = _test_context["test_servers"]
|
||||||
|
@ -3269,37 +3268,35 @@ class TestReplicatedObjectController(
|
||||||
self.assertNotEqual(last_modified_put, last_modified_head)
|
self.assertNotEqual(last_modified_put, last_modified_head)
|
||||||
_do_conditional_GET_checks(last_modified_head)
|
_do_conditional_GET_checks(last_modified_head)
|
||||||
|
|
||||||
|
@unpatch_policies
|
||||||
def test_PUT_auto_content_type(self):
|
def test_PUT_auto_content_type(self):
|
||||||
with save_globals():
|
prolis = _test_sockets[0]
|
||||||
controller = ReplicatedObjectController(
|
|
||||||
self.app, 'account', 'container', 'object')
|
|
||||||
|
|
||||||
def test_content_type(filename, expected):
|
def do_test(ext, content_type):
|
||||||
# The three responses here are for account_info() (HEAD to
|
sock = connect_tcp(('localhost', prolis.getsockname()[1]))
|
||||||
# account server), container_info() (HEAD to container server)
|
fd = sock.makefile('rwb')
|
||||||
# and three calls to _connect_put_node() (PUT to three object
|
fd.write(b'PUT /v1/a/c/o.%s HTTP/1.1\r\n'
|
||||||
# servers)
|
b'Host: localhost\r\n'
|
||||||
set_http_connect(201, 201, 201, 201, 201,
|
b'X-Storage-Token: t\r\nContent-Length: 0\r\n\r\n' %
|
||||||
give_content_type=lambda content_type:
|
ext.encode())
|
||||||
self.assertEqual(content_type,
|
fd.flush()
|
||||||
next(expected)))
|
headers = readuntil2crlfs(fd)
|
||||||
# We need into include a transfer-encoding to get past
|
exp = b'HTTP/1.1 201'
|
||||||
# constraints.check_object_creation()
|
self.assertEqual(headers[:len(exp)], exp)
|
||||||
req = Request.blank('/v1/a/c/%s' % filename, {},
|
|
||||||
headers={'transfer-encoding': 'chunked'})
|
|
||||||
self.app.update_request(req)
|
|
||||||
self.app.memcache.store = {}
|
|
||||||
res = controller.PUT(req)
|
|
||||||
# If we don't check the response here we could miss problems
|
|
||||||
# in PUT()
|
|
||||||
self.assertEqual(res.status_int, 201)
|
|
||||||
|
|
||||||
test_content_type('test.jpg', iter(['', '', 'image/jpeg',
|
fd.write(b'GET /v1/a/c/o.%s HTTP/1.1\r\n'
|
||||||
'image/jpeg', 'image/jpeg']))
|
b'Host: localhost\r\nConnection: close\r\n'
|
||||||
test_content_type('test.html', iter(['', '', 'text/html',
|
b'X-Storage-Token: t\r\n\r\n' % ext.encode())
|
||||||
'text/html', 'text/html']))
|
fd.flush()
|
||||||
test_content_type('test.css', iter(['', '', 'text/css',
|
headers = readuntil2crlfs(fd)
|
||||||
'text/css', 'text/css']))
|
exp = b'HTTP/1.1 200'
|
||||||
|
self.assertIn(b'Content-Type: %s' % content_type.encode(),
|
||||||
|
headers.split(b'\r\n'))
|
||||||
|
sock.close()
|
||||||
|
|
||||||
|
do_test('jpg', 'image/jpeg')
|
||||||
|
do_test('html', 'text/html')
|
||||||
|
do_test('css', 'text/css')
|
||||||
|
|
||||||
def test_custom_mime_types_files(self):
|
def test_custom_mime_types_files(self):
|
||||||
swift_dir = mkdtemp()
|
swift_dir = mkdtemp()
|
||||||
|
|
|
@ -24,7 +24,7 @@ from swift.common.middleware.copy import ServerSideCopyMiddleware
|
||||||
from swift.common.storage_policy import StoragePolicy
|
from swift.common.storage_policy import StoragePolicy
|
||||||
from swift.common.swob import Request
|
from swift.common.swob import Request
|
||||||
from swift.common.utils import mkdirs, split_path
|
from swift.common.utils import mkdirs, split_path
|
||||||
from swift.common.wsgi import monkey_patch_mimetools, WSGIContext
|
from swift.common.wsgi import WSGIContext
|
||||||
from swift.obj import server as object_server
|
from swift.obj import server as object_server
|
||||||
from swift.proxy import server as proxy
|
from swift.proxy import server as proxy
|
||||||
import swift.proxy.controllers
|
import swift.proxy.controllers
|
||||||
|
@ -138,7 +138,6 @@ class TestObjectSysmeta(unittest.TestCase):
|
||||||
account_ring=FakeRing(replicas=1),
|
account_ring=FakeRing(replicas=1),
|
||||||
container_ring=FakeRing(replicas=1))
|
container_ring=FakeRing(replicas=1))
|
||||||
self.copy_app = ServerSideCopyMiddleware(self.app, {})
|
self.copy_app = ServerSideCopyMiddleware(self.app, {})
|
||||||
monkey_patch_mimetools()
|
|
||||||
self.tmpdir = mkdtemp()
|
self.tmpdir = mkdtemp()
|
||||||
self.testdir = os.path.join(self.tmpdir,
|
self.testdir = os.path.join(self.tmpdir,
|
||||||
'tmp_test_object_server_ObjectController')
|
'tmp_test_object_server_ObjectController')
|
||||||
|
|
4
tox.ini
4
tox.ini
|
@ -45,6 +45,7 @@ commands =
|
||||||
test/unit/common/middleware/test_copy.py \
|
test/unit/common/middleware/test_copy.py \
|
||||||
test/unit/common/middleware/test_crossdomain.py \
|
test/unit/common/middleware/test_crossdomain.py \
|
||||||
test/unit/common/middleware/test_domain_remap.py \
|
test/unit/common/middleware/test_domain_remap.py \
|
||||||
|
test/unit/common/middleware/test_formpost.py \
|
||||||
test/unit/common/middleware/test_gatekeeper.py \
|
test/unit/common/middleware/test_gatekeeper.py \
|
||||||
test/unit/common/middleware/test_healthcheck.py \
|
test/unit/common/middleware/test_healthcheck.py \
|
||||||
test/unit/common/middleware/test_keystoneauth.py \
|
test/unit/common/middleware/test_keystoneauth.py \
|
||||||
|
@ -96,6 +97,9 @@ commands =
|
||||||
[testenv:py36]
|
[testenv:py36]
|
||||||
commands = {[testenv:py35]commands}
|
commands = {[testenv:py35]commands}
|
||||||
|
|
||||||
|
[testenv:py37]
|
||||||
|
commands = {[testenv:py35]commands}
|
||||||
|
|
||||||
[testenv:pep8]
|
[testenv:pep8]
|
||||||
basepython = python2.7
|
basepython = python2.7
|
||||||
commands =
|
commands =
|
||||||
|
|
Loading…
Reference in New Issue