openstack
/
swift
Code Issues Proposed changes

Merge master into feature/mpu 

Change-Id: I5e17a9f6e06c7a021c86c73cb1534702505cd6a8
This commit is contained in:
Alistair Coles 2024-04-19 13:03:48 +01:00
parent f0e1713c8e 6387949a54
commit ece8c28ce1
16 changed files with 116 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.zuul.yaml
View File

@ -348,16 +348,16 @@
- tools/playbooks/probetests/post.yaml
- job:
name: swift-probetests-centos-8-stream
name: swift-probetests-centos-9-stream
parent: unittests
nodeset: centos-8-stream
nodeset: centos-9-stream
description: |
Setup a SAIO dev environment and run Swift's probe tests
under Python 3.
timeout: 7200
vars:
s3_acl: no
bindep_profile: test py36
bindep_profile: test py39
pre-run:
- tools/playbooks/common/install_dependencies.yaml
- tools/playbooks/saio_single_node_setup/setup_saio.yaml
@ -367,12 +367,12 @@
post-run: tools/playbooks/probetests/post.yaml
- job:
name: swift-probetests-centos-8-stream-arm64
parent: swift-probetests-centos-8-stream
name: swift-probetests-centos-9-stream-arm64
parent: swift-probetests-centos-9-stream
nodeset:
nodes:
- name: swift-centos-8-stream-arm64
label: centos-8-stream-arm64
- name: swift-centos-9-stream-arm64
label: centos-9-stream-arm64
description: |
Setup a SAIO dev environment and run Swift's probe tests
under Python 3 on top of arm64 architecture.
@ -380,7 +380,7 @@
- job:
name: swift-func-cors
parent: swift-probetests-centos-8-stream
parent: swift-probetests-centos-9-stream
description: |
Setup a SAIO dev environment and run Swift's CORS functional tests
timeout: 1200
@ -713,7 +713,7 @@
- ^doc/(requirements.txt|(manpages|s3api|source)/.*)$
- ^test/(cors|unit|functional|probe)/.*$
- ^(.gitreview|.mailmap|AUTHORS|CHANGELOG|.*\.rst)$
- swift-probetests-centos-8-stream:
- swift-probetests-centos-9-stream:
irrelevant-files: &probetest-irrelevant-files
- ^(api-ref|releasenotes)/.*$
# Keep doc/saio -- we use those sample configs in the saio playbooks
@ -769,7 +769,7 @@
- swift-tox-func-ec-py38
- swift-func-cors
- swift-tox-func-s3api-tests-tempauth
- swift-probetests-centos-8-stream:
- swift-probetests-centos-9-stream:
irrelevant-files: *probetest-irrelevant-files
- swift-dsvm-functional:
irrelevant-files: *functest-irrelevant-files

2
lower-constraints.txt
View File

@ -36,7 +36,7 @@ keystonemiddleware==4.17.0
linecache2==1.0.0
lxml==3.4.1
MarkupSafe==1.0
mock==2.0
mock==3.0
monotonic==1.4
msgpack==0.5.6
netaddr==0.7.19

6
releasenotes/source/2024.1.rst Normal file
View File

@ -0,0 +1,6 @@
===========================
2024.1 Series Release Notes
===========================
.. release-notes::
:branch: stable/2024.1

1
releasenotes/source/index.rst
View File

@ -6,6 +6,7 @@
:maxdepth: 1
current
2024.1
2023.2
2023.1
zed

10
swift/cli/recon_cron.py
View File

@ -18,18 +18,18 @@ import time
from eventlet import Timeout
from swift.common.utils import get_logger, dump_recon_cache, readconf, \
lock_path
lock_path, listdir
from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
from swift.obj.diskfile import ASYNCDIR_BASE
def get_async_count(device_dir):
async_count = 0
for i in os.listdir(device_dir):
for i in listdir(device_dir):
device = os.path.join(device_dir, i)
if not os.path.isdir(device):
continue
for asyncdir in os.listdir(device):
for asyncdir in listdir(device):
# skip stuff like "accounts", "containers", etc.
if not (asyncdir == ASYNCDIR_BASE or
asyncdir.startswith(ASYNCDIR_BASE + '-')):
@ -37,10 +37,10 @@ def get_async_count(device_dir):
async_pending = os.path.join(device, asyncdir)
if os.path.isdir(async_pending):
for entry in os.listdir(async_pending):
for entry in listdir(async_pending):
if os.path.isdir(os.path.join(async_pending, entry)):
async_hdir = os.path.join(async_pending, entry)
async_count += len(os.listdir(async_hdir))
async_count += len(listdir(async_hdir))
return async_count

2
swift/common/middleware/s3api/s3request.py
View File

@ -1142,7 +1142,7 @@ class S3Request(swob.Request):
Create a Swift request based on this request's environment.
"""
if self.account is None:
account = self.access_key
account = swob.str_to_wsgi(self.access_key)
else:
account = self.account

4
swift/common/middleware/s3api/s3token.py
View File

@ -65,7 +65,7 @@ import six
from six.moves import urllib
from swift.common.swob import Request, HTTPBadRequest, HTTPUnauthorized, \
HTTPException
HTTPException, str_to_wsgi
from swift.common.utils import config_true_value, split_path, get_logger, \
cache_from_env, append_underscore
from swift.common.wsgi import ConfigFileError
@ -404,7 +404,7 @@ class S3Token(object):
self._logger.debug('Connecting with tenant: %s', tenant_to_connect)
new_tenant_name = '%s%s' % (self._reseller_prefix, tenant_to_connect)
environ['PATH_INFO'] = environ['PATH_INFO'].replace(
account, new_tenant_name, 1)
str_to_wsgi(account), str_to_wsgi(new_tenant_name), 1)
return self._app(environ, start_response)

10
swift/common/middleware/tempauth.py
View File

@ -184,9 +184,11 @@ import base64
from eventlet import Timeout
import six
from swift.common.memcached import MemcacheConnectionError
from swift.common.swob import Response, Request, wsgi_to_str
from swift.common.swob import HTTPBadRequest, HTTPForbidden, HTTPNotFound, \
HTTPUnauthorized, HTTPMethodNotAllowed, HTTPServiceUnavailable
from swift.common.swob import (
Response, Request, wsgi_to_str, str_to_wsgi, wsgi_unquote,
HTTPBadRequest, HTTPForbidden, HTTPNotFound,
HTTPUnauthorized, HTTPMethodNotAllowed, HTTPServiceUnavailable,
)
from swift.common.request_helpers import get_sys_meta_prefix
from swift.common.middleware.acl import (
@ -469,7 +471,7 @@ class TempAuth(object):
if not s3_auth_details['check_signature'](user['key']):
return None
env['PATH_INFO'] = env['PATH_INFO'].replace(
account_user, account_id, 1)
str_to_wsgi(account_user), wsgi_unquote(account_id), 1)
groups = self._get_user_groups(account, account_user, account_id)
return groups

2
test-requirements.txt
View File

@ -8,7 +8,7 @@ coverage>=5.0.4 # Apache-2.0
pytest>=4.6.11 # MIT
pytest-cov>=2.12.1 # MIT
stestr>=2.0.0 # Apache-2.0
mock>=2.0 # BSD
mock>=3.0 # BSD
python-swiftclient>=3.2.0
python-keystoneclient!=2.1.0,>=2.0.0 # Apache-2.0
boto>=2.32.1

8
test/cors/main.py
View File

@ -194,8 +194,12 @@ def run(args, url):
try:
browser = driver(**kwargs)
except Exception as e:
if not ('needs to be in PATH' in str(e) or
'SafariDriver was not found' in str(e)):
if not any(x in str(e) for x in (
'needs to be in PATH',
'SafariDriver was not found',
'OSError: [Errno 8] Exec format error',
'safaridriver not available for download',
)):
traceback.print_exc()
results.append(('SKIP', browser_name, str(e).strip()))
continue

30
test/unit/cli/test_recon_cron.py
View File

@ -16,6 +16,7 @@
import tempfile
import shutil
import os
import mock
from unittest import TestCase
from swift.cli.recon_cron import get_async_count
@ -48,3 +49,32 @@ class TestReconCron(TestCase):
count = get_async_count(device_dir)
self.assertEqual(count, 3)
def test_get_async_count_deleted(self):
device_dir = os.path.join(self.temp_dir, 'device')
device_index = os.path.join(device_dir, '1')
async_dir = os.path.join(device_index, ASYNCDIR_BASE)
entry1 = os.path.join(async_dir, 'entry1')
entry2 = os.path.join(async_dir, 'entry2')
os.makedirs(entry1)
os.makedirs(entry2)
pending_file1 = os.path.join(entry1, 'pending_file1')
pending_file2 = os.path.join(entry1, 'pending_file2')
pending_file3 = os.path.join(entry2, 'pending_file3')
open(pending_file1, 'w').close()
open(pending_file2, 'w').close()
open(pending_file3, 'w').close()
orig_isdir = os.path.isdir
def racy_isdir(d):
result = orig_isdir(d)
if d == entry1:
# clean it up before caller has a chance to descend
shutil.rmtree(entry1)
return result
with mock.patch('os.path.isdir', racy_isdir):
count = get_async_count(device_dir)
self.assertEqual(count, 1)

4
test/unit/common/middleware/s3api/__init__.py
View File

@ -46,10 +46,12 @@ class FakeAuthApp(object):
E.g. '/v1/test:tester/bucket/object' will become
'/v1/AUTH_test/bucket/object'. This method emulates the behavior.
"""
tenant_user = env['s3api.auth_details']['access_key']
tenant_user = swob.str_to_wsgi(env['s3api.auth_details']['access_key'])
tenant, user = tenant_user.rsplit(':', 1)
path = env['PATH_INFO']
# Make sure it's valid WSGI
swob.wsgi_to_str(path)
env['PATH_INFO'] = path.replace(tenant_user, 'AUTH_' + tenant)
@staticmethod

17
test/unit/common/middleware/s3api/test_s3token.py
View File

@ -199,7 +199,8 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
self.middleware(req.environ, self.start_fake_response)
self.assertEqual(self.response_status, 200)
def _assert_authorized(self, req, account_path='/v1/AUTH_TENANT_ID/'):
def _assert_authorized(self, req, account_path='/v1/AUTH_TENANT_ID/',
access_key='access'):
self.assertTrue(
req.path.startswith(account_path),
'%r does not start with %r' % (req.path, account_path))
@ -224,7 +225,7 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
self.assertEqual(1, self.requests_mock.call_count)
request_call = self.requests_mock.request_history[0]
self.assertEqual(json.loads(request_call.body), {'credentials': {
'access': 'access',
'access': access_key,
'signature': 'signature',
'token': base64.urlsafe_b64encode(b'token').decode('ascii')}})
@ -501,6 +502,18 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
self._assert_authorized(req, account_path='/v1/')
self.assertEqual(req.environ['PATH_INFO'], '/v1/AUTH_TENANT_ID/c/o')
def test_authorize_with_unicode_access_key(self):
req = Request.blank('/v1/acc\xc3\xa9sskey/c/o')
req.environ['s3api.auth_details'] = {
'access_key': u'acc\u00e9ss',
'signature': u'signature',
'string_to_sign': u'token',
}
req.get_response(self.middleware)
self._assert_authorized(req, account_path='/v1/',
access_key=u'acc\u00e9ss')
self.assertEqual(req.environ['PATH_INFO'], '/v1/AUTH_TENANT_ID/c/o')
def test_authorize_with_access_key_and_unquote_chars(self):
req = Request.blank('/v1/access%key=/c/o')
req.environ['s3api.auth_details'] = {

23
test/unit/common/middleware/test_tempauth.py
View File

@ -308,6 +308,29 @@ class TestAuth(unittest.TestCase):
self.assertEqual(req.environ['swift.authorize'],
local_auth.authorize)
def test_auth_with_s3api_unicode_authorization_good(self):
local_app = FakeApp()
conf = {u'user_t\u00e9st_t\u00e9ster': u'p\u00e1ss .admin'}
access_key = u't\u00e9st:t\u00e9ster'
if six.PY2:
conf = {k.encode('utf8'): v.encode('utf8')
for k, v in conf.items()}
access_key = access_key.encode('utf8')
local_auth = auth.filter_factory(conf)(local_app)
req = self._make_request('/v1/t\xc3\xa9st:t\xc3\xa9ster', environ={
's3api.auth_details': {
'access_key': access_key,
'signature': b64encode('sig'),
'string_to_sign': 't',
'check_signature': lambda secret: True}})
resp = req.get_response(local_auth)
self.assertEqual(resp.status_int, 404)
self.assertEqual(local_app.calls, 1)
self.assertEqual(req.environ['PATH_INFO'], '/v1/AUTH_t\xc3\xa9st')
self.assertEqual(req.environ['swift.authorize'],
local_auth.authorize)
def test_auth_with_swift3_authorization_invalid(self):
local_app = FakeApp()
local_auth = auth.filter_factory(

4
tools/playbooks/common/install_dependencies.yaml
View File

@ -22,13 +22,13 @@
name: pip
extra_args: --upgrade
- name: install rsync-daemon - CentOS 8
- name: install rsync-daemon - CentOS 8, 9
package:
name: rsync-daemon
state: present
when:
- ansible_facts['distribution'] == "CentOS"
- ansible_facts['distribution_major_version'] == "8"
- ansible_facts['distribution_major_version'] != "7"
- name: install python modules with pip
pip: name={{ item }} state=present extra_args='--upgrade'

6
tools/playbooks/cors/install_selenium.yaml
View File

@ -7,7 +7,7 @@
state: present
- name: install selenium
pip:
name: selenium
name: 'selenium<4'
state: present
- name: install firefox
yum:
@ -15,13 +15,15 @@
state: present
- name: fetch firefox driver
get_url:
url: https://github.com/mozilla/geckodriver/releases/download/v0.26.0/geckodriver-v0.26.0-linux64.tar.gz
url: https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz
dest: /tmp/geckodriver.tar.gz
- name: unpack firefox driver
unarchive:
src: /tmp/geckodriver.tar.gz
dest: /usr/local/bin
remote_src: true
- name: check selenium version
command: pip show selenium
- name: check firefox version
command: firefox --version
#- name: install chromium