diff --git a/.zuul.yaml b/.zuul.yaml index 46107ee132..56975479b0 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -348,16 +348,16 @@ - tools/playbooks/probetests/post.yaml - job: - name: swift-probetests-centos-8-stream + name: swift-probetests-centos-9-stream parent: unittests - nodeset: centos-8-stream + nodeset: centos-9-stream description: | Setup a SAIO dev environment and run Swift's probe tests under Python 3. timeout: 7200 vars: s3_acl: no - bindep_profile: test py36 + bindep_profile: test py39 pre-run: - tools/playbooks/common/install_dependencies.yaml - tools/playbooks/saio_single_node_setup/setup_saio.yaml @@ -367,12 +367,12 @@ post-run: tools/playbooks/probetests/post.yaml - job: - name: swift-probetests-centos-8-stream-arm64 - parent: swift-probetests-centos-8-stream + name: swift-probetests-centos-9-stream-arm64 + parent: swift-probetests-centos-9-stream nodeset: nodes: - - name: swift-centos-8-stream-arm64 - label: centos-8-stream-arm64 + - name: swift-centos-9-stream-arm64 + label: centos-9-stream-arm64 description: | Setup a SAIO dev environment and run Swift's probe tests under Python 3 on top of arm64 architecture. @@ -380,7 +380,7 @@ - job: name: swift-func-cors - parent: swift-probetests-centos-8-stream + parent: swift-probetests-centos-9-stream description: | Setup a SAIO dev environment and run Swift's CORS functional tests timeout: 1200 @@ -713,7 +713,7 @@ - ^doc/(requirements.txt|(manpages|s3api|source)/.*)$ - ^test/(cors|unit|functional|probe)/.*$ - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG|.*\.rst)$ - - swift-probetests-centos-8-stream: + - swift-probetests-centos-9-stream: irrelevant-files: &probetest-irrelevant-files - ^(api-ref|releasenotes)/.*$ # Keep doc/saio -- we use those sample configs in the saio playbooks @@ -769,7 +769,7 @@ - swift-tox-func-ec-py38 - swift-func-cors - swift-tox-func-s3api-tests-tempauth - - swift-probetests-centos-8-stream: + - swift-probetests-centos-9-stream: irrelevant-files: *probetest-irrelevant-files - swift-dsvm-functional: irrelevant-files: *functest-irrelevant-files diff --git a/lower-constraints.txt b/lower-constraints.txt index 27085dc417..6706c9a347 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -36,7 +36,7 @@ keystonemiddleware==4.17.0 linecache2==1.0.0 lxml==3.4.1 MarkupSafe==1.0 -mock==2.0 +mock==3.0 monotonic==1.4 msgpack==0.5.6 netaddr==0.7.19 diff --git a/releasenotes/source/2024.1.rst b/releasenotes/source/2024.1.rst new file mode 100644 index 0000000000..4977a4f1a0 --- /dev/null +++ b/releasenotes/source/2024.1.rst @@ -0,0 +1,6 @@ +=========================== +2024.1 Series Release Notes +=========================== + +.. release-notes:: + :branch: stable/2024.1 diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index f0c0ae812d..e069fdc527 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,7 @@ :maxdepth: 1 current + 2024.1 2023.2 2023.1 zed diff --git a/swift/cli/recon_cron.py b/swift/cli/recon_cron.py index d2ba80361a..bd1dd22052 100644 --- a/swift/cli/recon_cron.py +++ b/swift/cli/recon_cron.py @@ -18,18 +18,18 @@ import time from eventlet import Timeout from swift.common.utils import get_logger, dump_recon_cache, readconf, \ - lock_path + lock_path, listdir from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH from swift.obj.diskfile import ASYNCDIR_BASE def get_async_count(device_dir): async_count = 0 - for i in os.listdir(device_dir): + for i in listdir(device_dir): device = os.path.join(device_dir, i) if not os.path.isdir(device): continue - for asyncdir in os.listdir(device): + for asyncdir in listdir(device): # skip stuff like "accounts", "containers", etc. if not (asyncdir == ASYNCDIR_BASE or asyncdir.startswith(ASYNCDIR_BASE + '-')): @@ -37,10 +37,10 @@ def get_async_count(device_dir): async_pending = os.path.join(device, asyncdir) if os.path.isdir(async_pending): - for entry in os.listdir(async_pending): + for entry in listdir(async_pending): if os.path.isdir(os.path.join(async_pending, entry)): async_hdir = os.path.join(async_pending, entry) - async_count += len(os.listdir(async_hdir)) + async_count += len(listdir(async_hdir)) return async_count diff --git a/swift/common/middleware/s3api/s3request.py b/swift/common/middleware/s3api/s3request.py index dcc0fb361b..9050745a39 100644 --- a/swift/common/middleware/s3api/s3request.py +++ b/swift/common/middleware/s3api/s3request.py @@ -1142,7 +1142,7 @@ class S3Request(swob.Request): Create a Swift request based on this request's environment. """ if self.account is None: - account = self.access_key + account = swob.str_to_wsgi(self.access_key) else: account = self.account diff --git a/swift/common/middleware/s3api/s3token.py b/swift/common/middleware/s3api/s3token.py index 3693b2f28d..23ebcc7086 100644 --- a/swift/common/middleware/s3api/s3token.py +++ b/swift/common/middleware/s3api/s3token.py @@ -65,7 +65,7 @@ import six from six.moves import urllib from swift.common.swob import Request, HTTPBadRequest, HTTPUnauthorized, \ - HTTPException + HTTPException, str_to_wsgi from swift.common.utils import config_true_value, split_path, get_logger, \ cache_from_env, append_underscore from swift.common.wsgi import ConfigFileError @@ -404,7 +404,7 @@ class S3Token(object): self._logger.debug('Connecting with tenant: %s', tenant_to_connect) new_tenant_name = '%s%s' % (self._reseller_prefix, tenant_to_connect) environ['PATH_INFO'] = environ['PATH_INFO'].replace( - account, new_tenant_name, 1) + str_to_wsgi(account), str_to_wsgi(new_tenant_name), 1) return self._app(environ, start_response) diff --git a/swift/common/middleware/tempauth.py b/swift/common/middleware/tempauth.py index 49541329b1..52d6659a72 100644 --- a/swift/common/middleware/tempauth.py +++ b/swift/common/middleware/tempauth.py @@ -184,9 +184,11 @@ import base64 from eventlet import Timeout import six from swift.common.memcached import MemcacheConnectionError -from swift.common.swob import Response, Request, wsgi_to_str -from swift.common.swob import HTTPBadRequest, HTTPForbidden, HTTPNotFound, \ - HTTPUnauthorized, HTTPMethodNotAllowed, HTTPServiceUnavailable +from swift.common.swob import ( + Response, Request, wsgi_to_str, str_to_wsgi, wsgi_unquote, + HTTPBadRequest, HTTPForbidden, HTTPNotFound, + HTTPUnauthorized, HTTPMethodNotAllowed, HTTPServiceUnavailable, +) from swift.common.request_helpers import get_sys_meta_prefix from swift.common.middleware.acl import ( @@ -469,7 +471,7 @@ class TempAuth(object): if not s3_auth_details['check_signature'](user['key']): return None env['PATH_INFO'] = env['PATH_INFO'].replace( - account_user, account_id, 1) + str_to_wsgi(account_user), wsgi_unquote(account_id), 1) groups = self._get_user_groups(account, account_user, account_id) return groups diff --git a/test-requirements.txt b/test-requirements.txt index 1e77045c9d..b5cf150d6e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ coverage>=5.0.4 # Apache-2.0 pytest>=4.6.11 # MIT pytest-cov>=2.12.1 # MIT stestr>=2.0.0 # Apache-2.0 -mock>=2.0 # BSD +mock>=3.0 # BSD python-swiftclient>=3.2.0 python-keystoneclient!=2.1.0,>=2.0.0 # Apache-2.0 boto>=2.32.1 diff --git a/test/cors/main.py b/test/cors/main.py index 5da06ae681..063f967247 100755 --- a/test/cors/main.py +++ b/test/cors/main.py @@ -194,8 +194,12 @@ def run(args, url): try: browser = driver(**kwargs) except Exception as e: - if not ('needs to be in PATH' in str(e) or - 'SafariDriver was not found' in str(e)): + if not any(x in str(e) for x in ( + 'needs to be in PATH', + 'SafariDriver was not found', + 'OSError: [Errno 8] Exec format error', + 'safaridriver not available for download', + )): traceback.print_exc() results.append(('SKIP', browser_name, str(e).strip())) continue diff --git a/test/unit/cli/test_recon_cron.py b/test/unit/cli/test_recon_cron.py index 8e53d1e0b0..b377038a6b 100644 --- a/test/unit/cli/test_recon_cron.py +++ b/test/unit/cli/test_recon_cron.py @@ -16,6 +16,7 @@ import tempfile import shutil import os +import mock from unittest import TestCase from swift.cli.recon_cron import get_async_count @@ -48,3 +49,32 @@ class TestReconCron(TestCase): count = get_async_count(device_dir) self.assertEqual(count, 3) + + def test_get_async_count_deleted(self): + device_dir = os.path.join(self.temp_dir, 'device') + device_index = os.path.join(device_dir, '1') + async_dir = os.path.join(device_index, ASYNCDIR_BASE) + entry1 = os.path.join(async_dir, 'entry1') + entry2 = os.path.join(async_dir, 'entry2') + os.makedirs(entry1) + os.makedirs(entry2) + + pending_file1 = os.path.join(entry1, 'pending_file1') + pending_file2 = os.path.join(entry1, 'pending_file2') + pending_file3 = os.path.join(entry2, 'pending_file3') + open(pending_file1, 'w').close() + open(pending_file2, 'w').close() + open(pending_file3, 'w').close() + + orig_isdir = os.path.isdir + + def racy_isdir(d): + result = orig_isdir(d) + if d == entry1: + # clean it up before caller has a chance to descend + shutil.rmtree(entry1) + return result + + with mock.patch('os.path.isdir', racy_isdir): + count = get_async_count(device_dir) + self.assertEqual(count, 1) diff --git a/test/unit/common/middleware/s3api/__init__.py b/test/unit/common/middleware/s3api/__init__.py index d34d7e83bf..3319a455fc 100644 --- a/test/unit/common/middleware/s3api/__init__.py +++ b/test/unit/common/middleware/s3api/__init__.py @@ -46,10 +46,12 @@ class FakeAuthApp(object): E.g. '/v1/test:tester/bucket/object' will become '/v1/AUTH_test/bucket/object'. This method emulates the behavior. """ - tenant_user = env['s3api.auth_details']['access_key'] + tenant_user = swob.str_to_wsgi(env['s3api.auth_details']['access_key']) tenant, user = tenant_user.rsplit(':', 1) path = env['PATH_INFO'] + # Make sure it's valid WSGI + swob.wsgi_to_str(path) env['PATH_INFO'] = path.replace(tenant_user, 'AUTH_' + tenant) @staticmethod diff --git a/test/unit/common/middleware/s3api/test_s3token.py b/test/unit/common/middleware/s3api/test_s3token.py index bef6cf9e7b..6a467bd3a0 100644 --- a/test/unit/common/middleware/s3api/test_s3token.py +++ b/test/unit/common/middleware/s3api/test_s3token.py @@ -199,7 +199,8 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase): self.middleware(req.environ, self.start_fake_response) self.assertEqual(self.response_status, 200) - def _assert_authorized(self, req, account_path='/v1/AUTH_TENANT_ID/'): + def _assert_authorized(self, req, account_path='/v1/AUTH_TENANT_ID/', + access_key='access'): self.assertTrue( req.path.startswith(account_path), '%r does not start with %r' % (req.path, account_path)) @@ -224,7 +225,7 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase): self.assertEqual(1, self.requests_mock.call_count) request_call = self.requests_mock.request_history[0] self.assertEqual(json.loads(request_call.body), {'credentials': { - 'access': 'access', + 'access': access_key, 'signature': 'signature', 'token': base64.urlsafe_b64encode(b'token').decode('ascii')}}) @@ -501,6 +502,18 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase): self._assert_authorized(req, account_path='/v1/') self.assertEqual(req.environ['PATH_INFO'], '/v1/AUTH_TENANT_ID/c/o') + def test_authorize_with_unicode_access_key(self): + req = Request.blank('/v1/acc\xc3\xa9sskey/c/o') + req.environ['s3api.auth_details'] = { + 'access_key': u'acc\u00e9ss', + 'signature': u'signature', + 'string_to_sign': u'token', + } + req.get_response(self.middleware) + self._assert_authorized(req, account_path='/v1/', + access_key=u'acc\u00e9ss') + self.assertEqual(req.environ['PATH_INFO'], '/v1/AUTH_TENANT_ID/c/o') + def test_authorize_with_access_key_and_unquote_chars(self): req = Request.blank('/v1/access%key=/c/o') req.environ['s3api.auth_details'] = { diff --git a/test/unit/common/middleware/test_tempauth.py b/test/unit/common/middleware/test_tempauth.py index a29652fe2c..9d2cb38abc 100644 --- a/test/unit/common/middleware/test_tempauth.py +++ b/test/unit/common/middleware/test_tempauth.py @@ -308,6 +308,29 @@ class TestAuth(unittest.TestCase): self.assertEqual(req.environ['swift.authorize'], local_auth.authorize) + def test_auth_with_s3api_unicode_authorization_good(self): + local_app = FakeApp() + conf = {u'user_t\u00e9st_t\u00e9ster': u'p\u00e1ss .admin'} + access_key = u't\u00e9st:t\u00e9ster' + if six.PY2: + conf = {k.encode('utf8'): v.encode('utf8') + for k, v in conf.items()} + access_key = access_key.encode('utf8') + local_auth = auth.filter_factory(conf)(local_app) + req = self._make_request('/v1/t\xc3\xa9st:t\xc3\xa9ster', environ={ + 's3api.auth_details': { + 'access_key': access_key, + 'signature': b64encode('sig'), + 'string_to_sign': 't', + 'check_signature': lambda secret: True}}) + resp = req.get_response(local_auth) + + self.assertEqual(resp.status_int, 404) + self.assertEqual(local_app.calls, 1) + self.assertEqual(req.environ['PATH_INFO'], '/v1/AUTH_t\xc3\xa9st') + self.assertEqual(req.environ['swift.authorize'], + local_auth.authorize) + def test_auth_with_swift3_authorization_invalid(self): local_app = FakeApp() local_auth = auth.filter_factory( diff --git a/tools/playbooks/common/install_dependencies.yaml b/tools/playbooks/common/install_dependencies.yaml index 355213f64c..bb801ed488 100644 --- a/tools/playbooks/common/install_dependencies.yaml +++ b/tools/playbooks/common/install_dependencies.yaml @@ -22,13 +22,13 @@ name: pip extra_args: --upgrade - - name: install rsync-daemon - CentOS 8 + - name: install rsync-daemon - CentOS 8, 9 package: name: rsync-daemon state: present when: - ansible_facts['distribution'] == "CentOS" - - ansible_facts['distribution_major_version'] == "8" + - ansible_facts['distribution_major_version'] != "7" - name: install python modules with pip pip: name={{ item }} state=present extra_args='--upgrade' diff --git a/tools/playbooks/cors/install_selenium.yaml b/tools/playbooks/cors/install_selenium.yaml index 682c36a875..f617388f36 100644 --- a/tools/playbooks/cors/install_selenium.yaml +++ b/tools/playbooks/cors/install_selenium.yaml @@ -7,7 +7,7 @@ state: present - name: install selenium pip: - name: selenium + name: 'selenium<4' state: present - name: install firefox yum: @@ -15,13 +15,15 @@ state: present - name: fetch firefox driver get_url: - url: https://github.com/mozilla/geckodriver/releases/download/v0.26.0/geckodriver-v0.26.0-linux64.tar.gz + url: https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz dest: /tmp/geckodriver.tar.gz - name: unpack firefox driver unarchive: src: /tmp/geckodriver.tar.gz dest: /usr/local/bin remote_src: true + - name: check selenium version + command: pip show selenium - name: check firefox version command: firefox --version #- name: install chromium