From af15ad53fb7818cdf14c92523fd6799bfabc12ec Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Thu, 14 Mar 2024 20:25:36 -0700 Subject: [PATCH 1/6] tests: Update CORS geckodriver Change-Id: I5ab762dfe0f85e346c4868ec4540884ba5f0a7f4 --- test/cors/main.py | 8 ++++++-- tools/playbooks/cors/install_selenium.yaml | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/test/cors/main.py b/test/cors/main.py index 5da06ae681..063f967247 100755 --- a/test/cors/main.py +++ b/test/cors/main.py @@ -194,8 +194,12 @@ def run(args, url): try: browser = driver(**kwargs) except Exception as e: - if not ('needs to be in PATH' in str(e) or - 'SafariDriver was not found' in str(e)): + if not any(x in str(e) for x in ( + 'needs to be in PATH', + 'SafariDriver was not found', + 'OSError: [Errno 8] Exec format error', + 'safaridriver not available for download', + )): traceback.print_exc() results.append(('SKIP', browser_name, str(e).strip())) continue diff --git a/tools/playbooks/cors/install_selenium.yaml b/tools/playbooks/cors/install_selenium.yaml index 682c36a875..b7d08c57f7 100644 --- a/tools/playbooks/cors/install_selenium.yaml +++ b/tools/playbooks/cors/install_selenium.yaml @@ -15,13 +15,15 @@ state: present - name: fetch firefox driver get_url: - url: https://github.com/mozilla/geckodriver/releases/download/v0.26.0/geckodriver-v0.26.0-linux64.tar.gz + url: https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz dest: /tmp/geckodriver.tar.gz - name: unpack firefox driver unarchive: src: /tmp/geckodriver.tar.gz dest: /usr/local/bin remote_src: true + - name: check selenium version + command: pip show selenium - name: check firefox version command: firefox --version #- name: install chromium From 1c03d586431862c2772cb1b532aefc6cf2e671dc Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 21 Jun 2023 10:21:32 -0700 Subject: [PATCH 2/6] CI: Move probe tests to centos 9 stream Pin selenium to 3.x for now, until we can run down the issues with 4.x Change-Id: I596415d17f77f48a6e8a63a61b734a8ca0865847 --- .zuul.yaml | 20 +++++++++---------- .../common/install_dependencies.yaml | 4 ++-- tools/playbooks/cors/install_selenium.yaml | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 46107ee132..56975479b0 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -348,16 +348,16 @@ - tools/playbooks/probetests/post.yaml - job: - name: swift-probetests-centos-8-stream + name: swift-probetests-centos-9-stream parent: unittests - nodeset: centos-8-stream + nodeset: centos-9-stream description: | Setup a SAIO dev environment and run Swift's probe tests under Python 3. timeout: 7200 vars: s3_acl: no - bindep_profile: test py36 + bindep_profile: test py39 pre-run: - tools/playbooks/common/install_dependencies.yaml - tools/playbooks/saio_single_node_setup/setup_saio.yaml @@ -367,12 +367,12 @@ post-run: tools/playbooks/probetests/post.yaml - job: - name: swift-probetests-centos-8-stream-arm64 - parent: swift-probetests-centos-8-stream + name: swift-probetests-centos-9-stream-arm64 + parent: swift-probetests-centos-9-stream nodeset: nodes: - - name: swift-centos-8-stream-arm64 - label: centos-8-stream-arm64 + - name: swift-centos-9-stream-arm64 + label: centos-9-stream-arm64 description: | Setup a SAIO dev environment and run Swift's probe tests under Python 3 on top of arm64 architecture. @@ -380,7 +380,7 @@ - job: name: swift-func-cors - parent: swift-probetests-centos-8-stream + parent: swift-probetests-centos-9-stream description: | Setup a SAIO dev environment and run Swift's CORS functional tests timeout: 1200 @@ -713,7 +713,7 @@ - ^doc/(requirements.txt|(manpages|s3api|source)/.*)$ - ^test/(cors|unit|functional|probe)/.*$ - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG|.*\.rst)$ - - swift-probetests-centos-8-stream: + - swift-probetests-centos-9-stream: irrelevant-files: &probetest-irrelevant-files - ^(api-ref|releasenotes)/.*$ # Keep doc/saio -- we use those sample configs in the saio playbooks @@ -769,7 +769,7 @@ - swift-tox-func-ec-py38 - swift-func-cors - swift-tox-func-s3api-tests-tempauth - - swift-probetests-centos-8-stream: + - swift-probetests-centos-9-stream: irrelevant-files: *probetest-irrelevant-files - swift-dsvm-functional: irrelevant-files: *functest-irrelevant-files diff --git a/tools/playbooks/common/install_dependencies.yaml b/tools/playbooks/common/install_dependencies.yaml index 355213f64c..bb801ed488 100644 --- a/tools/playbooks/common/install_dependencies.yaml +++ b/tools/playbooks/common/install_dependencies.yaml @@ -22,13 +22,13 @@ name: pip extra_args: --upgrade - - name: install rsync-daemon - CentOS 8 + - name: install rsync-daemon - CentOS 8, 9 package: name: rsync-daemon state: present when: - ansible_facts['distribution'] == "CentOS" - - ansible_facts['distribution_major_version'] == "8" + - ansible_facts['distribution_major_version'] != "7" - name: install python modules with pip pip: name={{ item }} state=present extra_args='--upgrade' diff --git a/tools/playbooks/cors/install_selenium.yaml b/tools/playbooks/cors/install_selenium.yaml index b7d08c57f7..f617388f36 100644 --- a/tools/playbooks/cors/install_selenium.yaml +++ b/tools/playbooks/cors/install_selenium.yaml @@ -7,7 +7,7 @@ state: present - name: install selenium pip: - name: selenium + name: 'selenium<4' state: present - name: install firefox yum: From 5d25108648a5676bf19a45e25ef714f0d4a2f9e1 Mon Sep 17 00:00:00 2001 From: OpenStack Release Bot Date: Fri, 15 Mar 2024 14:52:30 +0000 Subject: [PATCH 3/6] Update master for stable/2024.1 Add file to the reno documentation build to show release notes for stable/2024.1. Use pbr instruction to increment the minor version number automatically so that master versions are higher than the versions on stable/2024.1. Sem-Ver: feature Change-Id: Ic940ff424aef9cc402bf54ebe5e5fc16330fc25c --- releasenotes/source/2024.1.rst | 6 ++++++ releasenotes/source/index.rst | 1 + 2 files changed, 7 insertions(+) create mode 100644 releasenotes/source/2024.1.rst diff --git a/releasenotes/source/2024.1.rst b/releasenotes/source/2024.1.rst new file mode 100644 index 0000000000..4977a4f1a0 --- /dev/null +++ b/releasenotes/source/2024.1.rst @@ -0,0 +1,6 @@ +=========================== +2024.1 Series Release Notes +=========================== + +.. release-notes:: + :branch: stable/2024.1 diff --git a/releasenotes/source/index.rst b/releasenotes/source/index.rst index f0c0ae812d..e069fdc527 100644 --- a/releasenotes/source/index.rst +++ b/releasenotes/source/index.rst @@ -6,6 +6,7 @@ :maxdepth: 1 current + 2024.1 2023.2 2023.1 zed From f31b6f7353f6f8e91a452bf3586032b94b7898c8 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Wed, 20 Mar 2024 12:18:04 -0700 Subject: [PATCH 4/6] recon-cron: Tolerate missing directories Any of these directories may get unlinked between when we saw them in their parent's directory listing and when we go to descend. Change-Id: I1dfc0ee1d9e70cb0600557cde980bd5880bd40b3 --- swift/cli/recon_cron.py | 10 +++++----- test/unit/cli/test_recon_cron.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/swift/cli/recon_cron.py b/swift/cli/recon_cron.py index d2ba80361a..bd1dd22052 100644 --- a/swift/cli/recon_cron.py +++ b/swift/cli/recon_cron.py @@ -18,18 +18,18 @@ import time from eventlet import Timeout from swift.common.utils import get_logger, dump_recon_cache, readconf, \ - lock_path + lock_path, listdir from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH from swift.obj.diskfile import ASYNCDIR_BASE def get_async_count(device_dir): async_count = 0 - for i in os.listdir(device_dir): + for i in listdir(device_dir): device = os.path.join(device_dir, i) if not os.path.isdir(device): continue - for asyncdir in os.listdir(device): + for asyncdir in listdir(device): # skip stuff like "accounts", "containers", etc. if not (asyncdir == ASYNCDIR_BASE or asyncdir.startswith(ASYNCDIR_BASE + '-')): @@ -37,10 +37,10 @@ def get_async_count(device_dir): async_pending = os.path.join(device, asyncdir) if os.path.isdir(async_pending): - for entry in os.listdir(async_pending): + for entry in listdir(async_pending): if os.path.isdir(os.path.join(async_pending, entry)): async_hdir = os.path.join(async_pending, entry) - async_count += len(os.listdir(async_hdir)) + async_count += len(listdir(async_hdir)) return async_count diff --git a/test/unit/cli/test_recon_cron.py b/test/unit/cli/test_recon_cron.py index 8e53d1e0b0..b377038a6b 100644 --- a/test/unit/cli/test_recon_cron.py +++ b/test/unit/cli/test_recon_cron.py @@ -16,6 +16,7 @@ import tempfile import shutil import os +import mock from unittest import TestCase from swift.cli.recon_cron import get_async_count @@ -48,3 +49,32 @@ class TestReconCron(TestCase): count = get_async_count(device_dir) self.assertEqual(count, 3) + + def test_get_async_count_deleted(self): + device_dir = os.path.join(self.temp_dir, 'device') + device_index = os.path.join(device_dir, '1') + async_dir = os.path.join(device_index, ASYNCDIR_BASE) + entry1 = os.path.join(async_dir, 'entry1') + entry2 = os.path.join(async_dir, 'entry2') + os.makedirs(entry1) + os.makedirs(entry2) + + pending_file1 = os.path.join(entry1, 'pending_file1') + pending_file2 = os.path.join(entry1, 'pending_file2') + pending_file3 = os.path.join(entry2, 'pending_file3') + open(pending_file1, 'w').close() + open(pending_file2, 'w').close() + open(pending_file3, 'w').close() + + orig_isdir = os.path.isdir + + def racy_isdir(d): + result = orig_isdir(d) + if d == entry1: + # clean it up before caller has a chance to descend + shutil.rmtree(entry1) + return result + + with mock.patch('os.path.isdir', racy_isdir): + count = get_async_count(device_dir) + self.assertEqual(count, 1) From 8424b02290c75a7e1eb2e36296b41926f041249a Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 19 Mar 2024 15:36:26 -0700 Subject: [PATCH 5/6] s3api: Fix handling of non-ascii access keys We stuff the access key into the request path until we get back a more-authoritative account name from auth. But it needs to be a WSGI string when we do! Closes-Bug: #2058748 Change-Id: I34adb8141cc9e62d17a27f01c63f40d1dd25991c --- swift/common/middleware/s3api/s3request.py | 2 +- swift/common/middleware/s3api/s3token.py | 4 ++-- swift/common/middleware/tempauth.py | 10 ++++---- test/unit/common/middleware/s3api/__init__.py | 4 +++- .../common/middleware/s3api/test_s3token.py | 17 ++++++++++++-- test/unit/common/middleware/test_tempauth.py | 23 +++++++++++++++++++ 6 files changed, 50 insertions(+), 10 deletions(-) diff --git a/swift/common/middleware/s3api/s3request.py b/swift/common/middleware/s3api/s3request.py index dcc0fb361b..9050745a39 100644 --- a/swift/common/middleware/s3api/s3request.py +++ b/swift/common/middleware/s3api/s3request.py @@ -1142,7 +1142,7 @@ class S3Request(swob.Request): Create a Swift request based on this request's environment. """ if self.account is None: - account = self.access_key + account = swob.str_to_wsgi(self.access_key) else: account = self.account diff --git a/swift/common/middleware/s3api/s3token.py b/swift/common/middleware/s3api/s3token.py index 3693b2f28d..23ebcc7086 100644 --- a/swift/common/middleware/s3api/s3token.py +++ b/swift/common/middleware/s3api/s3token.py @@ -65,7 +65,7 @@ import six from six.moves import urllib from swift.common.swob import Request, HTTPBadRequest, HTTPUnauthorized, \ - HTTPException + HTTPException, str_to_wsgi from swift.common.utils import config_true_value, split_path, get_logger, \ cache_from_env, append_underscore from swift.common.wsgi import ConfigFileError @@ -404,7 +404,7 @@ class S3Token(object): self._logger.debug('Connecting with tenant: %s', tenant_to_connect) new_tenant_name = '%s%s' % (self._reseller_prefix, tenant_to_connect) environ['PATH_INFO'] = environ['PATH_INFO'].replace( - account, new_tenant_name, 1) + str_to_wsgi(account), str_to_wsgi(new_tenant_name), 1) return self._app(environ, start_response) diff --git a/swift/common/middleware/tempauth.py b/swift/common/middleware/tempauth.py index 49541329b1..52d6659a72 100644 --- a/swift/common/middleware/tempauth.py +++ b/swift/common/middleware/tempauth.py @@ -184,9 +184,11 @@ import base64 from eventlet import Timeout import six from swift.common.memcached import MemcacheConnectionError -from swift.common.swob import Response, Request, wsgi_to_str -from swift.common.swob import HTTPBadRequest, HTTPForbidden, HTTPNotFound, \ - HTTPUnauthorized, HTTPMethodNotAllowed, HTTPServiceUnavailable +from swift.common.swob import ( + Response, Request, wsgi_to_str, str_to_wsgi, wsgi_unquote, + HTTPBadRequest, HTTPForbidden, HTTPNotFound, + HTTPUnauthorized, HTTPMethodNotAllowed, HTTPServiceUnavailable, +) from swift.common.request_helpers import get_sys_meta_prefix from swift.common.middleware.acl import ( @@ -469,7 +471,7 @@ class TempAuth(object): if not s3_auth_details['check_signature'](user['key']): return None env['PATH_INFO'] = env['PATH_INFO'].replace( - account_user, account_id, 1) + str_to_wsgi(account_user), wsgi_unquote(account_id), 1) groups = self._get_user_groups(account, account_user, account_id) return groups diff --git a/test/unit/common/middleware/s3api/__init__.py b/test/unit/common/middleware/s3api/__init__.py index d34d7e83bf..3319a455fc 100644 --- a/test/unit/common/middleware/s3api/__init__.py +++ b/test/unit/common/middleware/s3api/__init__.py @@ -46,10 +46,12 @@ class FakeAuthApp(object): E.g. '/v1/test:tester/bucket/object' will become '/v1/AUTH_test/bucket/object'. This method emulates the behavior. """ - tenant_user = env['s3api.auth_details']['access_key'] + tenant_user = swob.str_to_wsgi(env['s3api.auth_details']['access_key']) tenant, user = tenant_user.rsplit(':', 1) path = env['PATH_INFO'] + # Make sure it's valid WSGI + swob.wsgi_to_str(path) env['PATH_INFO'] = path.replace(tenant_user, 'AUTH_' + tenant) @staticmethod diff --git a/test/unit/common/middleware/s3api/test_s3token.py b/test/unit/common/middleware/s3api/test_s3token.py index bef6cf9e7b..6a467bd3a0 100644 --- a/test/unit/common/middleware/s3api/test_s3token.py +++ b/test/unit/common/middleware/s3api/test_s3token.py @@ -199,7 +199,8 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase): self.middleware(req.environ, self.start_fake_response) self.assertEqual(self.response_status, 200) - def _assert_authorized(self, req, account_path='/v1/AUTH_TENANT_ID/'): + def _assert_authorized(self, req, account_path='/v1/AUTH_TENANT_ID/', + access_key='access'): self.assertTrue( req.path.startswith(account_path), '%r does not start with %r' % (req.path, account_path)) @@ -224,7 +225,7 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase): self.assertEqual(1, self.requests_mock.call_count) request_call = self.requests_mock.request_history[0] self.assertEqual(json.loads(request_call.body), {'credentials': { - 'access': 'access', + 'access': access_key, 'signature': 'signature', 'token': base64.urlsafe_b64encode(b'token').decode('ascii')}}) @@ -501,6 +502,18 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase): self._assert_authorized(req, account_path='/v1/') self.assertEqual(req.environ['PATH_INFO'], '/v1/AUTH_TENANT_ID/c/o') + def test_authorize_with_unicode_access_key(self): + req = Request.blank('/v1/acc\xc3\xa9sskey/c/o') + req.environ['s3api.auth_details'] = { + 'access_key': u'acc\u00e9ss', + 'signature': u'signature', + 'string_to_sign': u'token', + } + req.get_response(self.middleware) + self._assert_authorized(req, account_path='/v1/', + access_key=u'acc\u00e9ss') + self.assertEqual(req.environ['PATH_INFO'], '/v1/AUTH_TENANT_ID/c/o') + def test_authorize_with_access_key_and_unquote_chars(self): req = Request.blank('/v1/access%key=/c/o') req.environ['s3api.auth_details'] = { diff --git a/test/unit/common/middleware/test_tempauth.py b/test/unit/common/middleware/test_tempauth.py index a29652fe2c..9d2cb38abc 100644 --- a/test/unit/common/middleware/test_tempauth.py +++ b/test/unit/common/middleware/test_tempauth.py @@ -308,6 +308,29 @@ class TestAuth(unittest.TestCase): self.assertEqual(req.environ['swift.authorize'], local_auth.authorize) + def test_auth_with_s3api_unicode_authorization_good(self): + local_app = FakeApp() + conf = {u'user_t\u00e9st_t\u00e9ster': u'p\u00e1ss .admin'} + access_key = u't\u00e9st:t\u00e9ster' + if six.PY2: + conf = {k.encode('utf8'): v.encode('utf8') + for k, v in conf.items()} + access_key = access_key.encode('utf8') + local_auth = auth.filter_factory(conf)(local_app) + req = self._make_request('/v1/t\xc3\xa9st:t\xc3\xa9ster', environ={ + 's3api.auth_details': { + 'access_key': access_key, + 'signature': b64encode('sig'), + 'string_to_sign': 't', + 'check_signature': lambda secret: True}}) + resp = req.get_response(local_auth) + + self.assertEqual(resp.status_int, 404) + self.assertEqual(local_app.calls, 1) + self.assertEqual(req.environ['PATH_INFO'], '/v1/AUTH_t\xc3\xa9st') + self.assertEqual(req.environ['swift.authorize'], + local_auth.authorize) + def test_auth_with_swift3_authorization_invalid(self): local_app = FakeApp() local_auth = auth.filter_factory( From 6387949a540e871e19062de8131503a8ac2ceef7 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Thu, 4 Apr 2024 10:36:36 -0500 Subject: [PATCH 6/6] lower-constraints: update to mock 3.0 The main motivation here is that mock.call becomes a namedtuple and you can say `m.call_args_list[0].args` instead of `m.call_args_list[0][0]` Change-Id: Ibb1a64ef0bfdebf06d26636cdb6ea191c10705f7 --- lower-constraints.txt | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lower-constraints.txt b/lower-constraints.txt index 27085dc417..6706c9a347 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -36,7 +36,7 @@ keystonemiddleware==4.17.0 linecache2==1.0.0 lxml==3.4.1 MarkupSafe==1.0 -mock==2.0 +mock==3.0 monotonic==1.4 msgpack==0.5.6 netaddr==0.7.19 diff --git a/test-requirements.txt b/test-requirements.txt index 1e77045c9d..b5cf150d6e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,7 +8,7 @@ coverage>=5.0.4 # Apache-2.0 pytest>=4.6.11 # MIT pytest-cov>=2.12.1 # MIT stestr>=2.0.0 # Apache-2.0 -mock>=2.0 # BSD +mock>=3.0 # BSD python-swiftclient>=3.2.0 python-keystoneclient!=2.1.0,>=2.0.0 # Apache-2.0 boto>=2.32.1