Merge master into feature/mpu

Change-Id: I5e17a9f6e06c7a021c86c73cb1534702505cd6a8

.zuul.yaml

@@ -348,16 +348,16 @@
       - tools/playbooks/probetests/post.yaml
 
 - job:
-    name: swift-probetests-centos-8-stream
+    name: swift-probetests-centos-9-stream
     parent: unittests
-    nodeset: centos-8-stream
+    nodeset: centos-9-stream
     description: |
       Setup a SAIO dev environment and run Swift's probe tests
       under Python 3.
     timeout: 7200
     vars:
       s3_acl: no
-      bindep_profile: test py36
+      bindep_profile: test py39
     pre-run:
       - tools/playbooks/common/install_dependencies.yaml
       - tools/playbooks/saio_single_node_setup/setup_saio.yaml
@@ -367,12 +367,12 @@
     post-run: tools/playbooks/probetests/post.yaml
 
 - job:
-    name: swift-probetests-centos-8-stream-arm64
+    name: swift-probetests-centos-9-stream-arm64
-    parent: swift-probetests-centos-8-stream
+    parent: swift-probetests-centos-9-stream
     nodeset:
       nodes:
-        - name: swift-centos-8-stream-arm64
+        - name: swift-centos-9-stream-arm64
-          label: centos-8-stream-arm64
+          label: centos-9-stream-arm64
     description: |
       Setup a SAIO dev environment and run Swift's probe tests
       under Python 3 on top of arm64 architecture.
@@ -380,7 +380,7 @@
 
 - job:
     name: swift-func-cors
-    parent: swift-probetests-centos-8-stream
+    parent: swift-probetests-centos-9-stream
     description: |
       Setup a SAIO dev environment and run Swift's CORS functional tests
     timeout: 1200
@@ -713,7 +713,7 @@
               - ^doc/(requirements.txt|(manpages|s3api|source)/.*)$
               - ^test/(cors|unit|functional|probe)/.*$
               - ^(.gitreview|.mailmap|AUTHORS|CHANGELOG|.*\.rst)$
-        - swift-probetests-centos-8-stream:
+        - swift-probetests-centos-9-stream:
             irrelevant-files: &probetest-irrelevant-files
               - ^(api-ref|releasenotes)/.*$
               # Keep doc/saio -- we use those sample configs in the saio playbooks
@@ -769,7 +769,7 @@
         - swift-tox-func-ec-py38
         - swift-func-cors
         - swift-tox-func-s3api-tests-tempauth
-        - swift-probetests-centos-8-stream:
+        - swift-probetests-centos-9-stream:
             irrelevant-files: *probetest-irrelevant-files
         - swift-dsvm-functional:
             irrelevant-files: *functest-irrelevant-files

lower-constraints.txt

@@ -36,7 +36,7 @@ keystonemiddleware==4.17.0
 linecache2==1.0.0
 lxml==3.4.1
 MarkupSafe==1.0
-mock==2.0
+mock==3.0
 monotonic==1.4
 msgpack==0.5.6
 netaddr==0.7.19

releasenotes/source/2024.1.rst

@@ -0,0 +1,6 @@
+===========================
+2024.1 Series Release Notes
+===========================
+
+.. release-notes::
+   :branch: stable/2024.1

releasenotes/source/index.rst

@@ -6,6 +6,7 @@
    :maxdepth: 1
 
    current
+   2024.1
    2023.2
    2023.1
    zed

swift/cli/recon_cron.py

@@ -18,18 +18,18 @@ import time
 from eventlet import Timeout
 
 from swift.common.utils import get_logger, dump_recon_cache, readconf, \
-    lock_path
+    lock_path, listdir
 from swift.common.recon import RECON_OBJECT_FILE, DEFAULT_RECON_CACHE_PATH
 from swift.obj.diskfile import ASYNCDIR_BASE
 
 
 def get_async_count(device_dir):
     async_count = 0
-    for i in os.listdir(device_dir):
+    for i in listdir(device_dir):
         device = os.path.join(device_dir, i)
         if not os.path.isdir(device):
             continue
-        for asyncdir in os.listdir(device):
+        for asyncdir in listdir(device):
             # skip stuff like "accounts", "containers", etc.
             if not (asyncdir == ASYNCDIR_BASE or
                     asyncdir.startswith(ASYNCDIR_BASE + '-')):
@@ -37,10 +37,10 @@ def get_async_count(device_dir):
             async_pending = os.path.join(device, asyncdir)
 
             if os.path.isdir(async_pending):
-                for entry in os.listdir(async_pending):
+                for entry in listdir(async_pending):
                     if os.path.isdir(os.path.join(async_pending, entry)):
                         async_hdir = os.path.join(async_pending, entry)
-                        async_count += len(os.listdir(async_hdir))
+                        async_count += len(listdir(async_hdir))
     return async_count
 
 

swift/common/middleware/s3api/s3request.py

@@ -1142,7 +1142,7 @@ class S3Request(swob.Request):
         Create a Swift request based on this request's environment.
         """
         if self.account is None:
-            account = self.access_key
+            account = swob.str_to_wsgi(self.access_key)
         else:
             account = self.account
 

swift/common/middleware/s3api/s3token.py

@@ -65,7 +65,7 @@ import six
 from six.moves import urllib
 
 from swift.common.swob import Request, HTTPBadRequest, HTTPUnauthorized, \
-    HTTPException
+    HTTPException, str_to_wsgi
 from swift.common.utils import config_true_value, split_path, get_logger, \
     cache_from_env, append_underscore
 from swift.common.wsgi import ConfigFileError
@@ -404,7 +404,7 @@ class S3Token(object):
         self._logger.debug('Connecting with tenant: %s', tenant_to_connect)
         new_tenant_name = '%s%s' % (self._reseller_prefix, tenant_to_connect)
         environ['PATH_INFO'] = environ['PATH_INFO'].replace(
-            account, new_tenant_name, 1)
+            str_to_wsgi(account), str_to_wsgi(new_tenant_name), 1)
         return self._app(environ, start_response)
 
 

swift/common/middleware/tempauth.py

@@ -184,9 +184,11 @@ import base64
 from eventlet import Timeout
 import six
 from swift.common.memcached import MemcacheConnectionError
-from swift.common.swob import Response, Request, wsgi_to_str
+from swift.common.swob import (
-from swift.common.swob import HTTPBadRequest, HTTPForbidden, HTTPNotFound, \
+    Response, Request, wsgi_to_str, str_to_wsgi, wsgi_unquote,
-    HTTPUnauthorized, HTTPMethodNotAllowed, HTTPServiceUnavailable
+    HTTPBadRequest, HTTPForbidden, HTTPNotFound,
+    HTTPUnauthorized, HTTPMethodNotAllowed, HTTPServiceUnavailable,
+)
 
 from swift.common.request_helpers import get_sys_meta_prefix
 from swift.common.middleware.acl import (
@@ -469,7 +471,7 @@ class TempAuth(object):
             if not s3_auth_details['check_signature'](user['key']):
                 return None
             env['PATH_INFO'] = env['PATH_INFO'].replace(
-                account_user, account_id, 1)
+                str_to_wsgi(account_user), wsgi_unquote(account_id), 1)
             groups = self._get_user_groups(account, account_user, account_id)
 
         return groups

test-requirements.txt

@@ -8,7 +8,7 @@ coverage>=5.0.4 # Apache-2.0
 pytest>=4.6.11  # MIT
 pytest-cov>=2.12.1  # MIT
 stestr>=2.0.0 # Apache-2.0
-mock>=2.0 # BSD
+mock>=3.0 # BSD
 python-swiftclient>=3.2.0
 python-keystoneclient!=2.1.0,>=2.0.0 # Apache-2.0
 boto>=2.32.1

test/cors/main.py

@@ -194,8 +194,12 @@ def run(args, url):
         try:
             browser = driver(**kwargs)
         except Exception as e:
-            if not ('needs to be in PATH' in str(e) or
+            if not any(x in str(e) for x in (
-                    'SafariDriver was not found' in str(e)):
+                'needs to be in PATH',
+                'SafariDriver was not found',
+                'OSError: [Errno 8] Exec format error',
+                'safaridriver not available for download',
+            )):
                 traceback.print_exc()
             results.append(('SKIP', browser_name, str(e).strip()))
             continue

test/unit/cli/test_recon_cron.py

@@ -16,6 +16,7 @@
 import tempfile
 import shutil
 import os
+import mock
 
 from unittest import TestCase
 from swift.cli.recon_cron import get_async_count
@@ -48,3 +49,32 @@ class TestReconCron(TestCase):
 
         count = get_async_count(device_dir)
         self.assertEqual(count, 3)
+
+    def test_get_async_count_deleted(self):
+        device_dir = os.path.join(self.temp_dir, 'device')
+        device_index = os.path.join(device_dir, '1')
+        async_dir = os.path.join(device_index, ASYNCDIR_BASE)
+        entry1 = os.path.join(async_dir, 'entry1')
+        entry2 = os.path.join(async_dir, 'entry2')
+        os.makedirs(entry1)
+        os.makedirs(entry2)
+
+        pending_file1 = os.path.join(entry1, 'pending_file1')
+        pending_file2 = os.path.join(entry1, 'pending_file2')
+        pending_file3 = os.path.join(entry2, 'pending_file3')
+        open(pending_file1, 'w').close()
+        open(pending_file2, 'w').close()
+        open(pending_file3, 'w').close()
+
+        orig_isdir = os.path.isdir
+
+        def racy_isdir(d):
+            result = orig_isdir(d)
+            if d == entry1:
+                # clean it up before caller has a chance to descend
+                shutil.rmtree(entry1)
+            return result
+
+        with mock.patch('os.path.isdir', racy_isdir):
+            count = get_async_count(device_dir)
+        self.assertEqual(count, 1)

test/unit/common/middleware/s3api/__init__.py

@@ -46,10 +46,12 @@ class FakeAuthApp(object):
         E.g. '/v1/test:tester/bucket/object' will become
         '/v1/AUTH_test/bucket/object'. This method emulates the behavior.
         """
-        tenant_user = env['s3api.auth_details']['access_key']
+        tenant_user = swob.str_to_wsgi(env['s3api.auth_details']['access_key'])
         tenant, user = tenant_user.rsplit(':', 1)
 
         path = env['PATH_INFO']
+        # Make sure it's valid WSGI
+        swob.wsgi_to_str(path)
         env['PATH_INFO'] = path.replace(tenant_user, 'AUTH_' + tenant)
 
     @staticmethod

test/unit/common/middleware/s3api/test_s3token.py

@@ -199,7 +199,8 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
         self.middleware(req.environ, self.start_fake_response)
         self.assertEqual(self.response_status, 200)
 
-    def _assert_authorized(self, req, account_path='/v1/AUTH_TENANT_ID/'):
+    def _assert_authorized(self, req, account_path='/v1/AUTH_TENANT_ID/',
+                           access_key='access'):
         self.assertTrue(
             req.path.startswith(account_path),
             '%r does not start with %r' % (req.path, account_path))
@@ -224,7 +225,7 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
         self.assertEqual(1, self.requests_mock.call_count)
         request_call = self.requests_mock.request_history[0]
         self.assertEqual(json.loads(request_call.body), {'credentials': {
-            'access': 'access',
+            'access': access_key,
             'signature': 'signature',
             'token': base64.urlsafe_b64encode(b'token').decode('ascii')}})
 
@@ -501,6 +502,18 @@ class S3TokenMiddlewareTestGood(S3TokenMiddlewareTestBase):
         self._assert_authorized(req, account_path='/v1/')
         self.assertEqual(req.environ['PATH_INFO'], '/v1/AUTH_TENANT_ID/c/o')
 
+    def test_authorize_with_unicode_access_key(self):
+        req = Request.blank('/v1/acc\xc3\xa9sskey/c/o')
+        req.environ['s3api.auth_details'] = {
+            'access_key': u'acc\u00e9ss',
+            'signature': u'signature',
+            'string_to_sign': u'token',
+        }
+        req.get_response(self.middleware)
+        self._assert_authorized(req, account_path='/v1/',
+                                access_key=u'acc\u00e9ss')
+        self.assertEqual(req.environ['PATH_INFO'], '/v1/AUTH_TENANT_ID/c/o')
+
     def test_authorize_with_access_key_and_unquote_chars(self):
         req = Request.blank('/v1/access%key=/c/o')
         req.environ['s3api.auth_details'] = {

test/unit/common/middleware/test_tempauth.py

@@ -308,6 +308,29 @@ class TestAuth(unittest.TestCase):
         self.assertEqual(req.environ['swift.authorize'],
                          local_auth.authorize)
 
+    def test_auth_with_s3api_unicode_authorization_good(self):
+        local_app = FakeApp()
+        conf = {u'user_t\u00e9st_t\u00e9ster': u'p\u00e1ss .admin'}
+        access_key = u't\u00e9st:t\u00e9ster'
+        if six.PY2:
+            conf = {k.encode('utf8'): v.encode('utf8')
+                    for k, v in conf.items()}
+            access_key = access_key.encode('utf8')
+        local_auth = auth.filter_factory(conf)(local_app)
+        req = self._make_request('/v1/t\xc3\xa9st:t\xc3\xa9ster', environ={
+            's3api.auth_details': {
+                'access_key': access_key,
+                'signature': b64encode('sig'),
+                'string_to_sign': 't',
+                'check_signature': lambda secret: True}})
+        resp = req.get_response(local_auth)
+
+        self.assertEqual(resp.status_int, 404)
+        self.assertEqual(local_app.calls, 1)
+        self.assertEqual(req.environ['PATH_INFO'], '/v1/AUTH_t\xc3\xa9st')
+        self.assertEqual(req.environ['swift.authorize'],
+                         local_auth.authorize)
+
     def test_auth_with_swift3_authorization_invalid(self):
         local_app = FakeApp()
         local_auth = auth.filter_factory(

tools/playbooks/common/install_dependencies.yaml

@@ -22,13 +22,13 @@
         name: pip
         extra_args: --upgrade
 
-    - name: install rsync-daemon - CentOS 8
+    - name: install rsync-daemon - CentOS 8, 9
       package:
         name: rsync-daemon
         state: present
       when:
         - ansible_facts['distribution'] == "CentOS"
-        - ansible_facts['distribution_major_version'] == "8"
+        - ansible_facts['distribution_major_version'] != "7"
 
     - name: install python modules with pip
       pip: name={{ item }} state=present extra_args='--upgrade'

tools/playbooks/cors/install_selenium.yaml

@@ -7,7 +7,7 @@
         state: present
     - name: install selenium
       pip:
-        name: selenium
+        name: 'selenium<4'
         state: present
     - name: install firefox
       yum:
@@ -15,13 +15,15 @@
         state: present
     - name: fetch firefox driver
       get_url:
-        url: https://github.com/mozilla/geckodriver/releases/download/v0.26.0/geckodriver-v0.26.0-linux64.tar.gz
+        url: https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz
         dest: /tmp/geckodriver.tar.gz
     - name: unpack firefox driver
       unarchive:
         src: /tmp/geckodriver.tar.gz
         dest: /usr/local/bin
         remote_src: true
+    - name: check selenium version
+      command: pip show selenium
     - name: check firefox version
       command: firefox --version
     #- name: install chromium