diff --git a/.zuul.yaml b/.zuul.yaml
index bef962038c..f1fe39cf24 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -305,12 +305,13 @@
     name: swift-tox-func-s3api-ceph-s3tests-tempauth
     parent: unittests
     voting: false
-    nodeset: centos-7
+    nodeset: centos-8-stream
     description: |
       Setup a SAIO dev environment and run ceph-s3tests
     timeout: 5400
     vars:
       s3_acl: yes
+      bindep_profile: test py36
     pre-run:
       - tools/playbooks/common/install_dependencies.yaml
       - tools/playbooks/saio_single_node_setup/setup_saio.yaml
@@ -375,18 +376,18 @@
     post-run: tools/playbooks/cors/post.yaml
 
 - nodeset:
-    name: swift-five-nodes
+    name: swift-five-nodes-py3
     nodes:
       - name: test-runner1
-        label: centos-7
+        label: centos-8-stream
       - name: proxy1
-        label: centos-7
+        label: centos-8-stream
       - name: account1
-        label: centos-7
+        label: centos-8-stream
       - name: container1
-        label: centos-7
+        label: centos-8-stream
       - name: object1
-        label: centos-7
+        label: centos-8-stream
     groups:
       - name: test-runner
         nodes:
@@ -418,10 +419,12 @@
 - job:
     name: swift-multinode-rolling-upgrade
     parent: multinode
-    nodeset: swift-five-nodes
+    nodeset: swift-five-nodes-py3
     description: |
       Build a 4 node swift cluster and run functional tests
     timeout: 5400
+    vars:
+      bindep_profile: test py36
     pre-run:
       - tools/playbooks/multinode_setup/pre.yaml
       - tools/playbooks/common/install_dependencies.yaml
@@ -431,18 +434,6 @@
     run: tools/playbooks/multinode_setup/run.yaml
     post-run: tools/playbooks/probetests/post.yaml
 
-- job:
-    name: swift-multinode-rolling-upgrade-rocky
-    parent: swift-multinode-rolling-upgrade
-    vars:
-      previous_swift_version: origin/stable/rocky
-
-- job:
-    name: swift-multinode-rolling-upgrade-stein
-    parent: swift-multinode-rolling-upgrade
-    vars:
-      previous_swift_version: origin/stable/stein
-
 - job:
     name: swift-multinode-rolling-upgrade-train
     parent: swift-multinode-rolling-upgrade
diff --git a/doc/s3api/conf/ceph-known-failures-tempauth.yaml b/doc/s3api/conf/ceph-known-failures-tempauth.yaml
index 921bbe933c..d2922dc852 100644
--- a/doc/s3api/conf/ceph-known-failures-tempauth.yaml
+++ b/doc/s3api/conf/ceph-known-failures-tempauth.yaml
@@ -59,27 +59,19 @@ ceph_s3:
   s3tests_boto3.functional.test_headers.test_object_create_bad_contentlength_none: {status: KNOWN}
   s3tests_boto3.functional.test_headers.test_object_create_bad_date_none_aws2: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_100_continue: {status: KNOWN}
-  s3tests_boto3.functional.test_s3.test_abort_multipart_upload: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_atomic_conditional_write_1mb: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_atomic_dual_conditional_write_1mb: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_bucket_acl_grant_email: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_bucket_acl_grant_email_notexist: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_bucket_acl_grant_nonexist_user: {status: KNOWN}
-  s3tests_boto3.functional.test_s3.test_bucket_acl_grant_userid_fullcontrol: {status: KNOWN}
-  s3tests_boto3.functional.test_s3.test_bucket_acl_grant_userid_read: {status: KNOWN}
-  s3tests_boto3.functional.test_s3.test_bucket_acl_grant_userid_readacp: {status: KNOWN}
-  s3tests_boto3.functional.test_s3.test_bucket_acl_grant_userid_write: {status: KNOWN}
-  s3tests_boto3.functional.test_s3.test_bucket_acl_grant_userid_writeacp: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_bucket_acl_no_grants: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_bucket_create_exists: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_bucket_create_naming_bad_long: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_bucket_create_naming_bad_punctuation: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_bucket_create_naming_bad_short_empty: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_bucket_head_extended: {status: KNOWN}
-  s3tests_boto3.functional.test_s3.test_bucket_header_acl_grants: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_bucket_list_objects_anonymous: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_bucket_list_objects_anonymous_fail: {status: KNOWN}
-  s3tests_boto3.functional.test_s3.test_bucket_list_prefix_unreadable: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_bucket_list_return_data: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_bucket_list_return_data_versioning: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_bucket_list_unordered: {status: KNOWN}
@@ -116,7 +108,6 @@ ceph_s3:
   s3tests_boto3.functional.test_s3.test_encryption_sse_c_multipart_bad_download: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_encryption_sse_c_multipart_invalid_chunks_1: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_encryption_sse_c_multipart_invalid_chunks_2: {status: KNOWN}
-  s3tests_boto3.functional.test_s3.test_encryption_sse_c_multipart_upload: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_encryption_sse_c_no_key: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_encryption_sse_c_no_md5: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_encryption_sse_c_other_key: {status: KNOWN}
@@ -151,13 +142,10 @@ ceph_s3:
   s3tests_boto3.functional.test_s3.test_logging_toggle: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_multipart_copy_invalid_range: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_multipart_resend_first_finishes_last: {status: KNOWN}
-  s3tests_boto3.functional.test_s3.test_multipart_upload: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_multipart_upload_empty: {status: KNOWN}
-  s3tests_boto3.functional.test_s3.test_object_acl_canned_bucketownerread: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_object_anon_put: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_object_anon_put_write_access: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_object_delete_key_bucket_gone: {status: KNOWN}
-  s3tests_boto3.functional.test_s3.test_object_header_acl_grants: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_object_lock_delete_object_with_legal_hold_off: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_object_lock_delete_object_with_legal_hold_on: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_object_lock_delete_object_with_retention: {status: KNOWN}
@@ -263,3 +251,108 @@ ceph_s3:
   s3tests_boto3.functional.test_s3.test_versioning_bucket_multipart_upload_return_version_id: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_versioning_multi_object_delete_with_marker_create: {status: KNOWN}
   s3tests_boto3.functional.test_s3.test_versioning_obj_plain_null_version_overwrite: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_account_usage: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_head_bucket_usage: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_object_set_get_metadata_empty_to_unreadable_infix: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_bucket_recreate_overwrite_acl: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_bucket_recreate_new_acl: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_bucket_acl_grant_email_not_exist: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_buckets_list_ctime: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_bucket_create_naming_good_starts_alpha: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_bucket_create_naming_good_starts_digit: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_list_multipart_upload_owner: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_set_bucket_tagging: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_atomic_dual_write_1mb: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_atomic_dual_write_4mb: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_atomic_dual_write_8mb: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_lifecycle_expiration_tags1: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_lifecycle_expiration_tags2: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_lifecycle_expiration_versioned_tags2: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_lifecycle_expiration_noncur_tags1: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_lifecycle_expiration_header_tags_head: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_lifecycle_expiration_header_and_tags_head: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_object_lock_get_obj_retention_iso8601: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_object_lock_multi_delete_object_with_retention: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_without_bypass: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_compliance: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_object_read_unreadable: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_user_policy: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_get_bucket_policy_status: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_get_public_acl_bucket_policy_status: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_get_authpublic_acl_bucket_policy_status: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_get_publicpolicy_acl_bucket_policy_status: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_get_nonpublicpolicy_acl_bucket_policy_status: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_get_nonpublicpolicy_deny_bucket_policy_status: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_get_default_public_block: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_put_public_block: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_block_public_put_bucket_acls: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_block_public_object_canned_acls: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_block_public_policy: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_ignore_public_acls: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_multipart_upload_on_a_bucket_with_policy: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_put_bucket_encryption: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_get_bucket_encryption: {status: KNOWN}
+  s3tests_boto3.functional.test_s3.test_delete_bucket_encryption: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_generate_where_clause: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_generate_projection: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_count_operation: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_column_sum_min_max: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_nullif_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_nulliftrue_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_is_not_null_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_lowerupper_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_in_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_true_false_in_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_like_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_truefalselike_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_complex_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_alias: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_alias_cyclic_refernce: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_datetime: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_true_false_datetime: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_csv_parser: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_csv_definition: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_schema_definition: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_when_then_else_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_coalesce_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_cast_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_trim_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_truefalse_trim_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_escape_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_case_value_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_bool_cast_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_s3select.test_progress_expressions: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_get_session_token: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_get_session_token_permanent_creds_denied: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_allow: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_deny: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_creds_expiry: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_deny_head_nonexistent: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_allow_head_nonexistent: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_with_web_identity: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_session_policy_check_on_different_buckets: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_session_policy_check_on_same_bucket: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_session_policy_check_put_obj_denial: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_swapping_role_policy_and_session_policy: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_session_policy_check_different_op_permissions: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_session_policy_check_with_deny_effect: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_session_policy_check_with_deny_on_same_op: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_session_policy_bucket_policy_role_arn: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_session_policy_bucket_policy_session_arn: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_session_policy_copy_object: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_session_policy_no_bucket_role_policy: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_session_policy_bucket_policy_deny: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_with_web_identity_with_sub: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_with_web_identity_with_azp: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_with_web_identity_with_request_tag: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_with_web_identity_with_principal_tag: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_with_web_identity_for_all_values: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_with_web_identity_for_all_values_deny: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_with_web_identity_tag_keys_trust_policy: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_with_web_identity_tag_keys_role_policy: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_with_web_identity_resource_tag: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_with_web_identity_resource_tag_deny: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_with_web_identity_wrong_resource_tag_deny: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_with_web_identity_resource_tag_princ_tag: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_with_web_identity_resource_tag_copy_obj: {status: KNOWN}
+  s3tests_boto3.functional.test_sts.test_assume_role_with_web_identity_role_resource_tag: {status: KNOWN}
diff --git a/py2-constraints.txt b/py2-constraints.txt
index f2b26eb8b9..af78a2708a 100644
--- a/py2-constraints.txt
+++ b/py2-constraints.txt
@@ -77,3 +77,4 @@ python-keystoneclient===3.22.0
 dnspython===1.16.0
 setuptools===44.1.1
 pycadf===2.10.0
+smmap===3.0.5
diff --git a/tools/playbooks/ceph-s3tests/run.yaml b/tools/playbooks/ceph-s3tests/run.yaml
index ae5bd85baa..618c6821ac 100644
--- a/tools/playbooks/ceph-s3tests/run.yaml
+++ b/tools/playbooks/ceph-s3tests/run.yaml
@@ -28,11 +28,16 @@
           repo: "https://github.com/tipabu/s3compat.git"
           dest: "{{ ansible_env.HOME }}/s3compat"
 
+    - name: Install virtualenv
+      pip:
+        name: virtualenv
+      become: true
+
     - name: Install s3compat requirements
       pip:
           requirements: "{{ ansible_env.HOME }}/s3compat/requirements.txt"
           virtualenv: "{{ ansible_env.HOME }}/venv"
-          virtualenv_python: python2
+          virtualenv_python: python3
       become: true
 
     - name: Run s3compat tests
diff --git a/tools/playbooks/multinode_setup/make_rings.yaml b/tools/playbooks/multinode_setup/make_rings.yaml
index 0fef9d6262..6eda38dc5a 100644
--- a/tools/playbooks/multinode_setup/make_rings.yaml
+++ b/tools/playbooks/multinode_setup/make_rings.yaml
@@ -33,7 +33,7 @@
 
     - name: build swift
       shell:
-        cmd: python setup.py sdist
+        cmd: python3 setup.py sdist
         executable: /bin/bash
         chdir: '{{ zuul.project.src_dir }}'
 
diff --git a/tools/playbooks/multinode_setup/pre.yaml b/tools/playbooks/multinode_setup/pre.yaml
index d7e4670a80..55eb8d1991 100644
--- a/tools/playbooks/multinode_setup/pre.yaml
+++ b/tools/playbooks/multinode_setup/pre.yaml
@@ -2,7 +2,6 @@
   roles:
     # Run bindep and test-setup after devstack so that they won't interfere
     - role: bindep
-      bindep_profile: test
       bindep_dir: "{{ zuul_work_dir }}"
     - test-setup
     - ensure-tox
diff --git a/tools/playbooks/multinode_setup/run.yaml b/tools/playbooks/multinode_setup/run.yaml
index a2eca8eb7f..a186124c29 100644
--- a/tools/playbooks/multinode_setup/run.yaml
+++ b/tools/playbooks/multinode_setup/run.yaml
@@ -22,7 +22,7 @@
     - name: install swift
       become: true
       shell:
-        cmd: python setup.py develop
+        cmd: python3 setup.py develop
         executable: /bin/bash
         chdir: '{{ zuul.project.src_dir }}'
 
@@ -37,6 +37,6 @@
       include_role:
         name: tox
       vars:
-        tox_envlist: func
+        tox_envlist: func-py3
         tox_environment:
           SWIFT_TEST_CONFIG_FILE: /home/{{ ansible_ssh_user }}/test.conf