Merge "Keep CI up-to-date"

.zuul.d/install-jobs.yaml

@@ -6,16 +6,16 @@
     post-run: tests/ci/playbooks/fetch-html-and-json-reports.yaml
     timeout: 1800
 
-- job:
-    name: rally-install-ubuntu-bionic
-    parent: rally-install-base
-    nodeset: ubuntu-bionic
-
 - job:
     name: rally-install-ubuntu-focal
     parent: rally-install-base
     nodeset: ubuntu-focal
 
+- job:
+    name: rally-install-ubuntu-jammy
+    parent: rally-install-base
+    nodeset: ubuntu-jammy
+
 - job:
     name: rally-install-centos-8s
     parent: rally-install-base
@@ -24,4 +24,4 @@
 - job:
     name: rally-install-centos-9s
     parent: rally-install-base
-    nodeset: centos-8-stream
+    nodeset: centos-9-stream

.zuul.d/python-jobs.yaml

@@ -6,7 +6,7 @@
     post-run: tests/ci/playbooks/fetch-html-and-json-reports.yaml
     description: |
       Run test for rally project.
-    nodeset: ubuntu-bionic
+    nodeset: ubuntu-jammy
 
 - job:
     name: rally-tox-docs
@@ -28,16 +28,6 @@
     vars:
       tox_env: pep8
 
-- job:
-    name: rally-tox-functional-py38
-    parent: rally-tox-base
-    description: |
-      Run test for rally project.
-
-      Uses tox with the ``functional`` environment.
-    vars:
-      tox_env: functional-py38
-
 - job:
     name: rally-tox-functional
     parent: rally-tox-base
@@ -67,6 +57,7 @@
       Uses tox with the ``py36`` environment.
     vars:
       tox_env: py36
+    nodeset: ubuntu-bionic
 
 - job:
     name: rally-tox-py37
@@ -77,6 +68,7 @@
       Uses tox with the ``py37`` environment.
     vars:
       tox_env: py37
+    nodeset: ubuntu-bionic
 
 - job:
     name: rally-tox-py38
@@ -85,9 +77,9 @@
       Run unit test for rally project.
 
       Uses tox with the ``py38`` environment.
-    nodeset: ubuntu-focal
     vars:
       tox_env: py38
+    nodeset: ubuntu-focal
 
 - job:
     name: rally-tox-py39
@@ -96,9 +88,19 @@
       Run unit test for rally project.
 
       Uses tox with the ``py39`` environment.
-    nodeset: ubuntu-focal
     vars:
       tox_env: py39
+    nodeset: ubuntu-focal
+
+- job:
+    name: rally-tox-py310
+    parent: rally-tox-base
+    description: |
+      Run unit test for rally project.
+
+      Uses tox with the ``py310`` environment.
+    vars:
+      tox_env: py310
 
 - job:
     name: rally-tox-samples
@@ -117,7 +119,6 @@
       Run test for rally project.
 
       Uses tox with the ``cover`` environment.
-    nodeset: ubuntu-bionic
     vars:
       coverage_output_src: '{{ zuul.project.src_dir }}/cover/'
       zuul_executor_dest: '{{ zuul.executor.log_root }}/coverage/'

.zuul.d/zuul.yaml

@@ -13,12 +13,12 @@
         - rally-tox-py37
         - rally-tox-py38
         - rally-tox-py39
+        - rally-tox-py310
         - rally-tox-samples
         - rally-tox-functional
-        - rally-tox-functional-py38
         - rally-tox-self
-        - rally-install-ubuntu-bionic
         - rally-install-ubuntu-focal
+        - rally-install-ubuntu-jammy
         - rally-install-centos-8s
         - rally-install-centos-9s
         - rally-docker-build
@@ -31,10 +31,11 @@
         - rally-tox-py37
         - rally-tox-py38
         - rally-tox-py39
+        - rally-tox-py310
         - rally-tox-functional
         - rally-tox-self
-        - rally-install-ubuntu-bionic
         - rally-install-ubuntu-focal
+        - rally-install-ubuntu-jammy
         - rally-install-centos-8s
         - rally-install-centos-9s
     post:

CHANGELOG.rst

@@ -27,11 +27,19 @@ Fixed
 
   `Launchpad-bug #1956956 <https://launchpad.net/bugs/1956956>`_
 
-Changed
+Added
+~~~~~
+
+* Pin SQLAlchemy to <2.0.0
+* CI for running unit and functional tests using python 3.10
+* CI jobs that check Rally installation compatibility with CentOS 9 Stream and
+  Ubuntu Jammy
+
+Removed
 ~~~~~~~
 
-Check ability to install Rally on Centos 8 Stream and Centos 9 Stream and
+* CI jobs with installation compatibility checks for CentOS 7, CentOS 8
-stop checking Centos 7 and Centos 8
+  (CentOS 8 Stream is checked instead), Ubuntu Bionic.
 
 [3.3.0] - 2021-06-16
 --------------------

rally/cli/cliutils.py

@@ -616,11 +616,11 @@ def run(argv, categories):
         rapi = api.API(config_args=argv[1:], skip_db_check=True)
     except exceptions.RallyException as e:
         print(e)
-        return(2)
+        return 2
 
     if CONF.category.name == "bash-completion":
         print(_generate_bash_completion_script())
-        return(0)
+        return 0
 
     fn = CONF.category.action_fn
     fn_args = [encodeutils.safe_decode(arg)
@@ -652,7 +652,7 @@ def run(argv, categories):
                 if arg[1].get("dest", "").endswith(missing):
                     print(" " + arg[0][0])
                     break
-        return(1)
+        return 1
 
     try:
         validate_deprecated_args(argv, fn)

rally/cli/commands/task.py

@@ -579,7 +579,7 @@ class TaskCommands(object):
             print("Error: Invalid task status '%s'.\nAvailable statuses: %s"
                   % (status, ", ".join(consts.TaskStatus)),
                   file=sys.stderr)
-            return(1)
+            return 1
 
         if not all_deployments:
             filters["deployment"] = deployment

rally/task/engine.py

@@ -124,13 +124,13 @@ class ResultConsumer(object):
                                                     {"raw": results_chunk})
                     self.workload_data_count += 1
 
-            elif self.is_done.isSet():
+            elif self.is_done.is_set():
                 break
             else:
                 time.sleep(0.1)
 
     def _consume_events(self):
-        while not self.is_done.isSet() or self.runner.event_queue:
+        while not self.is_done.is_set() or self.runner.event_queue:
             if self.runner.event_queue:
                 event = self.runner.event_queue.popleft()
                 self.hook_executor.on_event(
@@ -201,7 +201,7 @@ class ResultConsumer(object):
         runner.run method.
         """
 
-        while not self.is_done.isSet():
+        while not self.is_done.is_set():
             if self.is_task_in_aborting_status(self.task["uuid"],
                                                check_soft=False):
                 self.runner.abort()

rally/task/hook.py

@@ -63,7 +63,7 @@ class HookExecutor(object):
         stopwatch = rutils.Stopwatch(stop_event=self._timer_stop_event)
         stopwatch.start()
         seconds_since_start = 0
-        while not self._timer_stop_event.isSet():
+        while not self._timer_stop_event.is_set():
             self.on_event(event_type="time", value=seconds_since_start)
             seconds_since_start += 1
             stopwatch.sleep(seconds_since_start)

setup.cfg

@@ -20,6 +20,8 @@ classifier =
     Programming Language :: Python :: 3.6
     Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
 
 [files]
 packages =

tests/ci/playbooks/rally-install-pre.yaml

@@ -1,62 +1,33 @@
 - hosts: all
   name: Prepare host to install Rally
   tasks:
-    - name: Check OS distro (CentOS)
+    - name: Uninstall python3-pyyaml (CentOS 8 & 9)
-      when: ansible_distribution == "CentOS"
-      set_fact:
-        # in case of centos we do not care about minor versions
-        os_distro: '{{ ansible_distribution }} {{ ansible_distribution_major_version }}'
-
-    - name: Check OS distro (Ubuntu)
-      when: ansible_distribution == "Ubuntu"
-      set_fact:
-        os_distro: '{{ ansible_distribution }} {{ ansible_distribution_version }}'
-
-    - name: Install required packages (Centos-7)
-      when: os_distro == "CentOS 7"
-      shell:
-        cmd: |
-          sudo yum remove -y python-crypto || true
-          sudo yum remove -y python36-PyYAML || true
-
-          sudo yum update
-          sudo yum install -y yum-utils
-          sudo yum groupinstall -y development
-
-          sudo yum install -y https://repo.ius.io/ius-release-el7.rpm https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-          sudo yum install -y python36u python36u-devel
-
-    - name: Uninstall required packages (Centos-8)
-      when: os_distro == "CentOS 8"
       become: true
-      shell: dnf remove -y python3-pyyaml
+      package:
+        state: absent
+        name: python3-pyyaml
 
-    - name: Install required packages (Ubuntu-Bionic)
+    - name: Install python3.8-dev (Ubuntu 20.04)
-      when: os_distro == "Ubuntu 18.04"
+      become: true
-      shell:
+      package:
-        chdir: '{{ zuul.project.src_dir }}'
+        state: present
-        cmd: |
+        name: python3.8-dev
-          # NOTE(pabelanger): We run apt-get update to ensure we dont have a stale
+      when: ansible_distribution == "Ubuntu" and ansible_distribution_version == "20.04"
-          # package cache in the gate.
-          sudo apt update
-          sudo apt install --yes python3.6-dev
 
-    - name: Install required packages (Ubuntu-Focal)
+    - name: Install python3.10-dev (Ubuntu 22.04)
-      when: os_distro == "Ubuntu 20.04"
+      become: true
-      shell:
+      package:
-        chdir: '{{ zuul.project.src_dir }}'
+        state: present
-        cmd: |
+        name: python3.10-dev
-          # NOTE(pabelanger): We run apt-get update to ensure we dont have a stale
+      when: ansible_distribution == "Ubuntu" and ansible_distribution_version == "22.04"
-          # package cache in the gate.
-          sudo apt update
-          sudo apt install --yes python3.8-dev
 
     - name: Install pip3 if needed
-      when: os_distro == "CentOS 7" or os_distro == "Ubuntu 18.04" or os_distro == "Ubuntu 20.04"
+      become: true
       shell:
         executable: /bin/bash
         chdir: '{{ zuul.project.src_dir }}'
         cmd: |
+          set -e
           python_version=`python3 --version`
           python_version=`echo $python_version |awk '{print $2}'`
           echo $python_version
@@ -66,19 +37,13 @@
               pip_url=https://bootstrap.pypa.io/get-pip.py
           fi
           curl $pip_url -o /tmp/get-pip.py
-          sudo python3 /tmp/get-pip.py
+          python3 /tmp/get-pip.py
-
-    - name: Update pip3 if needed
-      when: os_distro == "CentOS 8"
-      become: true
-      shell: pip3 install --upgrade pip
 
     - name: Install bindep
       become: true
-      shell: pip3 install --upgrade bindep PyYAML
+      shell: pip3 install --upgrade bindep
 
     - name: Prepare rally plugins stored at home dir
-      shell:
+      shell: |
-        cmd: |
+        mkdir --parents ~/.rally/plugins
-          mkdir --parents ~/.rally/plugins
+        cp --recursive {{ zuul.project.src_dir }}/rally-jobs/plugins/* ~/.rally/plugins
-          cp --recursive {{ zuul.project.src_dir }}/rally-jobs/plugins/* ~/.rally/plugins

tests/ci/playbooks/roles/rally-tox/defaults/main.yaml

@@ -0,0 +1,4 @@
+default_pip_url: "https://bootstrap.pypa.io/get-pip.py"
+
+versioned_pip_url:
+  python3.6: "https://bootstrap.pypa.io/pip/3.6/get-pip.py"

tests/ci/playbooks/roles/rally-tox/files/find_python_for_tox_env.py

@@ -0,0 +1,85 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import configparser
+import re
+
+
+PY_FACTORS_RE = re.compile("^(?!py$)(py|pypy|jython)([2-9][0-9]?[0-9]?)?$")
+
+
+def _parse_env_name(env_name):
+    for factor in env_name.split("-"):
+        # copy-pasted from tox codebase with custom extra check
+        # https://github.com/tox-dev/tox/blob/6b76e18fcaa7c9610b642555bcb94aab1d37f2b3/src/tox/config/__init__.py#L658-L669
+        match = PY_FACTORS_RE.match(factor)
+        if match:
+            base_exe = {"py": "python"}.get(match.group(1), match.group(1))
+
+            if base_exe != "python":
+                raise ValueError("We do not support '%s' interpreter yet."
+                                 % base_exe)
+            version_s = match.group(2)
+            if not version_s:
+                version_info = ()
+            elif len(version_s) == 1:
+                version_info = (version_s,)
+            else:
+                version_info = (version_s[0], version_s[1:])
+            implied_version = ".".join(version_info)
+            implied_python = "{}{}".format(base_exe, implied_version)
+            return implied_python
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--tox-cfg", metavar="<path>", type=str, required=True,
+        help="A path to tox.ini file to parse."
+    )
+    parser.add_argument(
+        "--tox-env", metavar="<env-name>", type=str, required=True,
+        help="Tox env name."
+    )
+    parser.add_argument(
+        "--default-python3-version", metavar="<python-interpreter>",
+        type=str, required=False, default="python3.10",
+        help="Default python3 interpreter to use for 'python3' case."
+    )
+    args = parser.parse_args()
+
+    tox_cfg = configparser.ConfigParser()
+    tox_cfg.read(args.tox_cfg)
+
+    python_version = None
+
+    # check python version specific to target tox env
+    env_section = "testenv:%s" % args.tox_env
+    if env_section in tox_cfg:
+        python_version = tox_cfg[env_section].get("basepython")
+
+    # try to determine python version based on env name like tox does
+    if python_version is None:
+        python_version = _parse_env_name(args.tox_env)
+
+    # check python version that is configured for all tox envs as default
+    if python_version is None:
+        python_version = tox_cfg["testenv"].get("basepython", "python3")
+
+    if python_version == "python3":
+        python_version = args.default_python3_version
+    print(python_version)
+
+
+if __name__ == "__main__":
+    main()

tests/ci/playbooks/roles/rally-tox/tasks/install.yaml

@@ -0,0 +1,35 @@
+- name: Check required version of Python
+  args:
+    executable: python3
+  script: "find_python_for_tox_env.py --tox-cfg {{ zuul.project.src_dir }}/tox.ini --tox-env {{ tox_env }}"
+  changed_when: false
+  register: python_exec_from_tox
+  when: python_exec is not defined
+
+- name: "Set python_exec var to {{ python_exec_from_tox }}"
+  set_fact:
+    python_exec: "{{ python_exec_from_tox.stdout.strip() }}"
+  when: python_exec_from_tox is defined and python_exec_from_tox.stdout.strip()
+
+- name: Install the proper python version and pip
+  become: True
+  become_user: root
+  shell: |
+    set -e
+
+    apt-get update
+    apt-get install {{ python_exec }}-dev --yes
+    
+    curl {{ versioned_pip_url.get(python_exec, default_pip_url) }} -o get-pip.py
+    {{ python_exec }} get-pip.py --force-reinstall
+  when: python_exec is defined
+
+- name: Install python tox
+  become: True
+  become_user: root
+  shell: "{{ python_exec }} -m pip install tox"
+
+
+- name: Install system deps
+  include_role:
+    name: bindep

tests/ci/playbooks/roles/rally-tox/tasks/main.yaml

@@ -0,0 +1 @@
+- include_tasks: "{{ action }}.yaml"

tests/ci/playbooks/roles/rally-tox/tasks/run.yaml

@@ -0,0 +1,5 @@
+---
+- name: Run tox
+  args:
+    chdir: "{{ zuul.project.src_dir }}"
+  command: "tox -e {{ tox_env }}"

tests/ci/playbooks/tox-install.yaml

@@ -1,70 +1,6 @@
-
 - hosts: all
-  name: Installs all required packages
   tasks:
-    - name: Check required version of Python
+    - include_role:
-      args:
+        name: "rally-tox"
-        chdir: "{{ zuul.project.src_dir }}"
+      vars:
-      shell:
+        action: "install"
-        executable: /bin/sh
-        cmd: |
-          set -e
-
-          iniget(){
-              local xtrace
-              xtrace=$(set +o | grep xtrace)
-              set +o xtrace
-              local section=$1
-              local file="tox.ini"
-              local option="basepython"
-              local line
-
-              line=$(sed -ne "/^\[$section\]/,/^\[.*\]/ { /^$option[ \t]*=/ p; }" "$file")
-              echo ${line#*= python}
-              $xtrace
-          }
-
-          tox_testenv_python=$(iniget 'testenv:{{ tox_env }}')
-          if [ "$tox_testenv_python" != "" ]; then
-              echo $tox_testenv_python
-          else
-              echo $(iniget 'testenv')
-          fi
-      register: python_version
-
-    - name: Install the proper python version
-      become: True
-      become_user: root
-      shell:
-        executable: /bin/sh
-        cmd: |
-          set -e
-
-          apt-get update
-          apt-get install python{{ python_version.stdout }}-dev --yes
-
-    - name: Install the proper python pip version
-      become: True
-      become_user: root
-      shell:
-        executable: /bin/bash
-        cmd: |
-          set -e
-          pip_url="https://bootstrap.pypa.io/get-pip.py"
-          if [ "{{ python_version.stdout }}" = "3.6" ];  then
-              pip_url="https://bootstrap.pypa.io/pip/3.6/get-pip.py"
-          elif [ "{{ python_version.stdout }}" = "3" ];  then
-              pip_url="https://bootstrap.pypa.io/pip/3.6/get-pip.py"
-          fi
-          curl $pip_url -o get-pip.py
-          python{{ python_version.stdout }} get-pip.py --force-reinstall
-
-    - name: Install python tox
-      become: True
-      become_user: root
-      shell:
-        executable: /bin/bash
-        cmd: |
-          python{{ python_version.stdout }} -m pip install tox
-  roles:
-    - bindep

tests/ci/playbooks/tox-run.yaml

@@ -1,6 +1,6 @@
 - hosts: all
   tasks:
-    - name: Run tox
+    - include_role:
-      args:
+        name: "rally-tox"
-        chdir: "{{ zuul.project.src_dir }}"
+      vars:
-      command: "tox -e{{ tox_env }}"
+        action: "run"

tests/hacking/checks.py

@@ -24,7 +24,6 @@ Guidelines for writing new hacking checks
 
 """
 
-import functools
 import re
 import tokenize
 
@@ -77,22 +76,6 @@ re_datetime_alias = re.compile(r"^(from|import) datetime(?!\s+as\s+dt$)")
 re_log_warn = re.compile(r"(.)*LOG\.(warn)\(\s*('|\"|_)")
 
 
-def skip_ignored_lines(func):
-
-    @functools.wraps(func)
-    def wrapper(physical_line, logical_line, filename):
-        line = physical_line.strip()
-        if not line or line.startswith("#") or line.endswith("# noqa"):
-            return
-        try:
-            for res in func(physical_line, logical_line, filename):
-                yield res
-        except StopIteration:
-            return
-
-    return wrapper
-
-
 def _parse_assert_mock_str(line):
     point = line.find(".assert_")
 
@@ -107,8 +90,7 @@ def _parse_assert_mock_str(line):
 
 
 @core.flake8ext
-@skip_ignored_lines
+def check_assert_methods_from_mock(logical_line, filename, noqa=False):
-def check_assert_methods_from_mock(physical_line, logical_line, filename):
     """Ensure that ``assert_*`` methods from ``mock`` library is used correctly
 
     N301 - base error number
@@ -116,6 +98,8 @@ def check_assert_methods_from_mock(physical_line, logical_line, filename):
     N303 - related to nonexistent "assert_called_once"
     N304 - related to nonexistent "called_once_with"
     """
+    if noqa:
+        return
 
     correct_names = ["assert_any_call", "assert_called_once_with",
                      "assert_called_with", "assert_has_calls",
@@ -161,12 +145,13 @@ def check_assert_methods_from_mock(physical_line, logical_line, filename):
 
 
 @core.flake8ext
-@skip_ignored_lines
+def check_import_of_logging(logical_line, filename, noqa=False):
-def check_import_of_logging(physical_line, logical_line, filename):
     """Check correctness import of logging module
 
     N310
     """
+    if noqa:
+        return
 
     excluded_files = ["./rally/common/logging.py",
                       "./tests/unit/common/test_logging.py",
@@ -185,12 +170,13 @@ def check_import_of_logging(physical_line, logical_line, filename):
 
 
 @core.flake8ext
-@skip_ignored_lines
+def check_import_of_config(logical_line, filename, noqa=False):
-def check_import_of_config(physical_line, logical_line, filename):
     """Check correctness import of config module
 
     N311
     """
+    if noqa:
+        return
 
     excluded_files = ["./rally/common/cfg.py"]
 
@@ -205,8 +191,7 @@ def check_import_of_config(physical_line, logical_line, filename):
 
 
 @core.flake8ext
-@skip_ignored_lines
+def no_use_conf_debug_check(logical_line, filename, noqa=False):
-def no_use_conf_debug_check(physical_line, logical_line, filename):
     """Check for "cfg.CONF.debug"
 
     Rally has two DEBUG level:
@@ -216,6 +201,8 @@ def no_use_conf_debug_check(physical_line, logical_line, filename):
 
     N312
     """
+    if noqa:
+        return
     excluded_files = ["./rally/common/logging.py"]
 
     point = logical_line.find("CONF.debug")
@@ -226,36 +213,39 @@ def no_use_conf_debug_check(physical_line, logical_line, filename):
 
 
 @core.flake8ext
-@skip_ignored_lines
+def assert_true_instance(logical_line, noqa=False):
-def assert_true_instance(physical_line, logical_line, filename):
     """Check for assertTrue(isinstance(a, b)) sentences
 
     N320
     """
+    if noqa:
+        return
     if re_assert_true_instance.match(logical_line):
         yield (0, "N320 assertTrue(isinstance(a, b)) sentences not allowed, "
                   "you should use assertIsInstance(a, b) instead.")
 
 
 @core.flake8ext
-@skip_ignored_lines
+def assert_equal_type(logical_line, noqa=False):
-def assert_equal_type(physical_line, logical_line, filename):
     """Check for assertEqual(type(A), B) sentences
 
     N321
     """
+    if noqa:
+        return
     if re_assert_equal_type.match(logical_line):
         yield (0, "N321 assertEqual(type(A), B) sentences not allowed, "
                   "you should use assertIsInstance(a, b) instead.")
 
 
 @core.flake8ext
-@skip_ignored_lines
+def assert_equal_none(logical_line, noqa=False):
-def assert_equal_none(physical_line, logical_line, filename):
     """Check for assertEqual(A, None) or assertEqual(None, A) sentences
 
     N322
     """
+    if noqa:
+        return
     res = (re_assert_equal_start_with_none.search(logical_line)
            or re_assert_equal_end_with_none.search(logical_line))
     if res:
@@ -265,8 +255,7 @@ def assert_equal_none(physical_line, logical_line, filename):
 
 
 @core.flake8ext
-@skip_ignored_lines
+def assert_true_or_false_with_in(logical_line, noqa=False):
-def assert_true_or_false_with_in(physical_line, logical_line, filename):
     """Check assertTrue/False(A in/not in B) with collection contents
 
     Check for assertTrue/False(A in B), assertTrue/False(A not in B),
@@ -275,6 +264,8 @@ def assert_true_or_false_with_in(physical_line, logical_line, filename):
 
     N323
     """
+    if noqa:
+        return
     res = (re_assert_true_false_with_in_or_not_in.search(logical_line)
            or re_assert_true_false_with_in_or_not_in_spaces.search(
                logical_line))
@@ -285,8 +276,7 @@ def assert_true_or_false_with_in(physical_line, logical_line, filename):
 
 
 @core.flake8ext
-@skip_ignored_lines
+def assert_equal_in(logical_line, noqa=False):
-def assert_equal_in(physical_line, logical_line, filename):
     """Check assertEqual(A in/not in B, True/False) with collection contents
 
     Check for assertEqual(A in B, True/False), assertEqual(True/False, A in B),
@@ -295,6 +285,8 @@ def assert_equal_in(physical_line, logical_line, filename):
 
     N324
     """
+    if noqa:
+        return
     res = (re_assert_equal_in_end_with_true_or_false.search(logical_line)
            or re_assert_equal_in_start_with_true_or_false.search(logical_line))
     if res:
@@ -304,12 +296,13 @@ def assert_equal_in(physical_line, logical_line, filename):
 
 
 @core.flake8ext
-@skip_ignored_lines
+def assert_not_equal_none(logical_line, noqa=False):
-def assert_not_equal_none(physical_line, logical_line, filename):
     """Check for assertNotEqual(A, None) or assertEqual(None, A) sentences
 
     N325
     """
+    if noqa:
+        return
     res = (re_assert_not_equal_start_with_none.search(logical_line)
            or re_assert_not_equal_end_with_none.search(logical_line))
     if res:
@@ -319,8 +312,7 @@ def assert_not_equal_none(physical_line, logical_line, filename):
 
 
 @core.flake8ext
-@skip_ignored_lines
+def assert_equal_true_or_false(logical_line, noqa=False):
-def assert_equal_true_or_false(physical_line, logical_line, filename):
     """Check for assertEqual(A, True/False) sentences
 
     Check for assertEqual(A, True/False) sentences or
@@ -328,6 +320,8 @@ def assert_equal_true_or_false(physical_line, logical_line, filename):
 
     N326
     """
+    if noqa:
+        return
     res = (re_assert_equal_end_with_true_or_false.search(logical_line)
            or re_assert_equal_start_with_true_or_false.search(logical_line))
     if res:
@@ -337,9 +331,7 @@ def assert_equal_true_or_false(physical_line, logical_line, filename):
 
 
 @core.flake8ext
-@skip_ignored_lines
+def check_no_direct_rally_objects_import(logical_line, filename, noqa=False):
-def check_no_direct_rally_objects_import(physical_line, logical_line,
-                                         filename):
     """Check if rally.common.objects are properly imported.
 
     If you import "from rally.common import objects" you are able to use
@@ -347,10 +339,9 @@ def check_no_direct_rally_objects_import(physical_line, logical_line,
 
     N340
     """
-    if filename == "./rally/common/objects/__init__.py":
+    if noqa:
         return
-
+    if filename == "./rally/common/objects/__init__.py":
-    if filename == "./rally/common/objects/endpoint.py":
         return
 
     if (logical_line.startswith("from rally.common.objects")
@@ -361,8 +352,7 @@ def check_no_direct_rally_objects_import(physical_line, logical_line,
 
 
 @core.flake8ext
-@skip_ignored_lines
+def check_no_oslo_deprecated_import(logical_line, noqa=False):
-def check_no_oslo_deprecated_import(physical_line, logical_line, filename):
     """Check if oslo.foo packages are not imported instead of oslo_foo ones.
 
     Libraries from oslo.foo namespace are deprecated because of namespace
@@ -370,6 +360,8 @@ def check_no_oslo_deprecated_import(physical_line, logical_line, filename):
 
     N341
     """
+    if noqa:
+        return
     if (logical_line.startswith("from oslo.")
        or logical_line.startswith("import oslo.")):
         yield (0, "N341: Import oslo module: `from oslo_xyz import ...`. "
@@ -378,12 +370,13 @@ def check_no_oslo_deprecated_import(physical_line, logical_line, filename):
 
 
 @core.flake8ext
-@skip_ignored_lines
+def check_quotes(logical_line, noqa=False):
-def check_quotes(physical_line, logical_line, filename):
     """Check that single quotation marks are not used
 
     N350
     """
+    if noqa:
+        return
 
     in_string = False
     in_multiline_string = False
@@ -432,12 +425,13 @@ def check_quotes(physical_line, logical_line, filename):
 
 
 @core.flake8ext
-@skip_ignored_lines
+def check_no_constructor_data_struct(logical_line, noqa=False):
-def check_no_constructor_data_struct(physical_line, logical_line, filename):
     """Check that data structs (lists, dicts) are declared using literals
 
     N351
     """
+    if noqa:
+        return
 
     match = re_no_construct_dict.search(logical_line)
     if match:
@@ -448,17 +442,12 @@ def check_no_constructor_data_struct(physical_line, logical_line, filename):
 
 
 @core.flake8ext
-def check_dict_formatting_in_string(logical_line, tokens):
+def check_dict_formatting_in_string(logical_line, tokens, noqa=False):
     """Check that strings do not use dict-formatting with a single replacement
 
     N352
     """
-    # NOTE(stpierre): Can't use @skip_ignored_lines here because it's
+    if noqa:
-    # a stupid decorator that only works on functions that take
-    # (logical_line, filename) as arguments.
-    if (not logical_line
-            or logical_line.startswith("#")
-            or logical_line.endswith("# noqa")):
         return
 
     current_string = ""
@@ -516,40 +505,30 @@ def check_dict_formatting_in_string(logical_line, tokens):
 
 
 @core.flake8ext
-@skip_ignored_lines
+def check_raises(logical_line, filename, noqa=False):
-def check_using_unicode(physical_line, logical_line, filename):
-    """Check crosspython unicode usage
-
-    N353
-    """
-
-    if re.search(r"\bunicode\(", logical_line):
-        yield (0, "N353 'unicode' function is absent in python3. Please "
-                  "use 'str' instead.")
-
-
-@core.flake8ext
-def check_raises(physical_line, logical_line, filename):
     """Check raises usage
 
     N354
     """
+    if noqa:
+        return
 
     ignored_files = ["./tests/unit/test_hacking.py",
                      "./tests/hacking/checks.py"]
     if filename not in ignored_files:
-        if re_raises.search(physical_line):
+        if re_raises.search(logical_line):
             yield (0, "N354 ':Please use ':raises Exception: conditions' "
                       "in docstrings.")
 
 
 @core.flake8ext
-@skip_ignored_lines
+def check_old_type_class(logical_line, noqa=False):
-def check_old_type_class(physical_line, logical_line, filename):
     """Use new-style Python classes
 
     N355
     """
+    if noqa:
+        return
 
     if re_old_type_class.search(logical_line):
         yield (0, "N355 This class does not inherit from anything and thus "
@@ -558,23 +537,25 @@ def check_old_type_class(physical_line, logical_line, filename):
 
 
 @core.flake8ext
-@skip_ignored_lines
+def check_datetime_alias(logical_line, noqa=False):
-def check_datetime_alias(physical_line, logical_line, filename):
     """Ensure using ``dt`` as alias for ``datetime``
 
     N356
     """
+    if noqa:
+        return
     if re_datetime_alias.search(logical_line):
         yield 0, "N356 Please use ``dt`` as alias for ``datetime``."
 
 
 @core.flake8ext
-@skip_ignored_lines
+def check_db_imports_in_cli(logical_line, filename, noqa=False):
-def check_db_imports_in_cli(physical_line, logical_line, filename):
     """Ensure that CLI modules do not use ``rally.common.db``
 
     N360
     """
+    if noqa:
+        return
     if (not filename.startswith("./rally/cli")
             or filename == "./rally/cli/commands/db.py"):
         return
@@ -584,8 +565,7 @@ def check_db_imports_in_cli(physical_line, logical_line, filename):
 
 
 @core.flake8ext
-@skip_ignored_lines
+def check_objects_imports_in_cli(logical_line, filename):
-def check_objects_imports_in_cli(physical_line, logical_line, filename):
     """Ensure that CLI modules do not use ``rally.common.objects``
 
     N361
@@ -598,19 +578,19 @@ def check_objects_imports_in_cli(physical_line, logical_line, filename):
 
 
 @core.flake8ext
-@skip_ignored_lines
+def check_log_warn(logical_line):
-def check_log_warn(physical_line, logical_line, filename):
     if re_log_warn.search(logical_line):
         yield 0, "N313 LOG.warn is deprecated, please use LOG.warning"
 
 
 @core.flake8ext
-@skip_ignored_lines
+def check_opts_import_path(logical_line, filename, noqa=False):
-def check_opts_import_path(physical_line, logical_line, filename):
     """Ensure that we load opts from correct paths only
 
-     N342
+    N342
-     """
+    """
+    if noqa:
+        return
     excluded_files = ["./rally/task/engine.py",
                       "./rally/task/context.py",
                       "./rally/task/scenario.py",

tests/unit/task/test_engine.py

@@ -705,7 +705,7 @@ class ResultConsumerTestCase(test.TestCase):
         runner = mock.MagicMock(result_queue=False)
 
         is_done = mock.MagicMock()
-        is_done.isSet.side_effect = (False, True)
+        is_done.is_set.side_effect = (False, True)
 
         task = mock.MagicMock()
         mock_task_get_status.return_value = consts.TaskStatus.ABORTED
@@ -923,7 +923,7 @@ class ResultConsumerTestCase(test.TestCase):
                                             consts.TaskStatus.ABORTING)
         mock_is_done = mock.MagicMock()
         mock_event.return_value = mock_is_done
-        mock_is_done.isSet.return_value = False
+        mock_is_done.is_set.return_value = False
         ctx_manager = mock.MagicMock()
 
         res = engine.ResultConsumer(workload_cfg, task=task, subtask=subtask,
@@ -954,7 +954,7 @@ class ResultConsumerTestCase(test.TestCase):
         mock_event.return_value = mock_is_done
         ctx_manager = mock.MagicMock()
 
-        mock_is_done.isSet.side_effect = [False, False, False, False, True]
+        mock_is_done.is_set.side_effect = [False, False, False, False, True]
 
         res = engine.ResultConsumer(workload_cfg, task=task, subtask=subtask,
                                     workload=workload, runner=runner,

tests/unit/test_hacking.py

@@ -10,6 +10,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import inspect
 import io
 import tokenize
 
@@ -23,12 +24,23 @@ from tests.unit import test
 class HackingTestCase(test.TestCase):
 
     def _assert_good_samples(self, checker, samples, module_file="f"):
+        spec = inspect.getfullargspec(checker)
+        base_args = {}
+        if "filename" in spec.args:
+            base_args["filename"] = module_file
         for s in samples:
-            self.assertEqual([], list(checker(s, s, module_file)), s)
+            args = {"logical_line": s, **base_args}
+            self.assertEqual([], list(checker(*args)), s)
 
     def _assert_bad_samples(self, checker, samples, module_file="f"):
+        spec = inspect.getfullargspec(checker)
+        base_args = {}
+        if "filename" in spec.args:
+            base_args["filename"] = module_file
+
         for s in samples:
-            self.assertEqual(1, len(list(checker(s, s, module_file))), s)
+            args = {"logical_line": s, **base_args}
+            self.assertEqual(1, len(list(checker(**args))), s)
 
     def test__parse_assert_mock_str(self):
         pos, method, obj = checks._parse_assert_mock_str(
@@ -43,21 +55,6 @@ class HackingTestCase(test.TestCase):
         self.assertIsNone(method)
         self.assertIsNone(obj)
 
-    @ddt.data(
-        {"line": "fdafadfdas  # noqa", "result": []},
-        {"line": "  # fdafadfdas", "result": []},
-        {"line": "  ", "result": []},
-        {"line": "otherstuff", "result": [42]}
-    )
-    @ddt.unpack
-    def test_skip_ignored_lines(self, line, result):
-
-        @checks.skip_ignored_lines
-        def any_gen(physical_line, logical_line, file_name):
-            yield 42
-
-        self.assertEqual(result, list(any_gen(line, line, "f")))
-
     def test_correct_usage_of_assert_from_mock(self):
         correct_method_names = ["assert_any_call", "assert_called_once_with",
                                 "assert_called_with", "assert_has_calls"]
@@ -71,7 +68,7 @@ class HackingTestCase(test.TestCase):
         fake_method = "rtfm.assert_something()"
 
         actual_number, actual_msg = next(checks.check_assert_methods_from_mock(
-            fake_method, fake_method, "./tests/fake/test"))
+            fake_method, "./tests/fake/test"))
         self.assertEqual(4, actual_number)
         self.assertTrue(actual_msg.startswith("N301"))
 
@@ -79,7 +76,7 @@ class HackingTestCase(test.TestCase):
         fake_method = "rtfm.assert_called()"
 
         actual_number, actual_msg = next(checks.check_assert_methods_from_mock(
-            fake_method, fake_method, "./tests/fake/test"))
+            fake_method, "./tests/fake/test", False))
         self.assertEqual(4, actual_number)
         self.assertTrue(actual_msg.startswith("N302"))
 
@@ -87,7 +84,7 @@ class HackingTestCase(test.TestCase):
         fake_method = "rtfm.assert_called_once()"
 
         actual_number, actual_msg = next(checks.check_assert_methods_from_mock(
-            fake_method, fake_method, "./tests/fake/test"))
+            fake_method, "./tests/fake/test", False))
         self.assertEqual(4, actual_number)
         self.assertTrue(actual_msg.startswith("N303"))
 
@@ -100,16 +97,16 @@ class HackingTestCase(test.TestCase):
                         "import rally.common.logging"]
 
         for bad in bad_imports:
-            checkres = checks.check_import_of_logging(bad, bad, "fakefile")
+            checkres = checks.check_import_of_logging(bad, "fakefile")
             self.assertIsNotNone(next(checkres))
 
         for bad in bad_imports:
             checkres = checks.check_import_of_logging(
-                bad, bad, "./rally/common/logging.py")
+                bad, "./rally/common/logging.py")
             self.assertEqual([], list(checkres))
 
         for good in good_imports:
-            checkres = checks.check_import_of_logging(good, good, "fakefile")
+            checkres = checks.check_import_of_logging(good, "fakefile")
             self.assertEqual([], list(checkres))
 
     def test_no_use_conf_debug_check(self):
@@ -134,8 +131,7 @@ class HackingTestCase(test.TestCase):
     )
     @ddt.unpack
     def test_assert_true_instance(self, line, result):
-        self.assertEqual(
+        self.assertEqual(result, len(list(checks.assert_true_instance(line))))
-            result, len(list(checks.assert_true_instance(line, line, "f"))))
 
     @ddt.data(
         {
@@ -149,8 +145,7 @@ class HackingTestCase(test.TestCase):
     )
     @ddt.unpack
     def test_assert_equal_type(self, line, result):
-        self.assertEqual(result,
+        self.assertEqual(result, len(list(checks.assert_equal_type(line))))
-                         len(list(checks.assert_equal_type(line, line, "f"))))
 
     @ddt.data(
         {"line": "self.assertEqual(A, None)", "result": 1},
@@ -160,8 +155,7 @@ class HackingTestCase(test.TestCase):
     @ddt.unpack
     def test_assert_equal_none(self, line, result):
 
-        self.assertEqual(result,
+        self.assertEqual(result, len(list(checks.assert_equal_none(line))))
-                         len(list(checks.assert_equal_none(line, line, "f"))))
 
     @ddt.data(
         {"line": "self.assertNotEqual(A, None)", "result": 1},
@@ -171,9 +165,7 @@ class HackingTestCase(test.TestCase):
     @ddt.unpack
     def test_assert_not_equal_none(self, line, result):
 
-        self.assertEqual(result,
+        self.assertEqual(result, len(list(checks.assert_not_equal_none(line))))
-                         len(list(checks.assert_not_equal_none(line,
-                                                               line, "f"))))
 
     def test_assert_true_or_false_with_in_or_not_in(self):
         good_lines = [
@@ -334,16 +326,6 @@ class HackingTestCase(test.TestCase):
                 [],
                 list(checks.check_dict_formatting_in_string(sample, tokens)))
 
-    @ddt.data(
-        "text = unicode('sometext')",
-        "text = process(unicode('sometext'))"
-    )
-    def test_check_using_unicode(self, line):
-
-        checkres = checks.check_using_unicode(line, line, "fakefile")
-        self.assertIsNotNone(next(checkres))
-        self.assertEqual([], list(checkres))
-
     def test_check_raises(self):
         self._assert_bad_samples(
             checks.check_raises,
@@ -357,21 +339,17 @@ class HackingTestCase(test.TestCase):
     def test_check_db_imports_of_cli(self):
         line = "from rally.common import db"
 
-        next(checks.check_db_imports_in_cli(
+        next(checks.check_db_imports_in_cli(line, "./rally/cli/filename"))
-            line, line, "./rally/cli/filename"))
 
-        checkres = checks.check_db_imports_in_cli(
+        checkres = checks.check_db_imports_in_cli(line, "./filename")
-            line, line, "./filename")
         self.assertRaises(StopIteration, next, checkres)
 
     def test_check_objects_imports_of_cli(self):
         line = "from rally.common import objects"
 
-        next(checks.check_objects_imports_in_cli(
+        next(checks.check_objects_imports_in_cli(line, "./rally/cli/filename"))
-            line, line, "./rally/cli/filename"))
 
-        checkres = checks.check_objects_imports_in_cli(
+        checkres = checks.check_objects_imports_in_cli(line, "./filename")
-            line, line, "./filename")
         self.assertRaises(StopIteration, next, checkres)
 
     @ddt.data(
@@ -379,7 +357,7 @@ class HackingTestCase(test.TestCase):
         "class Oldstyle:"
     )
     def test_check_old_type_class(self, line):
-        checkres = checks.check_old_type_class(line, line, "fakefile")
+        checkres = checks.check_old_type_class(line)
         self.assertIsNotNone(next(checkres))
         self.assertEqual([], list(checkres))
 
@@ -390,12 +368,12 @@ class HackingTestCase(test.TestCase):
                  "from datetime import datetime as dtime"]
 
         for line in lines:
-            checkres = checks.check_datetime_alias(line, line, "fakefile")
+            checkres = checks.check_datetime_alias(line)
             self.assertIsNotNone(next(checkres))
             self.assertEqual([], list(checkres))
 
         line = "import datetime as dt"
-        checkres = checks.check_datetime_alias(line, line, "fakefile")
+        checks.check_datetime_alias(line)
 
     def test_check_log_warn(self):
         bad_samples = ["LOG.warn('foo')", "LOG.warn(_('bar'))"]

tox.ini

@@ -6,7 +6,6 @@ envlist = py36,py37,py38,pep8,samples
 [testenv]
 extras = {env:RALLY_EXTRAS:}
 setenv = VIRTUAL_ENV={envdir}
-         HOME={homedir}
          LANG=en_US.UTF-8
          LANGUAGE=en_US:en
          LC_ALL=C
@@ -15,6 +14,7 @@ setenv = VIRTUAL_ENV={envdir}
 allowlist_externals = find
                       rm
                       make
+                      mkdir
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
        -c{toxinidir}/upper-constraints.txt
@@ -23,7 +23,7 @@ commands =
   find . -type f -name "*.pyc" -delete
   python {toxinidir}/tests/ci/pytest_launcher.py tests/unit --posargs={posargs}
 distribute = false
-basepython = python3.6
+basepython = python3
 passenv =
           PYTEST_REPORT
           http_proxy
@@ -32,6 +32,7 @@ passenv =
           HTTPS_PROXY
           no_proxy
           NO_PROXY
+          HOME
 
 [testenv:pep8]
 commands = flake8
@@ -66,15 +67,6 @@ commands =
   find . -type f -name "*.pyc" -delete
   python {toxinidir}/tests/ci/pytest_launcher.py tests/functional --posargs={posargs}
 
-[testenv:functional-py38]
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
-       stestr
-basepython = python3.8
-commands =
-  find . -type f -name "*.pyc" -delete
-  python {toxinidir}/tests/ci/pytest_launcher.py tests/functional --posargs={posargs}
-
 [testenv:cover]
 commands = {toxinidir}/tests/ci/cover.sh {posargs}
 allowlist_externals = {toxinidir}/tests/ci/cover.sh
@@ -126,7 +118,6 @@ extension =
     N350 = checks:check_quotes
     N351 = checks:check_no_constructor_data_struct
     N352 = checks:check_dict_formatting_in_string
-    N353 = checks:check_using_unicode
     N354 = checks:check_raises
     N355 = checks:check_old_type_class
     N356 = checks:check_datetime_alias
@@ -177,5 +168,8 @@ filterwarnings =
     ignore:Using or importing the ABCs:DeprecationWarning:unittest2.*
     # python 3.8
     ignore:::.*netaddr.strategy.*
+    # python 3.10
+    ignore:The distutils package is deprecated and slated for removal in Python 3.12. Use setuptools or check PEP 632 for potential alternatives:DeprecationWarning:
+    ignore:pkg_resources is deprecated as an API:DeprecationWarning:
     # pytest-cov
     ignore:The --rsyncdir command line argument and rsyncdirs config variable are deprecated.:DeprecationWarning: