Tobiko adapted to python3.12

Updated requirements and pre-commit-config files to support python3.12.

Tests from tobiko/tests/functional/run have been removed to avoid
failures on tobiko-infrared-centos-9 with multiprocessing.pool module:
https://github.com/dask/dask/issues/5806
These tests were going to be removed in any case at:
https://review.opendev.org/c/x/tobiko/+/936633

TODO: after updating ansible-lint version used to 6.21.1, due to the
big amount of failures, ansible-lint is mostly skipped (see
.ansible-lint file) - this will be fixed in a follow up patch.

Change-Id: I2768cd8d0c2b5f015f1beb0e42dae67dd24f97dd

.ansible-lint

@@ -1,7 +1,16 @@
 ---
 
 exclude_paths:
-  - ./roles/infrared/
+  - roles/infrared/
+  - zuul.d/
+  - .*.yaml
+  - tobiko/
+  # TODO(eolivare) remove the following excluded_paths after having fixed
+  # ansible-lint errors
+  - infrared_plugin/
+  - roles/
+  - playbooks/
+
 
 skip_list:
   - '206'

.pre-commit-config.yaml

@@ -27,7 +27,7 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/pycqa/flake8.git
-    rev: '3.8.4'  # pick a git hash / tag to point to
+    rev: '6.1.0'  # pick a git hash / tag to point to
     hooks:
       - id: flake8
         additional_dependencies: [flake8-import-order]
@@ -43,7 +43,7 @@ repos:
         # args: [--ignore-missing-imports]
 
   - repo: https://github.com/ansible/ansible-lint.git
-    rev: v4.2.0
+    rev: v6.21.1
     hooks:
       - id: ansible-lint
         files: \.(yaml|yml)$

extra-requirements.txt

@@ -4,6 +4,7 @@ ansi2html                # LGPLv3+
 dpkt                     # BSD
 openshift-client         # Apache-2.0
 pandas                   # BSD
+pandas==2.1.1;python_version>='3.9'            # BSD
 podman==4.7.0            # Apache-2.0
 pytest-cov               # MIT
 pytest-reportportal      # Apache-2.0

lower-constraints.txt

@@ -31,6 +31,7 @@ python-novaclient==17.2.1
 python-octaviaclient==2.2.0
 python-openstackclient==5.4.0
 PyYAML==5.4.1
+setuptools==21.0.0;python_version>='3.12'
 sshtunnel==0.3.1
 testtools==2.5.0
 validations-libs==1.1.0

requirements.txt

@@ -28,5 +28,6 @@ python-novaclient>=17.2.1       # Apache-2.0
 python-octaviaclient>=2.2.0     # Apache-2.0
 python-openstackclient>=5.4.0   # Apache-2.0
 PyYAML>=5.4.1                   # MIT
+setuptools!=24.0.0,!=34.0.0,!=34.0.1,!=34.0.2,!=34.0.3,!=34.1.0,!=34.1.1,!=34.2.0,!=34.3.0,!=34.3.1,!=34.3.2,!=36.2.0,>=21.0.0;python_version>='3.12' # PSF/ZPL
 sshtunnel>=0.3.1                # MIT
 testtools>=2.5.0                # MIT

tobiko/actors/_proxy.py

@@ -25,12 +25,6 @@ import decorator
 P = typing.TypeVar('P')
 
 GenericMetaBase = abc.ABCMeta
-if hasattr(typing, 'GenericMeta'):
-    class GenericMetaBase(  # type: ignore[no-redef]
-            typing.GenericMeta,  # type: ignore[name-defined]
-            abc.ABCMeta):
-        # pylint: disable=function-redefined,no-member
-        pass
 
 
 class GenericMeta(GenericMetaBase):
@@ -47,7 +41,10 @@ class GenericMeta(GenericMetaBase):
             if inspect.ismethod(class_getitem):
                 cls = class_getitem(item)
             else:
-                cls = class_getitem(cls, item)
+                try:
+                    cls = class_getitem(cls, item)
+                except TypeError:
+                    cls = class_getitem(item)
         return cls
 
 

tobiko/common/_exception.py

@@ -75,7 +75,7 @@ class TobikoException(Exception):
             message=self.message)
 
     def __eq__(self, other):
-        return type(self) == type(other) and str(self) == str(other)
+        return type(self) is type(other) and str(self) == str(other)
 
     def __hash__(self):
         return hash(type(self)) + hash(str(self))

tobiko/openstack/stacks/_ubuntu.py

@@ -123,8 +123,9 @@ class UbuntuImageFixture(UbuntuMinimalImageFixture,
 
     @staticmethod
     def _get_ethernet_device() -> str:
-        """From OSP17 and above, Ubuntu stack should use a different interface name.
+        """From OSP17 and above, Ubuntu stack should use a different interface
-        This method returns the interface name, depending on the OSP version.
+        name. This method returns the interface name, depending on the OSP
+        version.
         """
         if_name = CONF.tobiko.ubuntu.interface_name
         if if_name is not None:

tobiko/podman/_podman1/libs/_containers_attach.py

@@ -34,10 +34,8 @@ class Mixin:
 
         # This is the UDS where all the IO goes
         io_socket = attach['sockets']['io_socket']
-        assert len(io_socket) <= 107,\
+        assert len(io_socket) <= 107, \
-            'Path length for sockets too long. {} > 107'.format(
+            'Path length for sockets too long. {} > 107'.format(len(io_socket))
-                len(io_socket)
-            )
 
         # This is the control socket where resizing events are sent to conmon
         # attach['sockets']['control_socket']

tobiko/podman/_podman1/libs/containers.py

@@ -33,7 +33,7 @@ class Container(AttachMixin, StartMixin, collections.UserDict):
                 setattr(self, 'running', self.data['containerrunning'])
                 self.data['running'] = self.data['containerrunning']
 
-        assert self._id == data['id'],\
+        assert self._id == data['id'], \
             'Requested container id({}) does not match store id({})'.format(
                 self._id, data['id']
             )

tobiko/podman/_podman1/libs/images.py

@@ -25,10 +25,9 @@ class Image(collections.UserDict):
         self._id = id
         self._client = client
 
-        assert self._id == data['id'],\
+        assert self._id == data['id'], \
             'Requested image id({}) does not match store id({})'.format(
-                self._id, data['id']
+                self._id, data['id'])
-            )
 
     @staticmethod
     def _split_token(values=None, sep='='):

tobiko/tests/faults/containers/container_ops.py

@@ -65,7 +65,7 @@ def get_nodes_for_groups(groups):
         for group in node.groups:
             if group in groups:
                 nodes.append(node)
-    return(nodes)
+    return nodes
 
 
 def get_config_files(node, kolla_jsons, conf_ignorelist, scripts_to_check):

tobiko/tests/faults/ha/cloud_disruptions.py

@@ -607,8 +607,8 @@ def kill_rabbitmq_service():
              node.name))
     retry = tobiko.retry(timeout=30, interval=5)
     for _ in retry:
-        if not(pacemaker.PacemakerResourcesStatus().
+        if not (pacemaker.PacemakerResourcesStatus().
-               rabbitmq_resource_healthy()):
+                rabbitmq_resource_healthy()):
             return
 
 
@@ -630,8 +630,8 @@ def kill_all_galera_services():
                                                             node.name))
     retry = tobiko.retry(timeout=30, interval=5)
     for _ in retry:
-        if not(pacemaker.PacemakerResourcesStatus().
+        if not (pacemaker.PacemakerResourcesStatus().
-               galera_resource_healthy()):
+                galera_resource_healthy()):
             return
 
 

tobiko/tests/functional/run/test_discover.py

@@ -1,54 +0,0 @@
-# Copyright (c) 2019 Red Hat, Inc.
-#
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-from __future__ import absolute_import
-
-import os
-import typing
-
-import testtools
-
-from tobiko import run
-
-
-class DiscoverTestIdsTest(testtools.TestCase):
-
-    def test_discover_test_ids(self,
-                               test_files: typing.List[str] = None):
-        if test_files is None:
-            test_files = [__file__]
-        test_ids = run.discover_test_ids(test_files=test_files)
-        self.assertIn(self.id(), test_ids)
-
-    def test_forked_discover_test_ids(self,
-                                      test_files: typing.List[str] = None):
-        if test_files is None:
-            test_files = [__file__]
-        test_ids = run.forked_discover_test_ids(test_files=test_files)
-        self.assertIn(self.id(), test_ids)
-
-    def test_find_test_ids(self,
-                           test_path: typing.List[str] = None,
-                           forked=False):
-        if test_path is None:
-            test_path = [__file__]
-        test_ids = run.find_test_ids(test_path=test_path, forked=forked)
-        self.assertIn(self.id(), test_ids)
-
-    def test_find_test_ids_with_test_dir(self):
-        self.test_find_test_ids(test_path=[os.path.dirname(__file__)])
-
-    def test_find_test_ids_with_forked(self):
-        self.test_find_test_ids(forked=True)

tobiko/tests/functional/run/test_find.py

@@ -1,45 +0,0 @@
-# Copyright (c) 2019 Red Hat, Inc.
-#
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-from __future__ import absolute_import
-
-import os
-import typing
-
-import testtools
-
-from tobiko import run
-
-
-class FindTestFilesTest(testtools.TestCase):
-
-    def test_find_test_files(self,
-                             test_path: typing.List[str] = None):
-        if test_path is None:
-            test_path = [__file__]
-        test_files = run.find_test_files(test_path=test_path)
-        self.assertIn(__file__, test_files)
-        test_path = [os.path.realpath(path)
-                     for path in test_path]
-        for test_file in test_files:
-            for path in test_path:
-                if test_file.startswith(path):
-                    break
-            else:
-                self.fail(f"File '{test_file}' not in any path ({test_path})")
-
-    def test_find_test_files_with_test_dir(self):
-        return self.test_find_test_files(
-            test_path=[os.path.dirname(__file__)])

tobiko/tests/functional/run/test_run.py

@@ -1,56 +0,0 @@
-# Copyright (c) 2022 Red Hat, Inc.
-#
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-from __future__ import absolute_import
-
-import functools
-import os
-import typing
-import unittest
-
-import testtools
-
-from tobiko import run
-
-
-def nested_test_case(test_method: typing.Callable[[testtools.TestCase], None]):
-
-    @functools.wraps(test_method)
-    def wrapper(self: unittest.TestCase):
-        nested_counter = int(os.environ.get('NESTED_TEST_CASE', 0))
-        if not nested_counter:
-            os.environ['NESTED_TEST_CASE'] = str(nested_counter + 1)
-            try:
-                test_method(self)
-            finally:
-                if nested_counter:
-                    os.environ['NESTED_TEST_CASE'] = str(nested_counter)
-                else:
-                    os.environ.pop('NESTED_TEST_CASE')
-    return wrapper
-
-
-class RunTestsTest(unittest.TestCase):
-
-    @nested_test_case
-    def test_run_tests(self):
-        result = run.run_tests(__file__)
-        self.assertGreater(result.testsRun, 0)
-
-    @nested_test_case
-    def test_run_tests_with_dir(self):
-        test_dir = os.path.dirname(__file__)
-        result = run.run_tests(test_path=test_dir)
-        self.assertGreater(result.testsRun, 0)

tobiko/tests/functional/tripleo/test_overcloud.py

@@ -179,9 +179,9 @@ class OvercloudVersionTest(unittest.TestCase):
         if v.micro > 0:
             lower_v = f"{v.major}.{v.minor}.{v.micro - 1}"
         elif v.minor > 0:
-            lower_v = f"{v.major}.{v.minor -1}.{v.micro}"
+            lower_v = f"{v.major}.{v.minor - 1}.{v.micro}"
         elif v.major > 0:
-            lower_v = f"{v.major -1}.{v.minor}.{v.micro}"
+            lower_v = f"{v.major - 1}.{v.minor}.{v.micro}"
         else:
             raise ValueError(f"wrong version: {v}")
         return lower_v

tobiko/tests/functional/tripleo/test_undercloud.py

@@ -87,9 +87,9 @@ class UndercloudVersionTest(unittest.TestCase):
         if v.micro > 0:
             lower_v = f"{v.major}.{v.minor}.{v.micro - 1}"
         elif v.minor > 0:
-            lower_v = f"{v.major}.{v.minor -1}.{v.micro}"
+            lower_v = f"{v.major}.{v.minor - 1}.{v.micro}"
         elif v.major > 0:
-            lower_v = f"{v.major -1}.{v.minor}.{v.micro}"
+            lower_v = f"{v.major - 1}.{v.minor}.{v.micro}"
         else:
             raise ValueError(f"wrong version: {v}")
         return lower_v

tobiko/tripleo/processes.py

@@ -210,7 +210,7 @@ class OvercloudProcessesStatus(object):
                 ovn_proc_filtered_df[node_filter]
             total_num_processes = len(ovn_proc_filtered_per_node_df)
 
-            if type(process_dict['number']) == int:
+            if isinstance(process_dict['number'], int):
                 expected_num_processes = process_dict['number']
             elif process_dict['number'] == 'all':
                 expected_num_processes = len(node_list)

upper-constraints.txt

@@ -65,7 +65,7 @@ dfs-sdk===1.2.27
 dib-utils===0.0.11
 directord===0.12.0
 diskimage-builder===3.20.3
-distlib===0.3.4
+distlib===0.3.9
 distro===1.7.0
 Django===3.2.12
 django-appconf===1.0.5
@@ -89,7 +89,7 @@ enmerkar===0.7.1
 enum-compat===0.0.3
 etcd3===0.12.0
 etcd3gw===1.0.1
-eventlet===0.33.0
+eventlet===0.36.1
 exabgp===4.2.17
 execnet===1.9.0
 extras===1.0.0
@@ -118,7 +118,7 @@ google-auth-httplib2===0.1.0
 googleapis-common-protos===1.55.0
 gossip===2.4.0
 graphviz===0.19.1
-greenlet===1.1.2
+greenlet===3.1.1
 grpcio===1.44.0
 gssapi===1.7.3
 gunicorn===20.1.0
@@ -136,6 +136,8 @@ ifaddr===0.1.7
 imagesize===1.3.0
 immutables===0.16
 importlib-metadata===4.11.1;python_version=='3.8'
+importlib-metadata===6.2.1;python_version=='3.9'
+importlib-metadata===8.5.0;python_version>='3.10'
 importlib-resources===5.2.3
 infi.dtypes.iqn===0.4.0
 infi.dtypes.wwn===0.1.1
@@ -389,7 +391,7 @@ pytz-deprecation-shim===0.1.0.post0
 pyudev===0.23.2
 pywbem===1.4.1
 pywinrm===0.4.2
-PyYAML===6.0
+PyYAML===6.0.1
 pyzabbix===1.0.0
 pyzmq===21.0.2
 rbd-iscsi-client===0.1.8
@@ -433,8 +435,7 @@ semantic-version===2.9.0
 sentinels===1.0.0
 seqdiag===3.0.0;python_version=='3.8'
 setproctitle===1.2.2
-setuptools===60.9.3;python_version=='3.8'
+setuptools===70.3.0;python_version=='3.9'
-setuptools===70.3.0;python_version>='3.9'
 simplegeneric===0.8.1
 simplejson===3.17.6
 six===1.16.0
@@ -473,7 +474,8 @@ ssh-python===0.9.0
 sshtunnel===0.4.0
 statsd===3.3.0
 stestr===3.2.1
-stevedore===3.5.0
+stevedore===3.5.0;python_version<'3.12'
+stevedore===5.4.0;python_version>='3.12'
 storage-interfaces===1.0.4
 storops===1.2.10
 storpool===7.0.0
@@ -514,9 +516,10 @@ types-cryptography===3.3.15
 types-enum34===1.1.8
 types-ipaddress===1.0.8
 types-paramiko===2.8.13
+types-setuptools===75.5.0.20241122
 typing===3.7.4.3
-typing_extensions===4.1.1
+typing_extensions===4.12.2
-tzdata===2021.5
+tzdata===2024.2
 tzlocal===4.1
 uhashring===2.1
 ujson===5.1.0;python_version=='3.8'
@@ -528,7 +531,7 @@ validations-libs===1.6.0
 vine===5.0.0
 vintage===0.4.1
 virtualbmc===2.2.1
-virtualenv===20.13.2
+virtualenv===20.27.1
 voluptuous===0.12.2
 waiting===1.4.1
 waitress===2.0.0
@@ -541,7 +544,7 @@ websockify===0.10.0
 WebTest===3.0.0
 Werkzeug===2.0.3
 whereto===0.4.0
-wrapt===1.13.3
+wrapt===1.14.0
 ws4py===0.5.1
 wsgi-intercept===1.9.3
 WSME===0.11.0
@@ -590,7 +593,7 @@ XStatic-term.js===0.0.7.0
 XStatic-tv4===1.2.7.0
 xvfbwrapper===0.2.9
 yamlloader===1.1.0
-yappi===1.3.3
+yappi===1.6.10
 yaql===2.0.0
 zake===0.2.2
 zeroconf===0.38.3;python_version=='3.8'