From 9ff67887bf848c5966bb4a2f37018500d30dbe45 Mon Sep 17 00:00:00 2001
From: Andrey Kurilin <andr.kurilin@gmail.com>
Date: Sun, 20 Aug 2023 16:36:19 +0200
Subject: [PATCH] Fix and re-enable function tox job

Change-Id: Ie6735bbc9a974af1ac44c9777a1d63e513f36469
---
 .zuul.d/zuul.yaml                             |  4 +-
 .../playbooks/roles/tox-run/tasks/main.yaml   | 10 -----
 .../ci/playbooks/tox-functional-env-run.yaml  | 43 +++++++++++++++++--
 tests/functional/utils.py                     | 13 +++++-
 tox.ini                                       |  2 +
 5 files changed, 55 insertions(+), 17 deletions(-)
 delete mode 100644 tests/ci/playbooks/roles/tox-run/tasks/main.yaml

diff --git a/.zuul.d/zuul.yaml b/.zuul.d/zuul.yaml
index 854090e8..d3a30eb9 100644
--- a/.zuul.d/zuul.yaml
+++ b/.zuul.d/zuul.yaml
@@ -8,7 +8,7 @@
         - rally-tox-py38
         - rally-tox-py39
         - rally-tox-py310
-        #- rally-dsvm-tox-functional
+        - rally-dsvm-tox-functional
         - rally-openstack-docker-build
         #- rally-task-basic-with-existing-users
         - rally-task-simple-job
@@ -90,7 +90,7 @@
         - rally-tox-py38
         - rally-tox-py39
         - rally-tox-py310
-        #- rally-dsvm-tox-functional
+        - rally-dsvm-tox-functional
         - rally-openstack-docker-build
         #- rally-task-basic-with-existing-users
         - rally-task-simple-job
diff --git a/tests/ci/playbooks/roles/tox-run/tasks/main.yaml b/tests/ci/playbooks/roles/tox-run/tasks/main.yaml
deleted file mode 100644
index 6315fe29..00000000
--- a/tests/ci/playbooks/roles/tox-run/tasks/main.yaml
+++ /dev/null
@@ -1,10 +0,0 @@
-- name: Run tox command
-  become: True
-  become_user: stack
-  shell:
-    executable: /bin/sh
-    cmd: |
-      set -e
-      cd /opt/stack/rally-openstack
-      export REQUESTS_CA_BUNDLE=/opt/stack/data/ca-bundle.pem
-      tox -e {{ tox_env }}
diff --git a/tests/ci/playbooks/tox-functional-env-run.yaml b/tests/ci/playbooks/tox-functional-env-run.yaml
index f13c2d84..609a1a12 100644
--- a/tests/ci/playbooks/tox-functional-env-run.yaml
+++ b/tests/ci/playbooks/tox-functional-env-run.yaml
@@ -10,6 +10,43 @@
     # this task will not be launched, but we need to specify something real to
     # pass a check at 'prepare-for-rally-task' role.
     rally_task: "rally-jobs/simple-job.yaml"
-  roles:
-    - prepare-for-rally-task
-    - tox-run
\ No newline at end of file
+    pip_install: "/opt/stack/rally-openstack/.tox/{{ tox_env }}/bin/pip install"
+  tasks:
+    - name: "Trigger prepare tasks of rally-task-at-devstack zuul job"
+      import_role:
+        name: prepare-for-rally-task
+
+    - block:
+
+      - name: "Precreate a virtualenv for tox {{ tox_env }}"
+        command: tox -e {{ tox_env }} --notest
+        args:
+          chdir: /opt/stack/rally-openstack
+
+      - name: Check rally db connection
+        command: rally db show
+        register: rally_db_connection
+
+      # DevStack enables a special tool for MySQL performance counting. Since
+      #   it is enabled globally, rally.conf includes it as well.
+      #   tox -e functional utils reuse the same config file and fails
+      #   if there is no this tool inside the venv
+      - name: Inject dbcounter plugin if needed
+        command: "{{ pip_install }} /opt/stack/devstack/tools/dbcounter"
+        when: "'plugin=dbcounter' in rally_db_connection.stdout"
+
+      - name: Install python lib for interacting with MySQL
+        command: "{{ pip_install }} PyMySQL>=0.7.6"
+        when: "'pymysql' in rally_db_connection.stdout"
+
+      - name: Run tox command
+        shell: |
+          set -e
+
+          export REQUESTS_CA_BUNDLE=/opt/stack/data/ca-bundle.pem
+
+          tox -e {{ tox_env }}
+        args:
+          chdir: /opt/stack/rally-openstack
+      become: True
+      become_user: stack
diff --git a/tests/functional/utils.py b/tests/functional/utils.py
index ddff6652..a20c9ea4 100644
--- a/tests/functional/utils.py
+++ b/tests/functional/utils.py
@@ -76,7 +76,12 @@ class Rally(object):
                                  "env", "show", "--only-spec"],
                                 stdout=f)
         with open(self.ENV_FILE) as f:
-            self.env_spec = json.loads(f.read())
+            data = f.read()
+            try:
+                self.env_spec = json.loads(data)
+            except ValueError:
+                print(f"JSON file that failed to load: {data}")
+                raise
             print(self.env_spec)
 
         # NOTE(sskripnick): we should change home dir to avoid races
@@ -133,7 +138,11 @@ class Rally(object):
              write_report=False)
 
     def __del__(self):
-        shutil.rmtree(self.tmp_dir)
+        # Rally object may fail before initialization of tmp_dir attribute
+        try:
+            shutil.rmtree(self.tmp_dir)
+        except AttributeError:
+            pass
 
     def _safe_make_dirs(self, dirs):
         try:
diff --git a/tox.ini b/tox.ini
index 4d38004f..c7e4c34e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -35,6 +35,8 @@ passenv =
   HOME
 
 [testenv:pep8]
+deps = -r{toxinidir}/test-requirements.txt
+skip_install = true
 commands = flake8
 distribute = false