[ci] Include pytest html reports for functional job

Change-Id: I35c68ab5ad3ff9ac2f12f383489a18d55d2d0bf1

.zuul.d/base.yaml

@@ -17,5 +17,7 @@
         KEYSTONE_ADMIN_ENDPOINT: true
       devstack_plugins:
         rally-openstack: https://opendev.org/openstack/rally-openstack
+      rally_home_dir: '/opt/stack/.rally'
+      rally_results_dir: '/opt/stack/.rally/results'
     run: tests/ci/playbooks/run-rally-task.yaml
     post-run: tests/ci/playbooks/post-rally-task.yaml

.zuul.d/docker-jobs.yaml

@@ -40,6 +40,7 @@
       docker_repository: "xrally/xrally-openstack"
       rally_plugin_name: "Dummy.openstack"
       rally_package_name: "rally-openstack"
+      rally_results_dir: "{{ zuul.project.src_dir }}/.test_results/"
 
 - job:
     name: rally-openstack-docker-build-and-push
@@ -59,3 +60,4 @@
       docker_repository: "xrally/xrally-openstack"
       rally_plugin_name: "Dummy.openstack"
       rally_package_name: "rally-openstack"
+      rally_results_dir: "{{ zuul.project.src_dir }}/.test_results/"

.zuul.d/rally-tox-functional.yaml

@@ -20,4 +20,9 @@
       devstack_plugins:
         rally-openstack: https://opendev.org/openstack/rally-openstack
       tox_env: "functional"
+      rally_home_dir: '/opt/stack/.rally'
+      rally_results_dir: '/opt/stack/.rally/results/'
+      html_report: 'tox_func_report.html'
+      non_default_html_report: 'tox_func_report.html'
     run: tests/ci/playbooks/tox-functional-env-run.yaml
+    post-run: tests/ci/playbooks/fetch-html-and-json-reports.yaml

tests/ci/playbooks/fetch-html-and-json-reports.yaml

@@ -1,10 +1,9 @@
 - hosts: all
   vars:
-    results_dir: "{{ zuul.project.src_dir }}/.test_results/"
+    html_report: "{% if non_default_html_report is defined %}{{ non_default_html_report }}{% else %}{{ tox_env | default('self') }}_report.html{% endif %}"
-    html_report: "{{ tox_env | default('self') }}_report.html"
     json_report: "{{ tox_env | default('self') }}_report.json"
   tasks:
-    - shell: "ls {{ results_dir }}"
+    - shell: "ls {{ rally_results_dir }}"
       register: results_dir_stat
       ignore_errors: True
 
@@ -12,7 +11,7 @@
       become: yes
       when: results_dir_stat.rc == 0
       synchronize:
-        src: "{{ results_dir }}"
+        src: "{{ rally_results_dir }}{% if not rally_results_dir.endswith('/')%}/{% endif %}"
         dest: '{{ zuul.executor.log_root }}/'
         mode: pull
         copy_links: true
@@ -23,20 +22,20 @@
         - --exclude=*
         - --prune-empty-dirs
 
-    - name: Return artifact to Zuul
+    - name: "Return artifact to Zuul - {{ html_report }}"
-      when: html_report in results_dir_stat.stdout
       zuul_return:
         data:
           zuul:
             artifacts:
               - name: "HTML report"
                 url: "{{ html_report }}"
+      when: html_report in results_dir_stat.stdout
 
-    - name: Return artifact to Zuul
+    - name: "Return artifact to Zuul - {{ json_report}}"
-      when: json_report in results_dir_stat.stdout
       zuul_return:
         data:
           zuul:
             artifacts:
               - name: "JSON report"
                 url: "{{ json_report }}"
+      when: json_report in results_dir_stat.stdout

tests/ci/playbooks/roles/prepare-for-rally-task/tasks/main.yaml

@@ -11,7 +11,7 @@
   become: True
   become_user: stack
   file:
-    path: '{{ rally_home_dir }}/results'
+    path: '{{ rally_results_dir }}'
     state: directory
     owner: stack
     group: stack
@@ -20,7 +20,7 @@
   become: True
   become_user: stack
   file:
-    path: '{{ rally_home_dir }}/results/{{ RALLY_OSPROFILER_CHART }}'
+    path: '{{ rally_results_dir }}/{{ RALLY_OSPROFILER_CHART }}'
     state: directory
     owner: stack
     group: stack

tests/ci/playbooks/tox-functional-env-run.yaml

@@ -4,49 +4,45 @@
 
 - hosts: controller
   vars:
-    rally_home_dir: '/opt/stack/.rally'
     rally_fake_image_path: '{{ rally_home_dir }}/extra/fake-image.img'
     rally_task_args_file: "100-percent-not-exist-file"
     # this task will not be launched, but we need to specify something real to
     # pass a check at 'prepare-for-rally-task' role.
     rally_task: "rally-jobs/simple-job.yaml"
     pip_install: "/opt/stack/rally-openstack/.tox/{{ tox_env }}/bin/pip install"
+  become: true
+  become_user: stack
   tasks:
     - name: "Trigger prepare tasks of rally-task-at-devstack zuul job"
       import_role:
         name: prepare-for-rally-task
 
-    - block:
+    - name: "Precreate a virtualenv for tox {{ tox_env }}"
+      command: tox -e {{ tox_env }} --notest
+      args:
+        chdir: "/opt/stack/rally-openstack"
 
-      - name: "Precreate a virtualenv for tox {{ tox_env }}"
+    - name: Check rally db connection
-        command: tox -e {{ tox_env }} --notest
+      command: rally db show
-        args:
+      register: rally_db_connection
-          chdir: /opt/stack/rally-openstack
 
-      - name: Check rally db connection
+    # DevStack enables a special tool for MySQL performance counting. Since
-        command: rally db show
+    #   it is enabled globally, rally.conf includes it as well.
-        register: rally_db_connection
+    #   tox -e functional utils reuse the same config file and fails
+    #   if there is no this tool inside the venv
+    - name: Inject dbcounter plugin if needed
+      command: "{{ pip_install }} /opt/stack/devstack/tools/dbcounter"
+      when: "'plugin=dbcounter' in rally_db_connection.stdout"
 
-      # DevStack enables a special tool for MySQL performance counting. Since
+    - name: Install python lib for interacting with MySQL
-      #   it is enabled globally, rally.conf includes it as well.
+      command: "{{ pip_install }} PyMySQL>=0.7.6"
-      #   tox -e functional utils reuse the same config file and fails
+      when: "'pymysql' in rally_db_connection.stdout"
-      #   if there is no this tool inside the venv
-      - name: Inject dbcounter plugin if needed
-        command: "{{ pip_install }} /opt/stack/devstack/tools/dbcounter"
-        when: "'plugin=dbcounter' in rally_db_connection.stdout"
 
-      - name: Install python lib for interacting with MySQL
+    - name: Run tox command
-        command: "{{ pip_install }} PyMySQL>=0.7.6"
+      shell: tox -e {{ tox_env }}
-        when: "'pymysql' in rally_db_connection.stdout"
+      environment:
-
+        REQUESTS_CA_BUNDLE: /opt/stack/data/ca-bundle.pem
-      - name: Run tox command
+        REPORTS_ROOT: "{{ rally_results_dir }}/reports"
-        shell: |
+        PYTEST_REPORT: "{{ rally_results_dir }}/{{ html_report }}"
-          set -e
+      args:
-
+        chdir: "/opt/stack/rally-openstack"
-          export REQUESTS_CA_BUNDLE=/opt/stack/data/ca-bundle.pem
-
-          tox -e {{ tox_env }}
-        args:
-          chdir: /opt/stack/rally-openstack
-      become: True
-      become_user: stack

tests/ci/pytest_launcher.py

@@ -17,9 +17,10 @@ import subprocess
 import sys
 
 
-PYTEST_REPORT = os.environ.get("PYTEST_REPORT",
+TOX_ENV_NAME = os.environ["TOX_ENV_NAME"]
-                               ".test_results/pytest_results.html")
+DEFAULT_REPORT_ROOT = os.environ.get("REPORTS_ROOT", ".test_results")
-TESTR_REPORT = "testr_results.html"
+DEFAULT_REPORT = f"{DEFAULT_REPORT_ROOT}/{TOX_ENV_NAME}_report.html"
+PYTEST_REPORT = os.environ.get("PYTEST_REPORT", DEFAULT_REPORT)
 PYTEST_ARGUMENTS = ("py.test"  # base command
                     " -vv"  # show test names in logs
                     " --html=%(html_report)s"  # html report
@@ -86,17 +87,7 @@ def main(args):
 
     print("Test(s) to launch (pytest format): %s" % path)
 
-    # NOTE(andreykurilin): we cannot publish pytest reports at gates, but we
+    args = PYTEST_ARGUMENTS % {"html_report": PYTEST_REPORT,
-    #   can mask them as testr reports. It looks like a dirty hack and I
-    #   prefer to avoid it, but I see no other solutions at this point.
-
-    # apply dirty hack only in gates.
-    if os.environ.get("ZUUL_PROJECT"):
-        pytest_report = TESTR_REPORT
-    else:
-        pytest_report = PYTEST_REPORT
-
-    args = PYTEST_ARGUMENTS % {"html_report": pytest_report,
                                "path": path,
                                "concurrency": args.concurrency or "auto"}
     try:
@@ -104,15 +95,6 @@ def main(args):
                               stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError:
         # NOTE(andreykurilin): it is ok, since tests can fail.
-        exit_code = 1
-    else:
-        exit_code = 0
-
-    if os.path.exists(pytest_report) and os.environ.get("ZUUL_PROJECT"):
-        subprocess.check_call(["gzip", "-9", "-f", pytest_report],
-                              stderr=subprocess.STDOUT)
-
-    if exit_code == 1:
         error("")
 
 

tox.ini

@@ -33,6 +33,8 @@ passenv =
   NO_PROXY
   REQUESTS_CA_BUNDLE
   HOME
+  REPORTS_ROOT
+  PYTEST_REPORT
 
 [testenv:pep8]
 deps = -r{toxinidir}/test-requirements.txt