Improve integration tests logging

* Add fetch-video-output role based on fetch-tox-output
* Fix test log collection to work with Python3
* Fix page HTML source log

Change-Id: I2e865cc5544d6e77bc3af5e413cbaedceb6df1c1

openstack_dashboard/test/integration_tests/helpers.py

@@ -263,7 +263,7 @@ class BaseTestCase(testtools.TestCase):
         test_log_path = os.path.join(self._test_report_dir, 'test.log')
         with self.log_exception("Attach test log"):
             with open(test_log_path, 'w') as f:
-                f.write(self._log_buffer.getvalue().encode('utf-8'))
+                f.write(self._log_buffer.getvalue())
 
     @contextlib.contextmanager
     def log_exception(self, label):
@@ -304,7 +304,7 @@ class BaseTestCase(testtools.TestCase):
         display html code generated/changed by javascript.
         """
         html_elem = self.driver.find_element_by_tag_name("html")
-        return html_elem.get_attribute("innerHTML").encode("utf-8")
+        return html_elem.get_property("innerHTML")
 
 
 @helpers.pytest_mark('integration')

playbooks/horizon-devstack-integration/post.yaml

@@ -3,3 +3,4 @@
   roles:
     - fetch-tox-output
     - fetch-subunit-output
+    - fetch-video-output

roles/fetch-video-output/README.rst

@@ -0,0 +1,16 @@
+Collect log output from a tox build
+
+**Role Variables**
+
+.. zuul:rolevar:: zuul_work_dir
+   :default: {{ zuul.project.src_dir }}
+
+   Directory tox was run in.
+
+.. zuul:rolevar:: zuul_use_fetch_output
+   :default: false
+
+   Whether to synchronize files to the executor work dir, or to copy them
+   on the test instance.
+   When set to false, the role synchronizes the file to the executor.
+   When set to true, the job needs to use the fetch-output role later.

roles/fetch-video-output/defaults/main.yaml

@@ -0,0 +1,5 @@
+---
+zuul_work_dir: "{{ zuul.project.src_dir }}"
+
+zuul_output_dir: "{{ ansible_user_dir }}/zuul-output"
+zuul_use_fetch_output: "{{ zuul_site_use_fetch_output|default(false) }}"

roles/fetch-video-output/tasks/main.yaml

@@ -0,0 +1,44 @@
+- name: Set tox log path for multiple nodes
+  set_fact:
+    log_path: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}/tox"
+  when: groups['all'] | length > 1
+
+- name: Set tox log path for single node
+  set_fact:
+    log_path: "{{ zuul.executor.log_root }}/tox"
+  when: log_path is not defined
+
+- name: Ensure local tox dir
+  file:
+    path: "{{ log_path }}"
+    state: directory
+  delegate_to: localhost
+
+- name: Copy tox logs
+  copy:
+    dest: "{{ zuul_output_dir }}/logs/tox/"
+    src: "{{ zuul_work_dir }}/openstack_dashboard/test/integration_tests/test_reports"
+    remote_src: true
+  loop: "{{ envlist }}"
+  loop_control:
+    loop_var: zj_testenv
+  # some tox runs may not create a virtualenv and thus have
+  # no ./tox/env directory
+  failed_when: false
+  when: zuul_use_fetch_output
+
+- name: Collect tox logs
+  synchronize:
+    dest: "{{ log_path }}"
+    mode: pull
+    src: "{{ zuul_work_dir }}/openstack_dashboard/test/integration_tests/test_reports"
+    verify_host: true
+    owner: no
+    group: no
+  loop: "{{ envlist }}"
+  loop_control:
+    loop_var: zj_testenv
+  # some tox runs may not create a virtualenv and thus have
+  # no ./tox/env directory
+  failed_when: false
+  when: not zuul_use_fetch_output