Merge "Implement video capture for failed tests"

openstack_dashboard/test/integration_tests/decorators.py

@@ -137,3 +137,15 @@ def skip_because(**kwargs):
                               ", ".join([bug for bug in bugs]))
         return obj
     return actual_decoration
+
+
+def attach_video(func):
+    """Notify test runner to attach test video in any case
+    """
+
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwgs):
+        self._need_attach_video = True
+        return func(self, *args, **kwgs)
+
+    return wrapper

openstack_dashboard/test/integration_tests/helpers.py

@@ -13,8 +13,10 @@
 import contextlib
 import logging
 import os
+import shutil
 from six import StringIO
 import socket
+import subprocess
 import tempfile
 import time
 import traceback
@@ -30,11 +32,21 @@ from horizon.test import webdriver
 from openstack_dashboard.test.integration_tests import config
 from openstack_dashboard.test.integration_tests.pages import loginpage
 from openstack_dashboard.test.integration_tests.regions import messages
+from openstack_dashboard.test.integration_tests.video_recorder import \
+    VideoRecorder
 
 LOGGER = logging.getLogger()
 LOGGER.setLevel(logging.DEBUG)
+IS_SELENIUM_HEADLESS = os.environ.get('SELENIUM_HEADLESS', False)
 ROOT_PATH = os.path.dirname(os.path.abspath(config.__file__))
 
+if not subprocess.call('which xdpyinfo > /dev/null 2>&1', shell=True):
+    SCREEN_SIZE = subprocess.check_output('xdpyinfo | grep dimensions',
+                                          shell=True).split()[1].split('x')
+else:
+    SCREEN_SIZE = (None, None)
+    LOGGER.info("X11 isn't installed. Should use xvfb to run tests.")
+
 
 def gen_random_resource_name(resource="", timestamp=True):
     """Generate random resource name using uuid and timestamp.
@@ -83,15 +95,27 @@ class BaseTestCase(testtools.TestCase):
     CONFIG = config.get_config()
 
     def setUp(self):
+        if not os.environ.get('INTEGRATION_TESTS', False):
+            raise self.skipException(
+                "The INTEGRATION_TESTS env variable is not set.")
+
         self._configure_log()
 
-        if not os.environ.get('INTEGRATION_TESTS', False):
-            msg = "The INTEGRATION_TESTS env variable is not set."
-            raise self.skipException(msg)
+        self.addOnException(
+            lambda exc_info: setattr(self, '_need_attach_test_log', True))
 
+        def cleanup():
+            if getattr(self, '_need_attach_test_log', None):
+                self._attach_test_log()
+
+        self.addCleanup(cleanup)
+
+        width, height = SCREEN_SIZE
+        display = '0.0'
         # Start a virtual display server for running the tests headless.
-        if os.environ.get('SELENIUM_HEADLESS', False):
-            self.vdisplay = xvfbwrapper.Xvfb(width=1920, height=1080)
+        if IS_SELENIUM_HEADLESS:
+            width, height = 1920, 1080
+            self.vdisplay = xvfbwrapper.Xvfb(width=width, height=height)
             args = []
 
             # workaround for memory leak in Xvfb taken from:
@@ -107,9 +131,25 @@ class BaseTestCase(testtools.TestCase):
             else:
                 self.vdisplay.xvfb_cmd.extend(args)
             self.vdisplay.start()
+            display = self.vdisplay.new_display
 
             self.addCleanup(self.vdisplay.stop)
 
+        self.video_recorder = VideoRecorder(width, height, display=display)
+        self.video_recorder.start()
+
+        self.addOnException(
+            lambda exc_info: setattr(self, '_need_attach_video', True))
+
+        def cleanup():
+            self.video_recorder.stop()
+            if getattr(self, '_need_attach_video', None):
+                self._attach_video()
+            else:
+                self.video_recorder.clear()
+
+        self.addCleanup(cleanup)
+
         # Increase the default Python socket timeout from nothing
         # to something that will cope with slow webdriver startup times.
         # This *just* affects the communication between this test process
@@ -123,16 +163,24 @@ class BaseTestCase(testtools.TestCase):
         )
         if self.CONFIG.selenium.maximize_browser:
             self.driver.maximize_window()
+            if IS_SELENIUM_HEADLESS:  # force full screen in xvfb
+                self.driver.set_window_size(width, height)
+
         self.driver.implicitly_wait(self.CONFIG.selenium.implicit_wait)
         self.driver.set_page_load_timeout(
             self.CONFIG.selenium.page_timeout)
 
-        self.addCleanup(self.driver.quit)
-
         self.addOnException(self._attach_page_source)
         self.addOnException(self._attach_screenshot)
-        self.addOnException(self._attach_browser_log)
-        self.addOnException(self._attach_test_log)
+        self.addOnException(
+            lambda exc_info: setattr(self, '_need_attach_browser_log', True))
+
+        def cleanup():
+            if getattr(self, '_need_attach_browser_log', None):
+                self._attach_browser_log()
+            self.driver.quit()
+
+        self.addCleanup(cleanup)
 
         super(BaseTestCase, self).setUp()
 
@@ -161,7 +209,8 @@ class BaseTestCase(testtools.TestCase):
     @property
     def _test_report_dir(self):
         report_dir = os.path.join(ROOT_PATH, 'test_reports',
-                                  self._testMethodName)
+                                  '{}.{}'.format(self.__class__.__name__,
+                                                 self._testMethodName))
         if not os.path.isdir(report_dir):
             os.makedirs(report_dir)
         return report_dir
@@ -177,14 +226,24 @@ class BaseTestCase(testtools.TestCase):
         with self.log_exception("Attach screenshot"):
             self.driver.get_screenshot_as_file(screen_path)
 
-    def _attach_browser_log(self, exc_info):
+    def _attach_video(self, exc_info=None):
+        with self.log_exception("Attach video"):
+            if not os.path.isfile(self.video_recorder.file_path):
+                LOGGER.warn("Can't find video {!r}".format(
+                    self.video_recorder.file_path))
+                return
+
+            shutil.move(self.video_recorder.file_path,
+                        os.path.join(self._test_report_dir, 'video.mp4'))
+
+    def _attach_browser_log(self, exc_info=None):
         browser_log_path = os.path.join(self._test_report_dir, 'browser.log')
         with self.log_exception("Attach browser log"):
             with open(browser_log_path, 'w') as f:
                 f.write(
                     self._unwrap_browser_log(self.driver.get_log('browser')))
 
-    def _attach_test_log(self, exc_info):
+    def _attach_test_log(self, exc_info=None):
         test_log_path = os.path.join(self._test_report_dir, 'test.log')
         with self.log_exception("Attach test log"):
             with open(test_log_path, 'w') as f:
@@ -240,7 +299,9 @@ class TestCase(BaseTestCase, AssertsMixin):
         super(TestCase, self).setUp()
         self.login_pg = loginpage.LoginPage(self.driver, self.CONFIG)
         self.login_pg.go_to_login_page()
-        self.zoom_out()
+        # TODO(schipiga): lets check that tests work without viewport changing,
+        # otherwise will uncomment.
+        # self.zoom_out()
         self.home_pg = self.login_pg.login(self.TEST_USER_NAME,
                                            self.TEST_PASSWORD)
         self.home_pg.change_project(self.HOME_PROJECT)

openstack_dashboard/test/integration_tests/video_recorder.py

@@ -0,0 +1,82 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
+import os
+import signal
+import subprocess
+from tempfile import mktemp
+from threading import Thread
+import time
+
+LOGGER = logging.getLogger(__name__)
+
+
+class VideoRecorder(object):
+
+    def __init__(self, width, height, display='0.0', frame_rate=15):
+        self.is_launched = False
+        self.file_path = mktemp() + '.mp4'
+        # avconv -f x11grab -r 15 -s 1920x1080 -i :0.0 -codec libx264 out.mp4
+        self._cmd = ['avconv', '-f', 'x11grab', '-r', str(frame_rate),
+                     '-s', '{}x{}'.format(width, height),
+                     '-i', ':{}'.format(display),
+                     '-codec', 'libx264', self.file_path]
+
+    def start(self):
+        if self.is_launched:
+            LOGGER.warn('Video recording is running already')
+            return
+
+        if not os.environ.get('AVCONV_INSTALLED', False):
+            LOGGER.error("avconv isn't installed. Video recording is skipped")
+            return
+
+        fnull = open(os.devnull, 'w')
+        LOGGER.info('Record video via {!r}'.format(' '.join(self._cmd)))
+        self._popen = subprocess.Popen(self._cmd, stdout=fnull, stderr=fnull)
+        self.is_launched = True
+
+    def stop(self):
+        if not self.is_launched:
+            LOGGER.warn('Video recording is stopped already')
+            return
+
+        self._popen.send_signal(signal.SIGINT)
+
+        def terminate_avconv():
+            limit = time.time() + 10
+
+            while time.time() < limit:
+                time.sleep(0.1)
+                if self._popen.poll() is not None:
+                    return
+
+            os.kill(self._popen.pid, signal.SIGTERM)
+
+        t = Thread(target=terminate_avconv)
+        t.start()
+
+        self._popen.communicate()
+        t.join()
+        self.is_launched = False
+
+    def clear(self):
+        if self.is_launched:
+            LOGGER.error("Video recording is running still")
+            return
+
+        if not os.path.isfile(self.file_path):
+            LOGGER.warn("{!r} is absent already".format(self.file_path))
+            return
+
+        os.remove(self.file_path)

tools/gate/integration/post_test_hook.sh

@@ -4,12 +4,14 @@
 
 set -x
 
+# install avconv to capture video of failed tests
+sudo apt-get install -y libav-tools && export AVCONV_INSTALLED=1
+
 cd /opt/stack/new/horizon
-sudo -H -u stack tox -e py27integration
+sudo -H -E -u stack tox -e py27integration
 retval=$?
 
 if [ -d openstack_dashboard/test/integration_tests/test_reports/ ]; then
   cp -r openstack_dashboard/test/integration_tests/test_reports/ /home/jenkins/workspace/gate-horizon-dsvm-integration/
 fi
 exit $retval
-

tox.ini

@@ -111,6 +111,7 @@ commands = pip install django>=1.9,<1.10
 
 [testenv:py27integration]
 # Run integration tests only
+passenv = AVCONV_INSTALLED
 setenv =
   PYTHONHASHSEED=0
   INTEGRATION_TESTS=1