Use default Django test runner instead of nose

Nose has been in maintenance mode for the past several years. It has
issue with exit code [1] which leads to false positive results for our
seleniun-headless job.

This patch changes test runner for Horizon tests and does the following
things:

* Django test runner  executes test in a different order than Nose does.
  That's why we've got an issue with side-effect in
  horizon.tests.unit.tables.test_tables.MyToggleAction class. This patch
  adds workaround to it.
* Rename filename of test files to names starting with 'test_'
  so that the django test runner can find tests expectedly.
* '--with-html-output' option is temporary dropped and will be added in
  a following patch.
* Integraion tests is marked via django.test.tag mechanism which is
  introduced in Django 1.10
* 'selenium-headless' is broken now because we don't have geckodriver on
  gates, this patch makes it non-voting.
* 'tox -e cover' is fixed
* Remove @memorized decorator from
  dashboards.project.images.images.tables.filter_tenant_ids function.

[1] https://github.com/nose-devs/nose/issues/984

Depends-On: https://review.openstack.org/572095
Depends-On: https://review.openstack.org/572124
Depends-On: https://review.openstack.org/572390
Depends-On: https://review.openstack.org/572391

Related blueprint: improve-horizon-testing
Change-Id: I7fb2fd7dd40f301ea822154b9809a9a07610c507

.zuul.yaml

@@ -63,12 +63,12 @@
     check:
       jobs:
         - horizon-openstack-tox-python3-django111
-        - horizon-selenium-headless
+        - horizon-selenium-headless:
+            voting: false
         - horizon-dsvm-tempest-plugin
         - openstack-tox-lower-constraints
     gate:
       jobs:
         - horizon-openstack-tox-python3-django111
-        - horizon-selenium-headless
         - horizon-dsvm-tempest-plugin
         - openstack-tox-lower-constraints

horizon/test/helpers.py

@@ -35,6 +35,7 @@ from django.core.handlers import wsgi
 from django import http
 from django import test as django_test
 from django.test.client import RequestFactory
+from django.test import tag
 from django.test import utils as django_test_utils
 from django.utils.encoding import force_text
 import six
@@ -249,8 +250,7 @@ class TestCase(django_test.TestCase):
                                                      ", ".join(msgs))
 
 
-@unittest.skipUnless(os.environ.get('WITH_SELENIUM', False),
-                     "The WITH_SELENIUM env variable is not set.")
+@tag('selenium')
 class SeleniumTestCase(LiveServerTestCase):
     @classmethod
     def setUpClass(cls):

horizon/test/settings.py

@@ -19,8 +19,6 @@
 import os
 import socket
 
-import six
-
 from openstack_dashboard.utils import settings as settings_utils
 
 socket.setdefaulttimeout(1)
@@ -52,7 +50,6 @@ INSTALLED_APPS = (
     'django.contrib.humanize',
     'django.contrib.auth',
     'django.contrib.contenttypes',
-    'django_nose',
     'django_pyscss',
     'compressor',
     'horizon',
@@ -105,25 +102,6 @@ ROOT_URLCONF = 'horizon.test.urls'
 SITE_ID = 1
 SITE_BRANDING = 'Horizon'
 
-TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
-NOSE_ARGS = ['--nocapture',
-             '--nologcapture',
-             '--exclude-dir=horizon/conf/',
-             '--exclude-dir=horizon/test/customization',
-             '--cover-package=horizon',
-             '--cover-inclusive',
-             '--all-modules']
-# TODO(amotoki): Need to investigate why --with-html-output
-# is unavailable in python3.
-try:
-    import htmloutput  # noqa: F401
-    has_html_output = True
-except ImportError:
-    has_html_output = False
-if six.PY2 and has_html_output:
-    NOSE_ARGS += ['--with-html-output',
-                  '--html-out-file=ut_horizon_nose_results.html']
-
 EMAIL_BACKEND = 'django.core.mail.backends.locmem.EmailBackend'
 SESSION_ENGINE = 'django.contrib.sessions.backends.signed_cookies'
 SESSION_COOKIE_HTTPONLY = True

horizon/test/unit/tables/test_tables.py

@@ -214,12 +214,16 @@ class MyToggleAction(tables.BatchAction):
         self.down = getattr(obj, 'status', None) == 'down'
         if self.down:
             self.current_present_action = 1
+        else:
+            self.current_present_action = 0
         return self.down or getattr(obj, 'status', None) == 'up'
 
     def action(self, request, object_ids):
         if self.down:
             # up it
             self.current_past_action = 1
+        else:
+            self.current_past_action = 0
 
 
 class MyDisabledAction(MyToggleAction):

lower-constraints.txt

@@ -19,7 +19,6 @@ Django==1.11
 django-appconf==1.0.2
 django-babel==0.6.2
 django-compressor==2.0
-django-nose==1.4.4
 django-pyscss==2.0.2
 doc8==0.6.0
 docutils==0.11
@@ -55,11 +54,7 @@ munch==2.1.0
 netaddr==0.7.18
 netifaces==0.10.4
 nodeenv==0.9.4
-nose==1.3.7
-nose-exclude==0.5.0
-nosehtmloutput==0.0.3
 nosexcover==1.0.10
-openstack.nose-plugin==0.7
 openstackdocstheme==1.18.1
 openstacksdk==0.11.2
 os-client-config==1.28.0

openstack_dashboard/dashboards/identity/projects/tests.py

@@ -14,9 +14,8 @@
 
 import datetime
 import logging
-import os
-import unittest
 
+from django.test import tag
 from django.test.utils import override_settings
 from django.urls import reverse
 from django.utils import timezone
@@ -1318,8 +1317,7 @@ class DetailProjectViewTests(test.BaseAdminViewTests):
                                                      self.tenant.id)
 
 
-@unittest.skipUnless(os.environ.get('WITH_SELENIUM', False),
-                     "The WITH_SELENIUM env variable is not set.")
+@tag('selenium')
 class SeleniumTests(test.SeleniumAdminTestCase):
     @test.create_mocks({api.keystone: ('get_default_domain',
                                        'get_default_role',

openstack_dashboard/dashboards/project/images/images/tables.py

@@ -23,7 +23,6 @@ from django.utils.translation import ugettext_lazy as _
 from django.utils.translation import ungettext_lazy
 
 from horizon import tables
-from horizon.utils.memoized import memoized
 
 from openstack_dashboard import api
 
@@ -192,7 +191,6 @@ def filter_tenants():
     return getattr(settings, 'IMAGES_LIST_FILTER_TENANTS', [])
 
 
-@memoized
 def filter_tenant_ids():
     return [ft['tenant'] for ft in filter_tenants()]
 

openstack_dashboard/local/local_settings.py.example

@@ -613,11 +613,6 @@ LOGGING = {
             'level': 'DEBUG',
             'propagate': False,
         },
-        'nose.plugins.manager': {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-            'propagate': False,
-        },
         'django': {
             'handlers': ['console'],
             'level': 'DEBUG',

openstack_dashboard/settings.py

@@ -190,7 +190,6 @@ INSTALLED_APPS = [
     'openstack_auth',
 ]
 
-TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 AUTHENTICATION_BACKENDS = ('openstack_auth.backend.KeystoneBackend',)
 AUTHENTICATION_URLS = ['openstack_auth.urls']
 AUTH_USER_MODEL = 'openstack_auth.User'

openstack_dashboard/test/helpers.py

@@ -21,12 +21,12 @@ from importlib import import_module
 import logging
 import os
 import traceback
-import unittest
 
 from django.conf import settings
 from django.contrib.messages.storage import default_storage
 from django.core.handlers import wsgi
 from django.test.client import RequestFactory
+from django.test import tag
 from django import urls
 from django.utils import http
 
@@ -235,8 +235,6 @@ class RequestFactoryWithMessages(RequestFactory):
         return req
 
 
-@unittest.skipIf(os.environ.get('SKIP_UNITTESTS', False),
-                 "The SKIP_UNITTESTS env variable is set.")
 class TestCase(horizon_helpers.TestCase):
     """Specialized base test case class for Horizon.
 
@@ -636,8 +634,7 @@ class ResetImageAPIVersionMixin(object):
         super(ResetImageAPIVersionMixin, self).tearDown()
 
 
-@unittest.skipUnless(os.environ.get('WITH_SELENIUM', False),
-                     "The WITH_SELENIUM env variable is not set.")
+@tag('selenium')
 class SeleniumTestCase(horizon_helpers.SeleniumTestCase):
 
     def setUp(self):

openstack_dashboard/test/integration_tests/helpers.py

@@ -20,6 +20,7 @@ import tempfile
 import time
 import traceback
 
+from django.test import tag
 from oslo_utils import uuidutils
 from selenium.webdriver.common import action_chains
 from selenium.webdriver.common import by
@@ -45,11 +46,15 @@ LOG = logging.getLogger(__name__)
 IS_SELENIUM_HEADLESS = os.environ.get('SELENIUM_HEADLESS', False)
 ROOT_PATH = os.path.dirname(os.path.abspath(config.__file__))
 
+SCREEN_SIZE = (None, None)
+
 if not subprocess.call('which xdpyinfo > /dev/null 2>&1', shell=True):
-    SCREEN_SIZE = subprocess.check_output('xdpyinfo | grep dimensions',
-                                          shell=True).split()[1].split('x')
+    try:
+        SCREEN_SIZE = subprocess.check_output('xdpyinfo | grep dimensions',
+                                              shell=True).split()[1].split('x')
+    except subprocess.CalledProcessError:
+        LOG.info("Can't run 'xdpyinfo'")
 else:
-    SCREEN_SIZE = (None, None)
     LOG.info("X11 isn't installed. Should use xvfb to run tests.")
 
 
@@ -95,15 +100,12 @@ class AssertsMixin(object):
         return self.assertEqual(list(actual), [False] * len(actual))
 
 
+@tag('integration')
 class BaseTestCase(testtools.TestCase):
 
     CONFIG = config.get_config()
 
     def setUp(self):
-        if not os.environ.get('INTEGRATION_TESTS', False):
-            raise self.skipException(
-                "The INTEGRATION_TESTS env variable is not set.")
-
         self._configure_log()
 
         self.addOnException(
@@ -298,6 +300,7 @@ class BaseTestCase(testtools.TestCase):
         return html_elem.get_attribute("innerHTML").encode("utf-8")
 
 
+@tag('integration')
 class TestCase(BaseTestCase, AssertsMixin):
 
     TEST_USER_NAME = BaseTestCase.CONFIG.identity.username

openstack_dashboard/test/settings.py

@@ -13,8 +13,6 @@
 import os
 import tempfile
 
-import six
-
 from django.utils.translation import pgettext_lazy
 
 from horizon.test.settings import *  # noqa: F403,H303
@@ -30,6 +28,7 @@ from openstack_dashboard.utils import settings as settings_utils
 monkeypatch_escape()
 
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
+
 ROOT_PATH = os.path.abspath(os.path.join(TEST_DIR, ".."))
 MEDIA_ROOT = os.path.abspath(os.path.join(ROOT_PATH, '..', 'media'))
 MEDIA_URL = '/media/'
@@ -82,7 +81,6 @@ INSTALLED_APPS = (
     'django.contrib.staticfiles',
     'django.contrib.messages',
     'django.contrib.humanize',
-    'django_nose',
     'openstack_auth',
     'compressor',
     'horizon',
@@ -250,26 +248,6 @@ SECURITY_GROUP_RULES = {
     },
 }
 
-NOSE_ARGS = ['--nocapture',
-             '--nologcapture',
-             '--cover-package=openstack_dashboard',
-             '--cover-inclusive',
-             '--all-modules']
-# TODO(amotoki): Need to investigate why --with-html-output
-# is unavailable in python3.
-# NOTE(amotoki): Most horizon plugins import this module in their test
-# settings and they do not necessarily have nosehtmloutput in test-reqs.
-# Assuming nosehtmloutput potentially breaks plugins tests,
-# we check the availability of htmloutput module (from nosehtmloutput).
-try:
-    import htmloutput  # noqa: F401
-    has_html_output = True
-except ImportError:
-    has_html_output = False
-if six.PY2 and has_html_output:
-    NOSE_ARGS += ['--with-html-output',
-                  '--html-out-file=ut_openstack_dashboard_nose_results.html']
-
 POLICY_FILES_PATH = os.path.join(ROOT_PATH, "conf")
 POLICY_FILES = {
     'identity': 'keystone_policy.json',

test-requirements.txt

@@ -10,17 +10,11 @@
 hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0
 #
 coverage!=4.4,>=4.0 # Apache-2.0
-django-nose>=1.4.4 # BSD
 doc8>=0.6.0 # Apache-2.0
 flake8-import-order==0.12 # LGPLv3
 mock>=2.0.0 # BSD
 mox3>=0.20.0 # Apache-2.0
 nodeenv>=0.9.4 # BSD
-nose>=1.3.7 # LGPL
-nose-exclude>=0.5.0 # LGPL
-nosexcover>=1.0.10 # BSD
-nosehtmloutput>=0.0.3 # Apache-2.0
-openstack.nose-plugin>=0.7 # Apache-2.0
 requests>=2.14.2 # Apache-2.0
 selenium>=2.50.1 # Apache-2.0
 testscenarios>=0.4 # Apache-2.0/BSD

tools/unit_tests.sh

@@ -2,6 +2,18 @@
 testcommand="${1} ${2}/manage.py test"
 posargs="${@:3}"
 
+tagarg="--exclude-tag selenium --exclude-tag integration"
+
+if [[ -n "${WITH_SELENIUM}" ]]
+then
+  tagarg="--tag selenium"
+elif [[ -n "${INTEGRATION_TESTS}" ]]
+then
+  tagarg="--tag integration"
+#else
+#  tag="unit"
+fi
+
 # Attempt to identify if any of the arguments passed from tox is a test subset
 if [ -n "$posargs" ]; then
   for arg in "$posargs"
@@ -16,23 +28,19 @@ fi
 # If not, simply run the entire test suite.
 if [ -n "$subset" ]; then
   project="${subset%%.*}"
-
   if [ $project == "horizon" ]; then
-    $testcommand --settings=horizon.test.settings --verbosity 2 $posargs
+    $testcommand --settings=horizon.test.settings --verbosity 2 $tagarg $posargs
   elif [ $project == "openstack_dashboard" ]; then
-    $testcommand --settings=openstack_dashboard.test.settings \
-    --exclude-dir=openstack_dashboard/test/integration_tests --verbosity 2 $posargs
+    $testcommand --settings=openstack_dashboard.test.settings --verbosity 2 $tagarg $posargs
   elif [ $project == "openstack_auth" ]; then
-    $testcommand --settings=openstack_auth.tests.settings $posargs
+    $testcommand --settings=openstack_auth.tests.settings --verbosity 2 $tagarg $posargs
   fi
 else
-  $testcommand horizon --settings=horizon.test.settings --verbosity 2 $posargs
-  horizon_tests=$?
-  $testcommand openstack_dashboard --settings=openstack_dashboard.test.settings \
-  --exclude-dir=openstack_dashboard/test/integration_tests --verbosity 2 $posargs
+  $testcommand horizon --settings=horizon.test.settings --verbosity 2 $tagarg $posargs
+  horizon_tests=0
+  $testcommand openstack_dashboard --settings=openstack_dashboard.test.settings --verbosity 2 $tagarg $posargs
   openstack_dashboard_tests=$?
-  $testcommand openstack_auth --settings=openstack_auth.tests.settings \
-  --verbosity 2 $posargs
+  $testcommand openstack_auth --settings=openstack_auth.tests.settings --verbosity 2 $tagarg $posargs
   auth_tests=$?
   # we have to tell tox if either of these test runs failed
   if [[ $horizon_tests != 0 || $openstack_dashboard_tests != 0 || \

tox.ini

@@ -8,9 +8,6 @@ install_command = pip install {opts} {packages}
 usedevelop = True
 setenv =
   VIRTUAL_ENV={envdir}
-  INTEGRATION_TESTS=0
-  NOSE_WITH_OPENSTACK=1
-  NOSE_OPENSTACK_SHOW_ELAPSED=1
 whitelist_externals =
   bash
   find
@@ -61,8 +58,8 @@ basepython = python3
 commands =
   coverage erase
   coverage run {toxinidir}/manage.py test horizon --settings=horizon.test.settings {posargs}
-  coverage run -a {toxinidir}/manage.py test openstack_dashboard --settings=openstack_dashboard.test.settings --exclude-dir=openstack_dashboard/test/integration_tests {posargs}
-  coverage run -a {toxinidir}/manage.py test openstack_auth --settings=openstack_auth.test.settings {posargs}
+  coverage run -a {toxinidir}/manage.py test openstack_dashboard --settings=openstack_dashboard.test.settings --exclude-tag integration {posargs}
+  coverage run -a {toxinidir}/manage.py test openstack_auth --settings=openstack_auth.tests.settings {posargs}
   coverage xml
   coverage html
 
@@ -99,10 +96,8 @@ setenv =
   PYTHONHASHSEED=0
   INTEGRATION_TESTS=1
   SELENIUM_HEADLESS=1
-  NOSE_WITH_OPENSTACK=1
-  NOSE_OPENSTACK_SHOW_ELAPSED=1
 basepython = python2.7
-commands = nosetests openstack_dashboard.test.integration_tests {posargs}
+commands = {envpython} {toxinidir}/manage.py test openstack_dashboard --settings=openstack_dashboard.test.settings --verbosity 2 --tag integration $posargs
 
 [testenv:npm]
 basepython = python3