Add plugin to mark tests as expected to fail

Change-Id: I72d57e2ab64f580794cdbb623f52efd75f53eb44

plugin_test/vapor/pytest.ini

@@ -5,4 +5,4 @@ markers =
     performance_test: mark test as Performance.
 
 #addopts = -vv --color=yes --junit-xml=report.xml
-addopts = -vv --color=yes
+addopts = -vv --color=yes -ra

plugin_test/vapor/tox.ini

@@ -1,6 +1,7 @@
 # content of: tox.ini , put in same dir as setup.py
 [tox]
 envlist = py27
+skipdist = True
 
 [testenv]
 passenv=
@@ -8,7 +9,7 @@ passenv=
 deps=
     -e.
     -rrequirements.txt
-commands=py.test {posargs}
+commands=py.test vapor {posargs}
 
 [flake8]
 filename=*.py

plugin_test/vapor/vapor/conftest.py

@@ -20,5 +20,7 @@ from vapor.fixtures.skip import *  # noqa
 from vapor.fixtures.subnets import *  # noqa
 
 pytest_plugins = [
-    'stepler.third_party.destructive_dispatcher'
+    'stepler.third_party.destructive_dispatcher',
+    'stepler.third_party.idempotent_id',
+    'vapor.plugins.xfail',
 ]

plugin_test/vapor/vapor/plugins/xfail.py

@@ -0,0 +1,50 @@
+"""
+Pytest plugin to mark tests with xfail from config yaml file.
+
+Note:
+
+    Xfail file should be valid yaml file with format:
+
+    .. code:: yaml
+
+       test_name: reason
+       vapor/tests/common/test_sg_policy.py::another_test_name: // WO reason
+       yet_more_test_name: reason2
+"""
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+import yaml
+
+
+def pytest_addoption(parser):
+    """Add ``--xfail-file`` option to pytest."""
+    parser.addoption("--xfail-file", action="store", default='xfail.yaml',
+                     help="Path to yaml file with xfail marks for tests.")
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_collection_modifyitems(config, items):
+    """Hook to skip tests with opened bugs."""
+    if not config.option.xfail_file:
+        return
+
+    with open(config.option.xfail_file) as f:
+        xfails = yaml.safe_load(f)
+
+    for item in items:
+        for key, message in xfails.items():
+            if key == item.name or key == item.nodeid:
+                item.add_marker(pytest.mark.xfail(reason=message))

plugin_test/vapor/vapor/settings.py

@@ -4,6 +4,8 @@ import os
 
 import yaml
 
+BASE_DIR = os.path.dirname(__file__)
+
 PATH_TO_CERT = '/tmp/cert.crt'
 VERIFY_SSL = False
 DISABLE_SSL = True
@@ -97,7 +99,7 @@ CONTRAIL_ROLES_SERVICES_MAPPING = {
 CONTRAIL_ROLES_DISTRIBUTION_YAML = os.environ.get(
     'CONTRAIL_ROLES_DISTRIBUTION_YAML',
     os.path.join(
-        os.path.dirname(__file__), '../roles_distribution_example.yaml'))
+        BASE_DIR, '../roles_distribution_example.yaml'))
 
 with open(CONTRAIL_ROLES_DISTRIBUTION_YAML) as f:
     CONTRAIL_ROLES_DISTRIBUTION = yaml.safe_load(f) or {}
@@ -133,7 +135,6 @@ CONTRAIL_ANALYTIC_PROCESSES = {
     ],
 }
 
-BASE_DIR = os.path.dirname(__file__)
 HEAT_TEMPLATES_PATH = os.path.join(BASE_DIR, 'heat')
 
-VROUTER_HEADLESS_MODE_CMD = "grep -iP '^headless_mode\s*=\s*true' /etc/contrail/contrail-vrouter-agent.conf"  # noqa
+VROUTER_HEADLESS_MODE_CMD = r"grep -iP '^headless_mode\s*=\s*true' /etc/contrail/contrail-vrouter-agent.conf"  # noqa

plugin_test/vapor/xfail.yaml

@@ -0,0 +1,3 @@
+test_name: reason
+vapor/tests/common/test_sg_policy.py::another_test_name: // Without reason
+yet_more_test_name: reason2