Add contrail failover tests

Change-Id: I4a1df314663ddc0aac669843fa58ec9cfe04adc1

plugin_test/vapor/README.rst

@@ -3,6 +3,18 @@ Vapor
 
 Open Stack Contrail plugin test suite
 
+Requirements
+------------
+
+It's strongly advised to use virtualenv and update pip, tox, virtualenv and setuptools.
+
+.. code-block:: bash
+
+    virtualenv venv
+    . venv/bin/activate
+    pip install -U pip tox virtualenv setuptools
+
+
 Run tests
 ---------
 .. code:: bash

plugin_test/vapor/vapor/conftest.py

@@ -1,6 +1,9 @@
 # pytest settings and fixtures
 
 from stepler.conftest import *  # noqa
+from stepler.glance.fixtures import *  # noqa
+from stepler.neutron.fixtures import *  # noqa
+from stepler.nova.fixtures import *  # noqa
 
 from vapor.fixtures.contrail import *  # noqa
 from vapor.fixtures.networks import *  # noqa

plugin_test/vapor/vapor/fixtures/contrail.py

@@ -16,14 +16,14 @@ def client_contrail():
 @pytest.fixture
 def contrail_nodes(os_faults_steps):
     """Returns all nodes which have contrail-status command."""
-    return os_faults_steps.get_nodes_by_cmd('which contrail-status')
+    return os_faults_steps.get_nodes_by_cmd('contrail-status | grep .')
 
 
 @pytest.fixture
 def contrail_controllers(os_faults_steps, contrail_nodes):
     """Returns all contrail controller nodes."""
     return os_faults_steps.get_nodes_by_cmd(
-        'contrail-status | grep -v "Contrail vRouter"')
+        'contrail-status | grep  "Contrail Control"')
 
 
 @pytest.fixture(scope='module')

plugin_test/vapor/vapor/settings.py

@@ -16,3 +16,9 @@ CONTRAIL_CREDS = {
 CONTAIL_API_PORT = 8082
 
 NEUTRON_CONTRAIL_PLUGIN_CONFIG_PATH = '/etc/neutron/plugins/opencontrail/ContrailPlugin.ini'  # noqa
+
+# Time between poweroff and start contrail controller node
+CONTRAIL_CONTROLLER_RESTART_TIMEOUT = 5 * 60
+
+# Time between networks unplug and plug back to contrail controller node
+CONTRAIL_CONTROLLER_NET_REPLUG_TIMEOUT = 5 * 60

plugin_test/vapor/vapor/tests/test_failover.py

@@ -0,0 +1,61 @@
+import time
+
+import pytest
+from stepler import config as stepler_config
+
+from vapor import settings
+
+
+def restart_nodes(os_faults_steps, nodes):
+    for node in nodes:
+        node = os_faults_steps.get_nodes(fqdns=[node.fqdn])
+        node.poweroff()
+        time.sleep(settings.CONTRAIL_CONTROLLER_RESTART_TIMEOUT)
+        node.poweron()
+
+
+def stop_network_interfaces(os_faults_steps, nodes):
+    for node in nodes:
+        node = os_faults_steps.get_nodes(fqdns=[node.fqdn])
+        node.disconnect(network_name='management')
+        node.disconnect(network_name='private')
+        time.sleep(settings.CONTRAIL_CONTROLLER_NET_REPLUG_TIMEOUT)
+        node.connect(network_name='private')
+        node.connect(network_name='management')
+
+
+@pytest.mark.parametrize('action',
+                         [restart_nodes, stop_network_interfaces],
+                         ids=['restart', 'stop_network'])
+def test_with_destructive_action(
+        public_network,
+        cirros_image,
+        flavor,
+        security_group,
+        network,
+        subnet,
+        contrail_controllers,
+        create_floating_ip,
+        port_steps,
+        server_steps,
+        os_faults_steps,
+        action):
+
+    server1, server2 = server_steps.create_servers(
+        count=2,
+        image=cirros_image,
+        flavor=flavor,
+        networks=[network],
+        security_groups=[security_group],
+        username=stepler_config.CIRROS_USERNAME,
+        password=stepler_config.CIRROS_PASSWORD)
+
+    server1_port = port_steps.get_port(
+        device_owner=stepler_config.PORT_DEVICE_OWNER_SERVER,
+        device_id=server1.id)
+    floating_ip = create_floating_ip(public_network, port=server1_port)
+    server2_ip = server_steps.get_fixed_ip(server2)
+    with server_steps.get_server_ssh(
+            server1, ip=floating_ip['floating_ip_address']) as server1_ssh:
+        with server_steps.check_ping_loss_context(server2_ip, 0, server1_ssh):
+            action(os_faults_steps, contrail_controllers)

plugin_test/vapor/vapor/tests/test_smoke.py

@@ -22,8 +22,10 @@ def test_contrail_node_services_status(contrail_nodes, os_faults_steps):
     broken_services = []
     for node_result in os_faults_steps.execute_cmd(contrail_nodes, cmd):
         for line in node_result.payload['stdout_lines']:
-            line = line.strip()
-            name, status = line.split(None, 1)
+            values = line.strip().split(None, 1)
+            if len(values) < 2:
+                continue
+            name, status = values
             if status not in {'active', 'backup'}:
                 err_msg = "{node}:{service} - {status}".format(
                     node=node_result.host, service=name, status=status)