Merge "Add test_agent_cleanup_with_control_node_stop"

This commit is contained in:
Jenkins 2017-03-01 10:44:29 +00:00 committed by Gerrit Code Review
commit 71f5c0ae51
8 changed files with 181 additions and 18 deletions

View File

@ -24,3 +24,20 @@ def computes(os_faults_steps):
fqdns = settings.CONTRAIL_ROLES_DISTRIBUTION[
settings.ROLE_CONTRAIL_COMPUTE]
return os_faults_steps.get_nodes(fqdns)
@pytest.fixture
def stop_service(os_faults_steps):
"""Callable fixture to stop service on nodes."""
stopped = []
def _stop_service(nodes, service):
cmd = "service {} stop".format(service)
os_faults_steps.execute_cmd(nodes, cmd)
stopped.append([nodes, service])
yield _stop_service
for nodes, service in reversed(stopped):
cmd = "service {} start".format(service)
os_faults_steps.execute_cmd(nodes, cmd)

View File

@ -27,8 +27,8 @@ class Predicates(skip.Predicates):
def vrouter_headless_mode(self):
"""Define whether vrouter headless mode enabled."""
os_faults_steps = self._get_fixture('os_faults_steps')
fqdns = settings.CONTRAIL_ROLES_DISTRIBUTION[settings.
ROLE_CONTRAIL_COMPUTE]
fqdns = settings.CONTRAIL_ROLES_DISTRIBUTION[
settings.ROLE_CONTRAIL_COMPUTE]
actual_nodes = os_faults_steps.get_nodes_by_cmd(
settings.VROUTER_HEADLESS_MODE_CMD)
return set(actual_nodes.get_fqdns()) == set(fqdns)
@ -41,6 +41,13 @@ class Predicates(skip.Predicates):
sriov_device_mappings = sriov.get_sriov_device_mapping(agent_steps)
return len(sriov_device_mappings) > 0
@property
@_store_call
def contrail_control_nodes_count(self):
"""Return contrail control nodes count."""
return len(settings.CONTRAIL_ROLES_DISTRIBUTION[
settings.ROLE_CONTRAIL_CONTROLLER])
@pytest.fixture
def predicates(request):

View File

@ -65,6 +65,28 @@ def get_vna_xmpp_connection_status(session, ip, port):
return result
def get_vna_vm_list(session, ip, port):
"""Return vna vm list."""
response = session.get(
'http://{ip}:{port}/Snh_VmListReq?uuid='.format(
ip=ip, port=port))
response.raise_for_status()
tree = ET.fromstring(response.content)
avn = tree.findall('.//list/VmSandeshData/uuid')
return [x.text for x in avn]
def wait_vna_vm_list(session, ip, port, matcher, timeout):
"""Wait until received vm_list will satisfy the macher."""
def predicate():
list_of_vm = get_vna_vm_list(session, ip, port)
return waiter.expect_that(list_of_vm, matcher)
waiter.wait(predicate, timeout_seconds=timeout)
def get_processes_with_wrong_state(ops, module_id,
expected_state='Functional'):
"""Return ops with wrong state for module_id."""

View File

@ -75,37 +75,57 @@ class BaseSequencesMatcher(BaseMatcher):
return not self._get_wrong_items(item)
class IsSubsetOf(BaseSequencesMatcher):
class SubsetOf(BaseSequencesMatcher):
def _get_wrong_items(self, item):
return self._get_extra_items(item, self.sequence)
def _get_wrong_items(self, other):
return self._get_extra_items(other, self.sequence)
def describe_to(self, description):
description.append_text('a subset of ').append_text(self.sequence)
def describe_mismatch(self, item, mismatch_description):
super(IsSubsetOf, self).describe_mismatch(item, mismatch_description)
super(SubsetOf, self).describe_mismatch(item, mismatch_description)
mismatch_description.append_text(' with unexpected items ') \
.append_description_of(self._get_wrong_items(item))
class IsSupersetOf(BaseSequencesMatcher):
class SupersetOf(BaseSequencesMatcher):
def _get_wrong_items(self, item):
return self._get_extra_items(self.sequence, item)
def _get_wrong_items(self, other):
return self._get_extra_items(self.sequence, other)
def describe_to(self, description):
description.append_text('a superset of ').append_text(self.sequence)
def describe_mismatch(self, item, mismatch_description):
super(IsSupersetOf, self).describe_mismatch(item, mismatch_description)
super(SupersetOf, self).describe_mismatch(item, mismatch_description)
mismatch_description.append_text(' without expected items ') \
.append_description_of(self._get_wrong_items(item))
def is_subset_of(sequence):
return IsSubsetOf(sequence)
class IntersectsWith(BaseSequencesMatcher):
def _matches(self, other):
for item in self.sequence:
if item in other:
return True
return False
def describe_to(self, description):
description.append_text(
'a sequence which intersect with ').append_text(self.sequence)
def is_superset_of(sequence):
return IsSupersetOf(sequence)
def subset_of(other):
"""Matches if sequence is subset of `other`."""
return SubsetOf(other)
def superset_of(other):
"""Matches if sequence is superset of `other`."""
return SupersetOf(other)
def intersects_with(other):
"""Matches if sequence has intersections with `other`."""
return IntersectsWith(other)

View File

@ -47,6 +47,12 @@ PING_SUCCESS_TIMEOUT = 60 * 2
# Security group apply timeout
SECURITY_GROUP_APPLY_TIMEOUT = 20
# Time to wait for contrail agent cleanup after stopping control node
CONTRAIL_AGENT_CLEANUP_TIMEOUT = 20 * 60
# Time to wait for contrail agent vna vm list to contain server uuid
CONTRAIL_AGENT_VNA_VM_LIST_TIMEOUT = 3 * 60
ROLE_CONTRAIL_CONTROLLER = 'contrail-controller'
ROLE_CONTRAIL_ANALYTICS = 'contrail-analytics'
ROLE_CONTRAIL_DB = 'contrail-db'

View File

@ -7,7 +7,7 @@ from six.moves import filter
from vapor.helpers import analytic_steps
from vapor.helpers import asserts
from vapor.helpers.asserts import is_superset_of
from vapor.helpers.asserts import superset_of
from vapor import settings
@ -242,7 +242,7 @@ def test_uve_module_states(client_contrail_analytics, os_faults_steps, role,
'NodeStatus.process_info[].process_name', data)
process_list = [x.split(':')[0] for x in process_list]
collector.check(process_list,
is_superset_of(expected_process_list),
superset_of(expected_process_list),
'Processes are absent on {}'.format(node))
wrong_processes = analytic_steps.get_process_info_with_wrong_state(
data)

View File

@ -15,6 +15,8 @@ import pytest
from stepler import config as stepler_config
from vapor import settings
from vapor.helpers.asserts import intersects_with
from vapor.helpers import analytic_steps
pytestmark = pytest.mark.destructive
@ -139,3 +141,92 @@ def test_connectivity_after_contrail_services_restart(
# Verify network creation
network = request.getfixturevalue('contrail_network')
assert_that(network.uuid, is_not(None))
def test_agent_cleanup_with_control_node_stop(
session, nodes_ips, contrail_services_http_introspect_ports,
cirros_image, flavor, security_group, network, subnet, public_network,
create_floating_ip, stop_service, port_steps, server_steps,
os_faults_steps):
"""Stop all the control node and verify the cleanup process in agent.
Steps:
#. Create network, subnet
#. Create 2 servers
#. Add floating IP addresses to servers
#. Check ping to servers' floating ips
#. Get servers' ids from contrail_vrouter_agent
#. Get contrail control nodes connected to this contrail_vrouter_agent
#. Stop `contrail-control` service on found nodes
#. Check that servers' ids list from contrail_vrouter_agent is not
contain servers ids after some time
#. Start `contrail-control` service on found nodes
#. Check that servers' ids list from contrail_vrouter_agent is contain
servers ids after some time
#. Check ping to servers' floating ips
"""
# Create servers
servers = server_steps.create_servers(
count=2,
image=cirros_image,
flavor=flavor,
security_groups=[security_group],
networks=[network],
username=stepler_config.CIRROS_USERNAME,
password=stepler_config.CIRROS_PASSWORD)
# Create Floating IP
for server in servers:
server_port = port_steps.get_port(
device_owner=stepler_config.PORT_DEVICE_OWNER_SERVER,
device_id=server.id)
floating_ip = create_floating_ip(public_network, port=server_port)
server_steps.check_server_ip(
server,
floating_ip['floating_ip_address'],
timeout=settings.FLOATING_IP_BIND_TIMEOUT)
for server in servers:
server_steps.check_ping_to_server_floating(
server, timeout=stepler_config.PING_CALL_TIMEOUT)
servers_ids = [s.id for s in servers]
compute_fqdn = getattr(servers[0],
settings.SERVER_ATTR_HYPERVISOR_HOSTNAME)
agent_ip = nodes_ips[compute_fqdn][0]
agent_port = contrail_services_http_introspect_ports[
'contrail-vrouter-agent']['port']
analytic_steps.wait_vna_vm_list(
session, agent_ip, agent_port,
intersects_with(servers_ids),
settings.CONTRAIL_AGENT_VNA_VM_LIST_TIMEOUT)
# Collecting control nodes
controllers_fqdns = []
for entry in analytic_steps.get_vna_xmpp_connection_status(
session, agent_ip, agent_port):
ip = entry['controller_ip']
fqdn = next(fqnd for fqnd, ips in nodes_ips.items() if ip in ips)
controllers_fqdns.append(fqdn)
controller_nodes = os_faults_steps.get_nodes(fqdns=controllers_fqdns)
# Stop contrail-control service
stop_service(controller_nodes, 'contrail-control')
analytic_steps.wait_vna_vm_list(session, agent_ip, agent_port,
is_not(intersects_with(servers_ids)),
settings.CONTRAIL_AGENT_CLEANUP_TIMEOUT)
os_faults_steps.execute_cmd(controller_nodes,
'service contrail-control start')
analytic_steps.wait_vna_vm_list(
session, agent_ip, agent_port,
intersects_with(servers_ids),
settings.CONTRAIL_AGENT_VNA_VM_LIST_TIMEOUT)
for server in servers:
server_steps.check_ping_to_server_floating(
server, timeout=stepler_config.PING_CALL_TIMEOUT)

View File

@ -19,7 +19,7 @@ from stepler.third_party import utils
from vapor.helpers import contrail_status
from vapor.helpers import asserts
from vapor.helpers.asserts import is_superset_of
from vapor.helpers.asserts import superset_of
from vapor import settings
@ -38,7 +38,7 @@ def test_contrail_service_distribution(os_faults_steps, role):
if node not in nodes:
continue
services = [x.service for x in services]
collector.check(services, is_superset_of(expected_services))
collector.check(services, superset_of(expected_services))
def test_ifmap_service(os_faults_steps):