Add a command to check network connectivity between hosts

kayobe network connectivity check

ansible/network-connectivity.yml

@@ -0,0 +1,84 @@
+---
+- name: Check network connectivity between hosts
+  hosts: seed:seed-hypervisor:overcloud
+  vars:
+    # Set this to an external IP address to check.
+    nc_external_ip: 8.8.8.8
+    # Set this to an external hostname to check.
+    nc_external_hostname: google.com
+    # Number of bytes to subtract from MTU to allow for ICMP (8 bytes) and IP
+    # (20 bytes) headers.
+    icmp_overhead_bytes: 28
+  tasks:
+    - name: Display next action
+      debug:
+        msg: >
+          Checking whether hosts have access to an external IP address,
+          {{ nc_external_ip }}.
+      run_once: True
+
+    - name: Ensure an external IP is reachable
+      command: ping -c1 {{ nc_external_ip }}
+      changed_when: False
+
+    - name: Display next action
+      debug:
+        msg: >
+          Checking whether hosts have access to an external hostname,
+          {{ nc_external_hostname }}.
+      run_once: True
+
+    - name: Ensure an external host is reachable
+      command: ping -c1 {{ nc_external_hostname }}
+      changed_when: False
+
+    - name: Display next action
+      debug:
+        msg: >
+          Checking whether hosts have access to any configured gateways.
+      run_once: True
+
+    - name: Ensure the gateway is reachable
+      command: >
+        ping {{ item | net_gateway }} -c1 -M do {% if mtu %} -s {{ mtu | int - icmp_overhead_bytes }}{% endif %}
+      with_items: "{{ network_interfaces }}"
+      when:
+        - item | net_ip
+        - item | net_gateway
+      changed_when: False
+      vars:
+        mtu: "{{ item | net_mtu }}"
+
+    # For each network on this host, pick a random remote host also on the
+    # network and try to ping it. Set the packet size according to the
+    # network's MTU.
+
+    - name: Display next action
+      debug:
+        msg: >
+          Checking whether hosts have access to other hosts on the same
+          network.
+      run_once: True
+
+    - name: Ensure hosts on the same network are reachable
+      command: >
+        ping {{ remote_ip }} -c1 -M do {% if mtu %} -s {{ mtu | int - icmp_overhead_bytes }}{% endif %}
+      with_items: "{{ network_interfaces }}"
+      when:
+        - item | net_ip
+        - remote_hosts | length > 0
+      changed_when: False
+      vars:
+        # Select other hosts targeted by this play which have this network
+        # interface (item).
+        remote_hosts: >
+          {{ hostvars.values() |
+             selectattr('inventory_hostname', 'is_in', play_hosts) |
+             selectattr('network_interfaces', 'defined') |
+             selectattr('network_interfaces', 'issuperset', [item]) |
+             rejectattr('inventory_hostname', 'equalto', inventory_hostname) |
+             map(attribute='inventory_hostname') |
+             list }}
+        remote_host: "{{ remote_hosts | random }}"
+        remote_ip: "{{ item | net_ip(remote_host) }}"
+        mtu: "{{ item | net_mtu }}"

ansible/test_plugins/functional.py

@@ -0,0 +1,27 @@
+# Copyright (c) 2017 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+def is_in(item, container):
+    """Tests whether an item is in a container (e.g. a list)."""
+    return item in container
+
+
+class TestModule(object):
+    """'Functional' programming tests."""
+
+    def tests(self):
+        return {
+            'is_in': is_in,
+        }

doc/source/administration.rst

@@ -111,6 +111,20 @@ specified directory, with one subdirectory per container. This command may be
 followed by ``kayobe ovecloud service configuration save`` to gather the
 generated configuration to the ansible control host.
 
+Checking Network Connectivity
+=============================
+
+In complex networking environments it can be useful to be able to automatically
+check network connectivity and diagnose networking issues.  To perform some
+simple connectivity checks::
+
+    (kayobe) $ kayobe network connectivity check
+
+Note that this will run on the seed, seed hypervisor, and overcloud hosts. If
+any of these hosts are not expected to be active (e.g. prior to overcloud
+deployment), the set of target hosts may be limited using the ``--limit``
+argument.
+
 Running Kayobe Playbooks on Demand
 ==================================
 

doc/source/release-notes.rst

@@ -17,6 +17,8 @@ Features
 * Adds support for configuration of custom fluentd filters, and additional
   config file templates for heat, ironic, keystone, magnum, murano, sahara, and
   swift in ``$KAYOBE_CONFIG_PATH/kolla/config/<component>/``.
+* Adds the command ``kayobe network connectivity check`` which can be used to
+  verify network connectivity in the cloud hosts.
 
 Upgrade Notes
 -------------

kayobe/cli/commands.py

@@ -972,3 +972,17 @@ class OvercloudPostConfigure(KayobeAnsibleMixin, VaultMixin, Command):
             "overcloud-introspection-rules-dell-lldp-workaround",
             "provision-net")
         self.run_kayobe_playbooks(parsed_args, playbooks)
+
+
+class NetworkConnectivityCheck(KayobeAnsibleMixin, VaultMixin, Command):
+    """Check network connectivity between hosts in the control plane.
+
+    Checks for access to an external IP address, an external hostname, any
+    configured gateways, and between hosts on the same subnets. The MTU of
+    each network is validated by sending ping packets of maximum size.
+    """
+
+    def take_action(self, parsed_args):
+        self.app.LOG.debug("Performing network connectivity check")
+        playbooks = _build_playbook_list("network-connectivity")
+        self.run_kayobe_playbooks(parsed_args, playbooks)

kayobe/tests/unit/cli/test_commands.py

@@ -68,3 +68,16 @@ class TestCase(unittest.TestCase):
                       tags="install"),
         ]
         self.assertEqual(expected_calls, mock_run.call_args_list)
+
+    @mock.patch.object(commands.KayobeAnsibleMixin,
+                       "run_kayobe_playbooks")
+    def test_network_connectivity_check(self, mock_run):
+        command = commands.NetworkConnectivityCheck(TestApp(), [])
+        parser = command.get_parser("test")
+        parsed_args = parser.parse_args([])
+        result = command.run(parsed_args)
+        self.assertEqual(0, result)
+        expected_calls = [
+            mock.call(mock.ANY, ["ansible/network-connectivity.yml"]),
+        ]
+        self.assertEqual(expected_calls, mock_run.call_args_list)

setup.py

@@ -55,6 +55,7 @@ setup(
             'control_host_upgrade = kayobe.cli.commands:ControlHostUpgrade',
             'configuration_dump = kayobe.cli.commands:ConfigurationDump',
             'kolla_ansible_run = kayobe.cli.commands:KollaAnsibleRun',
+            'network_connectivity_check = kayobe.cli.commands:NetworkConnectivityCheck',
             'overcloud_bios_raid_configure = kayobe.cli.commands:OvercloudBIOSRAIDConfigure',
             'overcloud_container_image_build = kayobe.cli.commands:OvercloudContainerImageBuild',
             'overcloud_container_image_pull = kayobe.cli.commands:OvercloudContainerImagePull',