From 107de9323ad8025d3adbfb7fdd2b59bd1d164b30 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 1 Nov 2017 15:44:05 +0000 Subject: [PATCH] Add a command to check network connectivity between hosts kayobe network connectivity check --- ansible/network-connectivity.yml | 84 ++++++++++++++++++++++++++ ansible/test_plugins/functional.py | 27 +++++++++ doc/source/administration.rst | 14 +++++ doc/source/release-notes.rst | 2 + kayobe/cli/commands.py | 14 +++++ kayobe/tests/unit/cli/test_commands.py | 13 ++++ setup.py | 1 + 7 files changed, 155 insertions(+) create mode 100644 ansible/network-connectivity.yml create mode 100644 ansible/test_plugins/functional.py diff --git a/ansible/network-connectivity.yml b/ansible/network-connectivity.yml new file mode 100644 index 000000000..cf7a4043d --- /dev/null +++ b/ansible/network-connectivity.yml @@ -0,0 +1,84 @@ +--- +- name: Check network connectivity between hosts + hosts: seed:seed-hypervisor:overcloud + vars: + # Set this to an external IP address to check. + nc_external_ip: 8.8.8.8 + # Set this to an external hostname to check. + nc_external_hostname: google.com + # Number of bytes to subtract from MTU to allow for ICMP (8 bytes) and IP + # (20 bytes) headers. + icmp_overhead_bytes: 28 + tasks: + - name: Display next action + debug: + msg: > + Checking whether hosts have access to an external IP address, + {{ nc_external_ip }}. + run_once: True + + - name: Ensure an external IP is reachable + command: ping -c1 {{ nc_external_ip }} + changed_when: False + + - name: Display next action + debug: + msg: > + Checking whether hosts have access to an external hostname, + {{ nc_external_hostname }}. + run_once: True + + - name: Ensure an external host is reachable + command: ping -c1 {{ nc_external_hostname }} + changed_when: False + + - name: Display next action + debug: + msg: > + Checking whether hosts have access to any configured gateways. + run_once: True + + - name: Ensure the gateway is reachable + command: > + ping {{ item | net_gateway }} -c1 -M do {% if mtu %} -s {{ mtu | int - icmp_overhead_bytes }}{% endif %} + with_items: "{{ network_interfaces }}" + when: + - item | net_ip + - item | net_gateway + changed_when: False + vars: + mtu: "{{ item | net_mtu }}" + + # For each network on this host, pick a random remote host also on the + # network and try to ping it. Set the packet size according to the + # network's MTU. + + - name: Display next action + debug: + msg: > + Checking whether hosts have access to other hosts on the same + network. + run_once: True + + - name: Ensure hosts on the same network are reachable + command: > + ping {{ remote_ip }} -c1 -M do {% if mtu %} -s {{ mtu | int - icmp_overhead_bytes }}{% endif %} + with_items: "{{ network_interfaces }}" + when: + - item | net_ip + - remote_hosts | length > 0 + changed_when: False + vars: + # Select other hosts targeted by this play which have this network + # interface (item). + remote_hosts: > + {{ hostvars.values() | + selectattr('inventory_hostname', 'is_in', play_hosts) | + selectattr('network_interfaces', 'defined') | + selectattr('network_interfaces', 'issuperset', [item]) | + rejectattr('inventory_hostname', 'equalto', inventory_hostname) | + map(attribute='inventory_hostname') | + list }} + remote_host: "{{ remote_hosts | random }}" + remote_ip: "{{ item | net_ip(remote_host) }}" + mtu: "{{ item | net_mtu }}" diff --git a/ansible/test_plugins/functional.py b/ansible/test_plugins/functional.py new file mode 100644 index 000000000..b8b531232 --- /dev/null +++ b/ansible/test_plugins/functional.py @@ -0,0 +1,27 @@ +# Copyright (c) 2017 StackHPC Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def is_in(item, container): + """Tests whether an item is in a container (e.g. a list).""" + return item in container + + +class TestModule(object): + """'Functional' programming tests.""" + + def tests(self): + return { + 'is_in': is_in, + } diff --git a/doc/source/administration.rst b/doc/source/administration.rst index a896a764c..406f8dd90 100644 --- a/doc/source/administration.rst +++ b/doc/source/administration.rst @@ -111,6 +111,20 @@ specified directory, with one subdirectory per container. This command may be followed by ``kayobe ovecloud service configuration save`` to gather the generated configuration to the ansible control host. +Checking Network Connectivity +============================= + +In complex networking environments it can be useful to be able to automatically +check network connectivity and diagnose networking issues. To perform some +simple connectivity checks:: + + (kayobe) $ kayobe network connectivity check + +Note that this will run on the seed, seed hypervisor, and overcloud hosts. If +any of these hosts are not expected to be active (e.g. prior to overcloud +deployment), the set of target hosts may be limited using the ``--limit`` +argument. + Running Kayobe Playbooks on Demand ================================== diff --git a/doc/source/release-notes.rst b/doc/source/release-notes.rst index c05c2345a..3f3aec1a8 100644 --- a/doc/source/release-notes.rst +++ b/doc/source/release-notes.rst @@ -17,6 +17,8 @@ Features * Adds support for configuration of custom fluentd filters, and additional config file templates for heat, ironic, keystone, magnum, murano, sahara, and swift in ``$KAYOBE_CONFIG_PATH/kolla/config//``. +* Adds the command ``kayobe network connectivity check`` which can be used to + verify network connectivity in the cloud hosts. Upgrade Notes ------------- diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index e83be4fd0..e6a6534f5 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -972,3 +972,17 @@ class OvercloudPostConfigure(KayobeAnsibleMixin, VaultMixin, Command): "overcloud-introspection-rules-dell-lldp-workaround", "provision-net") self.run_kayobe_playbooks(parsed_args, playbooks) + + +class NetworkConnectivityCheck(KayobeAnsibleMixin, VaultMixin, Command): + """Check network connectivity between hosts in the control plane. + + Checks for access to an external IP address, an external hostname, any + configured gateways, and between hosts on the same subnets. The MTU of + each network is validated by sending ping packets of maximum size. + """ + + def take_action(self, parsed_args): + self.app.LOG.debug("Performing network connectivity check") + playbooks = _build_playbook_list("network-connectivity") + self.run_kayobe_playbooks(parsed_args, playbooks) diff --git a/kayobe/tests/unit/cli/test_commands.py b/kayobe/tests/unit/cli/test_commands.py index 27c7e6195..4c3d06891 100644 --- a/kayobe/tests/unit/cli/test_commands.py +++ b/kayobe/tests/unit/cli/test_commands.py @@ -68,3 +68,16 @@ class TestCase(unittest.TestCase): tags="install"), ] self.assertEqual(expected_calls, mock_run.call_args_list) + + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_playbooks") + def test_network_connectivity_check(self, mock_run): + command = commands.NetworkConnectivityCheck(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args([]) + result = command.run(parsed_args) + self.assertEqual(0, result) + expected_calls = [ + mock.call(mock.ANY, ["ansible/network-connectivity.yml"]), + ] + self.assertEqual(expected_calls, mock_run.call_args_list) diff --git a/setup.py b/setup.py index 3e862525f..20abb8d38 100644 --- a/setup.py +++ b/setup.py @@ -55,6 +55,7 @@ setup( 'control_host_upgrade = kayobe.cli.commands:ControlHostUpgrade', 'configuration_dump = kayobe.cli.commands:ConfigurationDump', 'kolla_ansible_run = kayobe.cli.commands:KollaAnsibleRun', + 'network_connectivity_check = kayobe.cli.commands:NetworkConnectivityCheck', 'overcloud_bios_raid_configure = kayobe.cli.commands:OvercloudBIOSRAIDConfigure', 'overcloud_container_image_build = kayobe.cli.commands:OvercloudContainerImageBuild', 'overcloud_container_image_pull = kayobe.cli.commands:OvercloudContainerImagePull',