From 8aac27fb83409f7a4652d146245c0a1c87397504 Mon Sep 17 00:00:00 2001 From: Tomas Sedovic Date: Wed, 13 Jul 2016 15:34:52 +0100 Subject: [PATCH] Port the network environment validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The validation and the associated library and unit tests: https://github.com/rthallisey/clapper/blob/master/ansible-tests/validations/network_environment.yaml Co-Authored-By: Martin André Co-Authored-By: Florian Fuchs Change-Id: Ie6bdfcaa78cf831bcb182a2de9fc813ae2fb83be --- ...vironment-validation-68f51e604819bfdf.yaml | 6 + tripleo_validations/tests/library/__init__.py | 0 .../tests/library/test_network_environment.py | 781 ++++++++++++++++++ validations/library/network_environment.py | 454 ++++++++++ validations/network-environment.yaml | 22 + 5 files changed, 1263 insertions(+) create mode 100644 releasenotes/notes/network-environment-validation-68f51e604819bfdf.yaml create mode 100644 tripleo_validations/tests/library/__init__.py create mode 100644 tripleo_validations/tests/library/test_network_environment.py create mode 100644 validations/library/network_environment.py create mode 100644 validations/network-environment.yaml diff --git a/releasenotes/notes/network-environment-validation-68f51e604819bfdf.yaml b/releasenotes/notes/network-environment-validation-68f51e604819bfdf.yaml new file mode 100644 index 000000000..89048dd48 --- /dev/null +++ b/releasenotes/notes/network-environment-validation-68f51e604819bfdf.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Added a network environment validation that checks network settings + based on environments/network-environment.yaml and nic config files + stored in the plan's Swift container. diff --git a/tripleo_validations/tests/library/__init__.py b/tripleo_validations/tests/library/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tripleo_validations/tests/library/test_network_environment.py b/tripleo_validations/tests/library/test_network_environment.py new file mode 100644 index 000000000..5ee91d292 --- /dev/null +++ b/tripleo_validations/tests/library/test_network_environment.py @@ -0,0 +1,781 @@ +# Copyright 2016 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from tripleo_validations.tests import base +import validations.library.network_environment as validation + + +class TestNicConfigs(base.TestCase): + + def test_non_dict(self): + errors = validation.check_nic_configs("controller.yaml", None) + self.assertEqual(len(errors), 1) + self.assertEqual('The nic_data parameter must be a dictionary.', + errors[0]) + + def _test_resources_invalid(self, nic_data): + errors = validation.check_nic_configs("controller.yaml", nic_data) + self.assertEqual(len(errors), 1) + self.assertEqual("The nic_data must contain the 'resources' key and it" + " must be a dictionary.", errors[0]) + + def test_resources_dict(self): + self._test_resources_invalid({}) + self._test_resources_invalid({'resources': None}) + + def test_resource_not_dict(self): + nic_data = {'resources': {'foo': None}} + errors = validation.check_nic_configs("controller.yaml", nic_data) + self.assertEqual(len(errors), 1) + self.assertEqual("'foo' is not a valid resource.", errors[0]) + + def test_resource_config_not_dict(self): + nic_data = {'resources': {'foo': {'properties': {'config': None}}}} + errors = validation.check_nic_configs("controller.yaml", nic_data) + self.assertEqual(len(errors), 1) + self.assertEqual("The 'config' property of 'foo' must be" + " a dictionary.", errors[0]) + + def test_get_network_config(self): + # Test config lookup using current format (t-h-t >= Ocata) + resources = { + 'properties': { + 'config': { + 'str_replace': { + 'params': { + '$network_config': { + 'network_config': [ + 'current' + ] + } + } + } + } + } + } + self.assertEqual( + validation.get_network_config(resources, 'foo')[0], + 'current') + + def test_get_network_config_returns_none_if_not_found(self): + # get_network_config should return None if + # any of the keys cannot be found in the resources tree: + # `properties`, `config`, `network_config` + no_properties = { + 'bar': { + 'config': { + 'str_replace': { + 'params': { + '$network_config': { + 'network_config': [ + 'current' + ] + } + } + } + } + } + } + no_config = { + 'properties': { + 'bar': { + 'str_replace': { + 'params': { + '$network_config': { + 'network_config': [ + 'current' + ] + } + } + } + } + } + } + no_network_config = { + 'properties': { + 'config': { + 'str_replace': { + 'params': { + '$network_config': { + 'bar': { + 'some': 'val' + } + } + } + } + } + } + } + self.assertEqual( + validation.get_network_config(no_properties, 'foo'), None) + self.assertEqual(validation.get_network_config(no_config, 'foo'), None) + self.assertEqual( + validation.get_network_config(no_network_config, 'foo'), None) + + def test_get_network_config_old_format(self): + # Test config lookup using format used in t-h-t <= Newton + resources = { + 'properties': { + 'config': { + 'os_net_config': { + 'network_config': [ + 'old' + ] + } + } + } + } + self.assertEqual( + validation.get_network_config(resources, 'foo')[0], + 'old') + + def nic_data(self, bridges): + return { + 'resources': { + 'foo': { + 'properties': { + 'config': { + 'str_replace': { + 'params': { + '$network_config': { + 'network_config': bridges + } + } + } + } + } + } + } + } + + def test_network_config_not_list(self): + nic_data = self.nic_data(None) + errors = validation.check_nic_configs("controller.yaml", nic_data) + self.assertEqual(len(errors), 1) + self.assertEqual("The 'network_config' property of 'foo' must be" + " a list.", errors[0]) + + def test_bridge_has_type(self): + nic_data = self.nic_data([{ + 'name': 'storage', + 'members': [], + }]) + errors = validation.check_nic_configs("controller.yaml", nic_data) + self.assertEqual(len(errors), 1) + self.assertIn('must have a type', errors[0]) + + def test_bridge_has_name(self): + nic_data = self.nic_data([{ + 'type': 'ovs_bridge', + 'members': [], + }]) + errors = validation.check_nic_configs("controller.yaml", nic_data) + self.assertEqual(len(errors), 1) + self.assertIn('must have a name', errors[0]) + + def test_ovs_bridge_has_members(self): + nic_data = self.nic_data([{ + 'name': 'storage', + 'type': 'ovs_bridge', + 'members': None, + }]) + errors = validation.check_nic_configs("controller.yaml", nic_data) + self.assertEqual(len(errors), 1) + self.assertIn("must contain a 'members' list", errors[0]) + + def test_ovs_bridge_members_dict(self): + nic_data = self.nic_data([{ + 'name': 'storage', + 'type': 'ovs_bridge', + 'members': [None], + }]) + errors = validation.check_nic_configs("controller.yaml", nic_data) + self.assertEqual(len(errors), 2) + self.assertIn("must be a dictionary.", errors[0]) + self.assertIn("at least 1 interface", errors[1]) + + def test_bonds_have_type(self): + nic_data = self.nic_data([{ + 'type': 'ovs_bridge', + 'name': 'storage', + 'members': [{}], + }]) + errors = validation.check_nic_configs("controller.yaml", nic_data) + self.assertEqual(len(errors), 2) + self.assertIn("must have a type.", errors[0]) + self.assertIn("at least 1 interface", errors[1]) + + def test_more_than_one_bond(self): + nic_data = self.nic_data([{ + 'type': 'ovs_bridge', + 'name': 'storage', + 'members': [ + {'type': 'ovs_bond'}, + {'type': 'ovs_bond'}, + ], + }]) + errors = validation.check_nic_configs("controller.yaml", nic_data) + self.assertEqual(len(errors), 1) + self.assertIn('Invalid bonding: There are 2 bonds for bridge storage', + errors[0]) + + def test_multiple_interfaces_without_bond(self): + nic_data = self.nic_data([{ + 'type': 'ovs_bridge', + 'name': 'storage', + 'members': [ + {'type': 'interface'}, + {'type': 'interface'}, + ], + }]) + errors = validation.check_nic_configs("controller.yaml", nic_data) + self.assertEqual(len(errors), 1) + self.assertIn('Invalid interface: When not using a bond, there can' + ' only be 1 interface for bridge storage', errors[0]) + + def test_one_interface_without_bond(self): + nic_data = self.nic_data([{ + 'type': 'ovs_bridge', + 'name': 'storage', + 'members': [ + {'type': 'interface'}, + ], + }]) + errors = validation.check_nic_configs("controller.yaml", nic_data) + self.assertEqual([], errors) + + def test_one_bond_no_interfaces(self): + nic_data = self.nic_data([{ + 'type': 'ovs_bridge', + 'name': 'storage', + 'members': [ + {'type': 'ovs_bond'}, + ], + }]) + errors = validation.check_nic_configs("controller.yaml", nic_data) + self.assertEqual([], errors) + + def test_one_bond_multiple_interfaces(self): + nic_data = self.nic_data([{ + 'type': 'ovs_bridge', + 'name': 'storage', + 'members': [ + {'type': 'ovs_bond'}, + {'type': 'interface'}, + {'type': 'interface'}, + ], + }]) + errors = validation.check_nic_configs("controller.yaml", nic_data) + self.assertEqual([], errors) + + +class TestCheckCidrOverlap(base.TestCase): + + def test_empty(self): + errors = validation.check_cidr_overlap([]) + self.assertEqual([], errors) + + def test_none(self): + errors = validation.check_cidr_overlap(None) + self.assertEqual(len(errors), 1) + self.assertEqual("The argument must be iterable.", errors[0]) + + def test_network_none(self): + errors = validation.check_cidr_overlap([None]) + self.assertEqual(len(errors), 1) + self.assertEqual("Invalid network: None", errors[0]) + + def test_single_network(self): + errors = validation.check_cidr_overlap(['172.16.0.0/24']) + self.assertEqual([], errors) + + def test_non_overlapping_networks(self): + networks = ['172.16.0.0/24', '172.17.0.0/24'] + errors = validation.check_cidr_overlap(networks) + self.assertEqual([], errors) + + def test_identical_networks(self): + networks = ['172.16.0.0/24', '172.16.0.0/24'] + errors = validation.check_cidr_overlap(networks) + self.assertEqual(len(errors), 1) + self.assertEqual('Networks 172.16.0.0/24 and 172.16.0.0/24 overlap.', + errors[0]) + + def test_first_cidr_is_subset_of_second(self): + networks = ['172.16.10.0/24', '172.16.0.0/16'] + errors = validation.check_cidr_overlap(networks) + self.assertEqual(len(errors), 1) + self.assertEqual('Networks 172.16.10.0/24 and 172.16.0.0/16 overlap.', + errors[0]) + + def test_second_cidr_is_subset_of_first(self): + networks = ['172.16.0.0/16', '172.16.10.0/24'] + errors = validation.check_cidr_overlap(networks) + self.assertEqual(len(errors), 1) + self.assertEqual('Networks 172.16.0.0/16 and 172.16.10.0/24 overlap.', + errors[0]) + + def test_multiple_overlapping_networks(self): + networks = ['172.16.0.0/16', '172.16.10.0/24', + '172.16.11.0/23', '172.17.0.0/24'] + errors = validation.check_cidr_overlap(networks) + self.assertEqual(len(errors), 3) + self.assertEqual('Networks 172.16.0.0/16 and 172.16.10.0/24 overlap.', + errors[0]) + self.assertEqual('Networks 172.16.0.0/16 and 172.16.11.0/23 overlap.', + errors[1]) + self.assertEqual('Networks 172.16.10.0/24 and 172.16.11.0/23 overlap.', + errors[2]) + + +class TestCheckAllocationPoolsPairing(base.TestCase): + + def test_empty(self): + errors = validation.check_allocation_pools_pairing({}, {}) + self.assertEqual([], errors) + + def test_non_dict(self): + errors = validation.check_allocation_pools_pairing(None, {}) + self.assertEqual(len(errors), 1) + self.assertEqual('The `filedata` argument must be a dictionary.', + errors[0]) + errors = validation.check_allocation_pools_pairing({}, None) + self.assertEqual(len(errors), 1) + self.assertEqual('The `pools` argument must be a dictionary.', + errors[0]) + + def test_pool_range_not_list(self): + pools = {'TestPools': None} + errors = validation.check_allocation_pools_pairing({}, pools) + self.assertEqual(len(errors), 1) + self.assertEqual('The IP ranges in TestPools must form a list.', + errors[0]) + + def _test_pool_invalid_range(self, addr_range): + filedata = {'TestNetCidr': '172.18.0.0/24'} + pools = {'TestAllocationPools': [addr_range]} + errors = validation.check_allocation_pools_pairing(filedata, pools) + self.assertEqual(len(errors), 1) + self.assertEqual('Invalid format of the IP range in' + ' TestAllocationPools: {}'.format(addr_range), + errors[0]) + + def test_pool_invalid_range(self): + broken_ranges = [None, + {}, + {'start': 'foo', 'end': 'bar'}, + {'start': '10.0.0.1', 'end': '10.0.0.0'}, + ] + for addr_range in broken_ranges: + self._test_pool_invalid_range(addr_range) + + def test_pool_with_correct_range(self): + filedata = { + 'StorageNetCidr': '172.18.0.0/24', + } + pools = { + 'StorageAllocationPools': [ + {'start': '172.18.0.10', 'end': '172.18.0.200'} + ] + } + errors = validation.check_allocation_pools_pairing(filedata, pools) + self.assertEqual([], errors) + + def test_pool_without_cidr(self): + filedata = {} + pools = { + 'StorageAllocationPools': [ + {'start': '172.18.0.10', 'end': '172.18.0.200'} + ] + } + errors = validation.check_allocation_pools_pairing(filedata, pools) + self.assertEqual(len(errors), 1) + self.assertEqual('The StorageNetCidr CIDR is not specified for' + ' StorageAllocationPools.', errors[0]) + + def test_pool_with_invalid_cidr(self): + filedata = { + 'StorageNetCidr': 'breakit', + } + pools = { + 'StorageAllocationPools': [ + {'start': '172.18.0.10', 'end': '172.18.0.200'} + ] + } + errors = validation.check_allocation_pools_pairing(filedata, pools) + self.assertEqual(len(errors), 1) + self.assertEqual('Invalid IP network: breakit', errors[0]) + + def test_pool_outside_cidr(self): + filedata = { + 'StorageNetCidr': '172.18.0.0/25', + } + pools = { + 'StorageAllocationPools': [ + {'start': '172.18.0.10', 'end': '172.18.0.200'} + ] + } + errors = validation.check_allocation_pools_pairing(filedata, pools) + self.assertEqual(len(errors), 1) + self.assertIn('outside of subnet StorageNetCidr', errors[0]) + + def test_multiple_ranges_and_pools(self): + filedata = { + 'StorageNetCidr': '172.18.0.0/24', + 'TenantNetCidr': '172.16.0.0/24', + } + pools = { + 'StorageAllocationPools': [ + {'start': '172.18.0.10', 'end': '172.18.0.20'}, + {'start': '172.18.0.100', 'end': '172.18.0.200'}, + ], + 'TenantAllocationPools': [ + {'start': '172.16.0.20', 'end': '172.16.0.30'}, + {'start': '172.16.0.70', 'end': '172.16.0.80'}, + ], + } + errors = validation.check_allocation_pools_pairing(filedata, pools) + self.assertEqual([], errors) + + +class TestStaticIpPoolCollision(base.TestCase): + + def test_empty(self): + errors = validation.check_static_ip_pool_collision({}, {}) + self.assertEqual([], errors) + + def test_non_dict(self): + errors = validation.check_static_ip_pool_collision(None, {}) + self.assertEqual(len(errors), 1) + self.assertEqual('The static IPs input must be a dictionary.', + errors[0]) + errors = validation.check_static_ip_pool_collision({}, None) + self.assertEqual(len(errors), 1) + self.assertEqual('The Pools input must be a dictionary.', + errors[0]) + + def test_pool_range_not_list(self): + pools = {'TestPools': None} + errors = validation.check_static_ip_pool_collision({}, pools) + self.assertEqual(len(errors), 1) + self.assertEqual('The IP ranges in TestPools must form a list.', + errors[0]) + + def _test_pool_invalid_range(self, addr_range): + static_ips = {} + pools = {'TestAllocationPools': [addr_range]} + errors = validation.check_static_ip_pool_collision(static_ips, pools) + self.assertEqual(len(errors), 1) + self.assertEqual('Invalid format of the IP range in' + ' TestAllocationPools: {}'.format(addr_range), + errors[0]) + + def test_pool_invalid_range(self): + broken_ranges = [None, + {}, + {'start': 'foo', 'end': 'bar'}, + {'start': '10.0.0.1', 'end': '10.0.0.0'}, + ] + for addr_range in broken_ranges: + self._test_pool_invalid_range(addr_range) + + def test_pool_with_correct_range(self): + static_ips = {} + pools = { + 'StorageAllocationPools': [ + {'start': '172.18.0.10', 'end': '172.18.0.200'} + ] + } + errors = validation.check_static_ip_pool_collision(static_ips, pools) + self.assertEqual([], errors) + + def test_static_ip_service_not_dict(self): + static_ips = {'ComputeIPs': None} + errors = validation.check_static_ip_pool_collision(static_ips, {}) + self.assertEqual(len(errors), 1) + self.assertEqual('The ComputeIPs must be a dictionary.', errors[0]) + + def test_static_ips_not_lists(self): + static_ips = { + 'ComputeIPs': { + 'internal_api': None + } + } + errors = validation.check_static_ip_pool_collision(static_ips, {}) + self.assertEqual(len(errors), 1) + self.assertEqual('The ComputeIPs->internal_api must be an array.', + errors[0]) + + def test_static_ips_not_parseable(self): + static_ips = { + 'ComputeIPs': { + 'internal_api': ['nonsense', None, '270.0.0.1'], + } + } + pools = {} + errors = validation.check_static_ip_pool_collision(static_ips, pools) + self.assertEqual(len(errors), 3) + self.assertIn('nonsense is not a valid IP address', errors[0]) + self.assertIn('None is not a valid IP address', errors[1]) + self.assertIn('270.0.0.1 is not a valid IP address', errors[2]) + + def test_static_ip_collide_with_pool(self): + static_ips = { + 'ControllerIps': { + 'internal_api': ['10.35.191.150', '10.35.191.60'] + } + } + pools = { + 'InternalApiAllocationPools': [ + {'start': '10.35.191.150', 'end': '10.35.191.240'} + ] + } + errors = validation.check_static_ip_pool_collision(static_ips, pools) + self.assertEqual(len(errors), 1) + self.assertEqual('IP address 10.35.191.150 from ' + 'ControllerIps[internal_api] is in the ' + 'InternalApiAllocationPools pool.', errors[0]) + + def test_static_ip_no_collisions(self): + static_ips = { + 'ControllerIps': { + 'internal_api': ['10.35.191.50', '10.35.191.60'], + 'storage': ['192.168.100.20', '192.168.100.30'], + }, + 'ComputeIps': { + 'internal_api': ['10.35.191.100', '10.35.191.110'], + 'storage': ['192.168.100.45', '192.168.100.46'] + } + } + pools = { + 'InternalApiAllocationPools': [ + {'start': '10.35.191.150', 'end': '10.35.191.240'} + ] + } + errors = validation.check_static_ip_pool_collision(static_ips, pools) + self.assertEqual([], errors) + + +class TestVlanIds(base.TestCase): + + def test_empty(self): + errors = validation.check_vlan_ids({}) + self.assertEqual([], errors) + + def test_non_dict(self): + errors = validation.check_vlan_ids(None) + self.assertEqual(len(errors), 1) + errors = validation.check_vlan_ids(42) + self.assertEqual(len(errors), 1) + errors = validation.check_vlan_ids("Ceci n'est pas un dict.") + self.assertEqual(len(errors), 1) + + def test_id_collision(self): + vlans = { + 'TenantNetworkVlanID': 204, + 'StorageMgmtNetworkVlanID': 203, + 'StorageNetworkVlanID': 202, + 'ExternalNetworkVlanID': 100, + 'InternalApiNetworkVlanID': 202, + } + errors = validation.check_vlan_ids(vlans) + self.assertEqual(len(errors), 1) + self.assertIn('Vlan ID 202', errors[0]) + self.assertIn('already exists', errors[0]) + + def test_id_no_collisions(self): + vlans = { + 'TenantNetworkVlanID': 204, + 'StorageMgmtNetworkVlanID': 203, + 'StorageNetworkVlanID': 202, + 'ExternalNetworkVlanID': 100, + 'InternalApiNetworkVlanID': 201, + } + errors = validation.check_vlan_ids(vlans) + self.assertEqual([], errors) + + +class TestStaticIpInCidr(base.TestCase): + + def test_empty(self): + errors = validation.check_static_ip_in_cidr({}, {}) + self.assertEqual([], errors) + + def test_non_dict(self): + errors = validation.check_static_ip_in_cidr(None, {}) + self.assertEqual(len(errors), 1) + self.assertEqual('The networks argument must be a dictionary.', + errors[0]) + errors = validation.check_static_ip_in_cidr({}, None) + self.assertEqual(len(errors), 1) + self.assertEqual('The static_ips argument must be a dictionary.', + errors[0]) + + def test_invalid_cidr(self): + errors = validation.check_static_ip_in_cidr( + {'StorageNetCidr': 'breakit'}, {}) + self.assertEqual(len(errors), 1) + self.assertEqual("Network 'StorageNetCidr' has an invalid CIDR:" + " 'breakit'", errors[0]) + + def test_service_not_a_dict(self): + static_ips = {'ControllerIps': None} + errors = validation.check_static_ip_in_cidr({}, static_ips) + self.assertEqual(len(errors), 1) + self.assertEqual('The ControllerIps must be a dictionary.', errors[0]) + + def test_static_ips_not_a_list(self): + networks = { + 'InternalApiNetCidr': '10.35.191.0/24', + } + static_ips = { + 'ControllerIps': { + 'internal_api': None, + } + } + errors = validation.check_static_ip_in_cidr(networks, static_ips) + self.assertEqual(len(errors), 1) + self.assertEqual('The ControllerIps->internal_api must be a list.', + errors[0]) + + def test_missing_cidr(self): + static_ips = { + 'ControllerIps': { + 'storage': ['192.168.100.120'] + } + } + errors = validation.check_static_ip_in_cidr({}, static_ips) + self.assertEqual(len(errors), 1) + self.assertEqual("Service 'storage' does not have a corresponding" + " range: 'StorageNetCidr'.", errors[0]) + + def test_address_not_within_cidr(self): + networks = { + 'StorageNetCidr': '192.168.100.0/24', + } + static_ips = { + 'ControllerIps': { + 'storage': ['192.168.100.120', '192.168.101.0'] + } + } + errors = validation.check_static_ip_in_cidr(networks, static_ips) + self.assertEqual(len(errors), 1) + self.assertEqual('The IP address 192.168.101.0 is outside of the' + ' StorageNetCidr range: 192.168.100.0/24', errors[0]) + + def test_addresses_within_cidr(self): + networks = { + 'StorageNetCidr': '192.168.100.0/24', + 'InternalApiNetCidr': '10.35.191.0/24', + } + static_ips = { + 'ControllerIps': { + 'storage': ['192.168.100.1', '192.168.100.2', '192.168.100.3'], + 'internal_api': ['10.35.191.60', '10.35.191.70'] + }, + 'ComputeIps': { + 'storage': ['192.168.100.125', '192.168.100.135'], + 'internal_api': ['10.35.191.100', '10.35.191.110'], + } + } + errors = validation.check_static_ip_in_cidr(networks, static_ips) + self.assertEqual([], errors) + + +class TestDuplicateStaticIps(base.TestCase): + + def test_empty(self): + errors = validation.duplicate_static_ips({}) + self.assertEqual([], errors) + + def test_not_a_dict(self): + errors = validation.duplicate_static_ips(None) + self.assertEqual(len(errors), 1) + self.assertEqual('The static_ips argument must be a dictionary.', + errors[0]) + + def test_service_not_a_dict(self): + static_ips = { + 'ControllerIps': None, + } + errors = validation.duplicate_static_ips(static_ips) + self.assertEqual(len(errors), 1) + self.assertEqual('The ControllerIps must be a dictionary.', + errors[0]) + + def test_static_ips_not_a_list(self): + static_ips = { + 'ControllerIps': { + 'internal_api': None, + } + } + errors = validation.duplicate_static_ips(static_ips) + self.assertEqual(len(errors), 1) + self.assertEqual('The ControllerIps->internal_api must be a list.', + errors[0]) + + def test_duplicate_ips_within_service(self): + static_ips = { + 'ControllerIps': { + 'internal_api': ['10.35.191.60', '10.35.191.60'] + }, + } + errors = validation.duplicate_static_ips(static_ips) + self.assertEqual(len(errors), 1) + self.assertIn('The 10.35.191.60 IP address was entered multiple times', + errors[0]) + + def test_duplicate_ips_across_services(self): + static_ips = { + 'ControllerIps': { + 'internal_api': ['10.35.191.60', '10.35.191.70'], + 'storage': ['192.168.100.1', '10.35.191.60', '192.168.100.3'], + }, + } + errors = validation.duplicate_static_ips(static_ips) + self.assertEqual(len(errors), 1) + self.assertIn('The 10.35.191.60 IP address was entered multiple times', + errors[0]) + + def test_duplicate_ips_across_roles(self): + static_ips = { + 'ControllerIps': { + 'storage': ['192.168.100.1', '192.168.100.2', '192.168.100.3'], + 'internal_api': ['10.35.191.60', '10.35.191.70'] + }, + 'ComputeIps': { + 'storage': ['192.168.100.125', '192.168.100.135'], + 'internal_api': ['10.35.191.60', '10.35.191.110'], + } + } + errors = validation.duplicate_static_ips(static_ips) + self.assertEqual(len(errors), 1) + self.assertIn('The 10.35.191.60 IP address was entered multiple times', + errors[0]) + + def test_no_duplicate_ips(self): + static_ips = { + 'ControllerIps': { + 'storage': ['192.168.100.1', '192.168.100.2', '192.168.100.3'], + 'internal_api': ['10.35.191.60', '10.35.191.70'] + }, + 'ComputeIps': { + 'storage': ['192.168.100.125', '192.168.100.135'], + 'internal_api': ['10.35.191.100', '10.35.191.110'], + } + } + errors = validation.duplicate_static_ips(static_ips) + self.assertEqual([], errors) diff --git a/validations/library/network_environment.py b/validations/library/network_environment.py new file mode 100644 index 000000000..4be412a33 --- /dev/null +++ b/validations/library/network_environment.py @@ -0,0 +1,454 @@ +#!/usr/bin/env python +# Copyright 2016 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +import itertools +import netaddr +import os.path +import yaml + +import six + +from ansible.module_utils.basic import AnsibleModule + + +def open_network_environment_files(netenv_path, template_files): + errors = [] + + try: + network_data = yaml.load(template_files[netenv_path]) + except Exception as e: + return ({}, {}, ["Can't open network environment file '{}': {}" + .format(netenv_path, e)]) + nic_configs = [] + resource_registry = network_data.get('resource_registry', {}) + for nic_name, relative_path in six.iteritems(resource_registry): + if nic_name.endswith("Net::SoftwareConfig"): + nic_config_path = os.path.normpath( + os.path.join(os.path.dirname(netenv_path), relative_path)) + try: + nic_configs.append(( + nic_name, nic_config_path, + yaml.load(template_files[nic_config_path]))) + except Exception as e: + errors.append( + "Can't open the resource '{}' reference file '{}': {}" + .format(nic_name, nic_config_path, e)) + + return (network_data, nic_configs, errors) + + +def validate(netenv_path, template_files): + network_data, nic_configs, errors = open_network_environment_files( + netenv_path, template_files) + errors.extend(validate_network_environment(network_data, nic_configs)) + return errors + + +def validate_network_environment(network_data, nic_configs): + errors = [] + + cidrinfo = {} + poolsinfo = {} + vlaninfo = {} + staticipinfo = {} + + for item, data in six.iteritems(network_data.get('parameter_defaults', + {})): + if item.endswith('NetCidr'): + cidrinfo[item] = data + elif item.endswith('AllocationPools'): + poolsinfo[item] = data + elif item.endswith('NetworkVlanID'): + vlaninfo[item] = data + elif item.endswith('IPs'): + staticipinfo[item] = data + + for nic_config_name, nic_config_path, nic_config in nic_configs: + errors.extend(check_nic_configs(nic_config_path, nic_config)) + + errors.extend(check_cidr_overlap(cidrinfo.values())) + errors.extend( + check_allocation_pools_pairing( + network_data.get('parameter_defaults', {}), poolsinfo)) + errors.extend(check_static_ip_pool_collision(staticipinfo, poolsinfo)) + errors.extend(check_vlan_ids(vlaninfo)) + errors.extend(check_static_ip_in_cidr(cidrinfo, staticipinfo)) + errors.extend(duplicate_static_ips(staticipinfo)) + + return errors + + +def get_network_config(resource, resource_name): + # Finds and returns `properties > config > network_config` inside + # a resources dictionary, with optional nesting levels in between. + + def deep_find_key(key_data, resource, resource_name): + key, instance_type, instance_name = key_data + if key in resource.keys(): + if not isinstance(resource[key], instance_type): + raise ValueError("The '{}' property of '{}' must be a {}." + "".format(key, resource_name, instance_name)) + return resource[key] + for item in resource.values(): + if isinstance(item, collections.Mapping): + return deep_find_key(key_data, item, resource_name) + return None + + keys = [ + ('properties', collections.Mapping, 'dictionary'), + ('config', collections.Mapping, 'dictionary'), + ('network_config', collections.Iterable, 'list'), + ] + current_value = resource + + if not isinstance(resource, collections.Mapping): + raise ValueError( + "'{}' is not a valid resource.".format(resource_name)) + + while len(keys) > 0: + key_data = keys.pop(0) + current_value = deep_find_key(key_data, current_value, resource_name) + if current_value is None: + break + + return current_value + + +def check_nic_configs(path, nic_data): + errors = [] + + if not isinstance(nic_data, collections.Mapping): + return ["The nic_data parameter must be a dictionary."] + + # Look though every resources bridges and make sure there is only a single + # bond per bridge and only 1 interface per bridge if there are no bonds. + resources = nic_data.get('resources') + if not isinstance(resources, collections.Mapping): + return ["The nic_data must contain the 'resources' key and it must be " + "a dictionary."] + for name, resource in six.iteritems(resources): + try: + bridges = get_network_config(resource, name) + except ValueError as e: + errors.append('{}'.format(e)) + continue + # Not all resources contain a network config: + if not bridges: + continue + for bridge in bridges: + if 'type' not in bridge: + errors.append("The bridge item {} in {} {} must have a type." + .format(bridge, name, path)) + continue + if 'name' not in bridge: + errors.append("The bridge item {} in {} {} must have a name." + .format(bridge, name, path)) + continue + if bridge['type'] == 'ovs_bridge': + bond_count = 0 + interface_count = 0 + if not isinstance(bridge.get('members'), collections.Iterable): + errors.append( + "OVS bridge {} in {} {} must contain a 'members' list." + .format(bridge, name, path)) + continue + for bridge_member in bridge['members']: + if not isinstance(bridge_member, collections.Mapping): + errors.append( + "The {} bridge member in {} {} must be a " + "dictionary." + .format(bridge_member, name, path)) + continue + if 'type' not in bridge_member: + errors.append( + "The {} bridge member in {} {} must have a type." + .format(bridge_member, name, path)) + continue + if bridge_member['type'] in ('ovs_bond', 'ovs_dpdk_bond'): + bond_count += 1 + elif bridge_member['type'] == 'interface': + interface_count += 1 + else: + # TODO(mandre) should we add an error for unknown + # bridge member types? + pass + + if bond_count == 2: + errors.append( + 'Invalid bonding: There are 2 bonds for' + ' bridge {} of resource {} in {}'.format( + bridge['name'], name, path)) + if bond_count == 0 and interface_count > 1: + errors.append( + 'Invalid interface: When not using a bond, ' + 'there can only be 1 interface for bridge {} ' + 'of resource {} in {}'.format( + bridge['name'], name, path)) + if bond_count == 0 and interface_count == 0: + errors.append( + 'Invalid config: There must be at least ' + '1 interface or 1 bond for bridge {}' + 'of resource {} in {}'.format( + bridge['name'], name, path)) + return errors + + +def check_cidr_overlap(networks): + errors = [] + objs = [] + if not isinstance(networks, collections.Iterable): + return ["The argument must be iterable."] + for x in networks: + try: + objs.append(netaddr.IPNetwork(x)) + except (ValueError, TypeError): + errors.append('Invalid network: {}'.format(x)) + + for net1, net2 in itertools.combinations(objs, 2): + if (net1 in net2 or net2 in net1): + errors.append( + 'Networks {} and {} overlap.' + .format(net1, net2)) + return errors + + +def check_allocation_pools_pairing(filedata, pools): + if not isinstance(filedata, collections.Mapping): + return ["The `filedata` argument must be a dictionary."] + if not isinstance(pools, collections.Mapping): + return ["The `pools` argument must be a dictionary."] + errors = [] + for poolitem, pooldata in six.iteritems(pools): + pool_objs = [] + if not isinstance(pooldata, collections.Iterable): + errors.append('The IP ranges in {} must form a list.' + .format(poolitem)) + continue + for dict_range in pooldata: + try: + pool_objs.append(netaddr.IPRange( + netaddr.IPAddress(dict_range['start']), + netaddr.IPAddress(dict_range['end']))) + except Exception: + errors.append("Invalid format of the IP range in {}: {}" + .format(poolitem, dict_range)) + continue + + subnet_item = poolitem.split('AllocationPools')[0] + 'NetCidr' + try: + network = filedata[subnet_item] + subnet_obj = netaddr.IPNetwork(network) + except KeyError: + errors.append('The {} CIDR is not specified for {}.' + .format(subnet_item, poolitem)) + continue + except Exception: + errors.append('Invalid IP network: {}'.format(network)) + continue + + for ranges in pool_objs: + for range in ranges: + if range not in subnet_obj: + errors.append('Allocation pool {} {} outside of subnet' + ' {}: {}'.format(poolitem, + pooldata, + subnet_item, + subnet_obj)) + break + return errors + + +def check_static_ip_pool_collision(static_ips, pools): + """Statically defined IP address must not conflict with allocation pools. + + The allocation pools come as a dict of items in the following format: + + InternalApiAllocationPools: [ + {'start': '10.35.191.150', 'end': '10.35.191.240'} + ] + + The static IP addresses are dicts of: + + ComputeIPs: { + 'internal_api': ['10.35.191.100', etc.], + 'storage': ['192.168.100.45', etc.] + } + """ + if not isinstance(static_ips, collections.Mapping): + return ["The static IPs input must be a dictionary."] + if not isinstance(pools, collections.Mapping): + return ["The Pools input must be a dictionary."] + errors = [] + pool_ranges = [] + for pool_name, ranges in six.iteritems(pools): + if not isinstance(ranges, collections.Iterable): + errors.append("The IP ranges in {} must form a list." + .format(pool_name)) + continue + for allocation_range in ranges: + try: + ip_range = netaddr.IPRange(allocation_range['start'], + allocation_range['end']) + except Exception: + errors.append("Invalid format of the IP range in {}: {}" + .format(pool_name, allocation_range)) + continue + pool_ranges.append((pool_name, ip_range)) + + for role, services in six.iteritems(static_ips): + if not isinstance(services, collections.Mapping): + errors.append("The {} must be a dictionary.".format(role)) + continue + for service, ips in six.iteritems(services): + if not isinstance(ips, collections.Iterable): + errors.append("The {}->{} must be an array." + .format(role, service)) + continue + for ip in ips: + try: + ip = netaddr.IPAddress(ip) + except netaddr.AddrFormatError as e: + errors.append("{} is not a valid IP address: {}" + .format(ip, e)) + continue + ranges_with_conflict = ranges_conflicting_with_ip( + ip, pool_ranges) + if ranges_with_conflict: + for pool_name, ip_range in ranges_with_conflict: + msg = "IP address {} from {}[{}] is in the {} pool." + errors.append(msg.format( + ip, role, service, pool_name)) + return errors + + +def ranges_conflicting_with_ip(ip_address, ip_ranges): + """Check for all conflicts of the IP address conflicts. + + This takes a single IP address and a list of `(pool_name, + netenv.IPRange)`s. + + We return all ranges that the IP address conflicts with. This is to + improve the final error messages. + """ + return [(pool_name, ip_range) for (pool_name, ip_range) in ip_ranges + if ip_address in ip_range] + + +def check_vlan_ids(vlans): + if not isinstance(vlans, collections.Mapping): + return ["The vlans parameter must be a dictionary."] + errors = [] + invertdict = {} + for k, v in six.iteritems(vlans): + if v not in invertdict: + invertdict[v] = k + else: + errors.append('Vlan ID {} ({}) already exists in {}'.format( + v, k, invertdict[v])) + return errors + + +def check_static_ip_in_cidr(networks, static_ips): + """Check all static IP addresses are from the corresponding network range. + + """ + if not isinstance(networks, collections.Mapping): + return ["The networks argument must be a dictionary."] + if not isinstance(static_ips, collections.Mapping): + return ["The static_ips argument must be a dictionary."] + errors = [] + network_ranges = {} + # TODO(shadower): Refactor this so networks are always valid and already + # converted to `netaddr.IPNetwork` here. Will be useful in the other + # checks. + for name, cidr in six.iteritems(networks): + try: + network_ranges[name] = netaddr.IPNetwork(cidr) + except Exception: + errors.append("Network '{}' has an invalid CIDR: '{}'" + .format(name, cidr)) + for role, services in six.iteritems(static_ips): + if not isinstance(services, collections.Mapping): + errors.append("The {} must be a dictionary.".format(role)) + continue + for service, ips in six.iteritems(services): + range_name = service.title().replace('_', '') + 'NetCidr' + if range_name in network_ranges: + if not isinstance(ips, collections.Iterable): + errors.append("The {}->{} must be a list." + .format(role, service)) + continue + for ip in ips: + if ip not in network_ranges[range_name]: + errors.append( + "The IP address {} is outside of the {} range: {}" + .format(ip, range_name, networks[range_name])) + else: + errors.append( + "Service '{}' does not have a " + "corresponding range: '{}'.".format(service, range_name)) + return errors + + +def duplicate_static_ips(static_ips): + errors = [] + if not isinstance(static_ips, collections.Mapping): + return ["The static_ips argument must be a dictionary."] + ipset = collections.defaultdict(list) + # TODO(shadower): we're doing this netsted loop multiple times. Turn it + # into a generator or something. + for role, services in six.iteritems(static_ips): + if not isinstance(services, collections.Mapping): + errors.append("The {} must be a dictionary.".format(role)) + continue + for service, ips in six.iteritems(services): + if not isinstance(ips, collections.Iterable): + errors.append("The {}->{} must be a list." + .format(role, service)) + continue + for ip in ips: + ipset[ip].append((role, service)) + for ip, sources in six.iteritems(ipset): + if len(sources) > 1: + msg = "The {} IP address was entered multiple times: {}." + formatted_sources = ("{}[{}]" + .format(*source) for source in sources) + errors.append(msg.format(ip, ", ".join(formatted_sources))) + return errors + + +def main(): + module = AnsibleModule(argument_spec=dict( + path=dict(required=True, type='str'), + template_files=dict(required=True, type='list') + )) + + netenv_path = module.params.get('path') + template_files = {name: content[1] for (name, content) in + module.params.get('template_files')} + + errors = validate(netenv_path, template_files) + + if errors: + module.fail_json(msg="\n".join(errors)) + else: + module.exit_json(msg="No errors found for the '{}' file.".format( + netenv_path)) + + +if __name__ == '__main__': + main() diff --git a/validations/network-environment.yaml b/validations/network-environment.yaml new file mode 100644 index 000000000..482606c5a --- /dev/null +++ b/validations/network-environment.yaml @@ -0,0 +1,22 @@ +--- +- hosts: undercloud + vars: + metadata: + name: Validate the Heat environment file for network configuration + description: > + This validates the network environment and nic-config files + that specify the overcloud network configuration and are stored + in the current plan's Swift container. + + The deployers are expected to write these files themselves as + described in the Network Isolation guide: + + http://tripleo.org/advanced_deployment/network_isolation.html + groups: + - pre-deployment + network_environment_path: environments/network-environment.yaml + tasks: + - name: Validate the network environment files + network_environment: + path: "{{ network_environment_path }}" + template_files: "{{ lookup('tht') }}"