From b63f8687f955fe53c5c8fba2a2c275182e0dd3ff Mon Sep 17 00:00:00 2001 From: James Page Date: Sat, 11 Nov 2017 19:51:53 +1100 Subject: [PATCH] Update for Python 3 execution Refactor codebase and unit tests to default to execution under Python 3. Drop install shim as Python 3 is always present >= trusty. Drop legacy dhcp and network reassignment code from charm as a) this relies on a py3 neutronclient (not supported on older releases) and b) this function was superceeded by the ha-legacy-mode and then neutron router and network HA built in functionality. Use charmhelper provided get_host_ip as this superceeds the in charm version of this function. Change-Id: I0b28bf0851d44e85b1e856cbd97b71099faa76ae --- .gitignore | 1 + actions/actions.py | 2 +- actions/git_reinstall.py | 2 +- actions/openstack_upgrade.py | 2 +- hooks/install | 21 +- hooks/install.real | 1 - hooks/neutron_contexts.py | 24 +- hooks/neutron_hooks.py | 9 +- hooks/neutron_utils.py | 99 +------- test-requirements.txt | 6 +- tox.ini | 3 +- unit_tests/__init__.py | 1 + unit_tests/test_neutron_contexts.py | 61 +---- unit_tests/test_neutron_hooks.py | 8 +- unit_tests/test_neutron_utils.py | 339 ++++++---------------------- unit_tests/test_utils.py | 2 +- 16 files changed, 100 insertions(+), 481 deletions(-) mode change 100755 => 120000 hooks/install delete mode 120000 hooks/install.real diff --git a/.gitignore b/.gitignore index 2de27a4a..8155fd77 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ wily xenial .unit-state.db func-results.json +__pycache__ diff --git a/actions/actions.py b/actions/actions.py index 5771496f..fd903d83 100755 --- a/actions/actions.py +++ b/actions/actions.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import os import sys diff --git a/actions/git_reinstall.py b/actions/git_reinstall.py index 4e12b1bb..1b3d32e7 100755 --- a/actions/git_reinstall.py +++ b/actions/git_reinstall.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import sys import traceback diff --git a/actions/openstack_upgrade.py b/actions/openstack_upgrade.py index 866f7a85..89b14f11 100755 --- a/actions/openstack_upgrade.py +++ b/actions/openstack_upgrade.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import sys sys.path.append('hooks/') diff --git a/hooks/install b/hooks/install deleted file mode 100755 index 83a9d3ce..00000000 --- a/hooks/install +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# Wrapper to deal with newer Ubuntu versions that don't have py2 installed -# by default. - -declare -a DEPS=('apt' 'netaddr' 'netifaces' 'pip' 'yaml') - -check_and_install() { - pkg="${1}-${2}" - if ! dpkg -s ${pkg} 2>&1 > /dev/null; then - apt-get -y install ${pkg} - fi -} - -PYTHON="python" - -for dep in ${DEPS[@]}; do - check_and_install ${PYTHON} ${dep} -done - -exec ./hooks/install.real diff --git a/hooks/install b/hooks/install new file mode 120000 index 00000000..fed1c963 --- /dev/null +++ b/hooks/install @@ -0,0 +1 @@ +neutron_hooks.py \ No newline at end of file diff --git a/hooks/install.real b/hooks/install.real deleted file mode 120000 index fed1c963..00000000 --- a/hooks/install.real +++ /dev/null @@ -1 +0,0 @@ -neutron_hooks.py \ No newline at end of file diff --git a/hooks/neutron_contexts.py b/hooks/neutron_contexts.py index 77401e4f..17eee2fe 100644 --- a/hooks/neutron_contexts.py +++ b/hooks/neutron_contexts.py @@ -1,16 +1,11 @@ # vim: set ts=4:et import os import uuid -import socket from charmhelpers.core.hookenv import ( config, unit_get, - cached, network_get_primary_address, ) -from charmhelpers.fetch import ( - apt_install, -) from charmhelpers.contrib.openstack.context import ( OSContextGenerator, NeutronAPIContext, @@ -21,6 +16,7 @@ from charmhelpers.contrib.hahelpers.cluster import( ) from charmhelpers.contrib.network.ip import ( get_address_in_network, + get_host_ip, ) NEUTRON_ML2_PLUGIN = "ml2" @@ -149,24 +145,6 @@ class NeutronGatewayContext(NeutronAPIContext): return ctxt -@cached -def get_host_ip(hostname=None): - try: - import dns.resolver - except ImportError: - apt_install('python-dnspython', fatal=True) - import dns.resolver - hostname = hostname or unit_get('private-address') - try: - # Test to see if already an IPv4 address - socket.inet_aton(hostname) - return hostname - except socket.error: - answers = dns.resolver.query(hostname, 'A') - if answers: - return answers[0].address - - SHARED_SECRET = "/etc/{}/secret.txt" diff --git a/hooks/neutron_hooks.py b/hooks/neutron_hooks.py index 1f4e96c5..8e6ae316 100755 --- a/hooks/neutron_hooks.py +++ b/hooks/neutron_hooks.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 from base64 import b64decode @@ -25,7 +25,6 @@ from charmhelpers.core.host import ( ) from charmhelpers.contrib.hahelpers.cluster import( get_hacluster_config, - eligible_leader ) from charmhelpers.contrib.hahelpers.apache import( install_ca_cert @@ -64,7 +63,6 @@ from neutron_utils import ( remove_legacy_ha_files, install_legacy_ha_files, cleanup_ovs_netns, - reassign_agent_resources, stop_neutron_ha_monitor_daemon, use_l3ha, NEUTRON_COMMON, @@ -77,7 +75,7 @@ hooks = Hooks() CONFIGS = register_configs() -@hooks.hook('install.real') +@hooks.hook('install') @harden() def install(): status_set('maintenance', 'Executing pre-install') @@ -260,9 +258,6 @@ def cluster_departed(): log('Unable to re-assign agent resources for failed nodes with n1kv', level=WARNING) return - if not config('ha-legacy-mode') and eligible_leader(None): - reassign_agent_resources() - CONFIGS.write_all() @hooks.hook('cluster-relation-broken') diff --git a/hooks/neutron_utils.py b/hooks/neutron_utils.py index e227f554..60193c73 100644 --- a/hooks/neutron_utils.py +++ b/hooks/neutron_utils.py @@ -23,8 +23,6 @@ from charmhelpers.core.hookenv import ( INFO, ERROR, config, - relations_of_type, - unit_private_ip, is_relation_made, relation_ids, ) @@ -51,7 +49,6 @@ from charmhelpers.contrib.openstack.utils import ( git_install_requested, git_pip_venv_dir, git_src_dir, - get_hostname, make_assess_status_func, os_release, pause_unit, @@ -683,7 +680,7 @@ def stop_services(): plugin = config('plugin') config_files = resolve_config_files(plugin, release) svcs = set() - for ctxt in config_files[config('plugin')].itervalues(): + for ctxt in config_files[config('plugin')].values(): for svc in ctxt['services']: svcs.add(remap_service(svc)) for svc in svcs: @@ -705,7 +702,7 @@ def restart_map(release=None): config_files = resolve_config_files(plugin, release) _map = {} enable_vpn_agent = 'neutron-vpn-agent' in get_packages() - for f, ctxt in config_files[plugin].iteritems(): + for f, ctxt in config_files[plugin].items(): svcs = set() for svc in ctxt['services']: svcs.add(remap_service(svc)) @@ -725,90 +722,6 @@ def restart_map(release=None): INT_BRIDGE = "br-int" EXT_BRIDGE = "br-ex" -DHCP_AGENT = "DHCP Agent" -L3_AGENT = "L3 Agent" - - -# TODO: make work with neutron -def reassign_agent_resources(): - ''' Use agent scheduler API to detect down agents and re-schedule ''' - env = NetworkServiceContext()() - if not env: - log('Unable to re-assign resources at this time') - return - try: - from quantumclient.v2_0 import client - except ImportError: - ''' Try to import neutronclient instead for havana+ ''' - from neutronclient.v2_0 import client - - auth_url = '%(auth_protocol)s://%(keystone_host)s:%(auth_port)s/v2.0' % env - quantum = client.Client(username=env['service_username'], - password=env['service_password'], - tenant_name=env['service_tenant'], - auth_url=auth_url, - region_name=env['region']) - - partner_gateways = [unit_private_ip().split('.')[0]] - for partner_gateway in relations_of_type(reltype='cluster'): - gateway_hostname = get_hostname(partner_gateway['private-address']) - partner_gateways.append(gateway_hostname.partition('.')[0]) - - agents = quantum.list_agents(agent_type=DHCP_AGENT) - dhcp_agents = [] - l3_agents = [] - networks = {} - for agent in agents['agents']: - if not agent['alive']: - log('DHCP Agent %s down' % agent['id']) - for network in \ - quantum.list_networks_on_dhcp_agent( - agent['id'])['networks']: - networks[network['id']] = agent['id'] - else: - if agent['host'].partition('.')[0] in partner_gateways: - dhcp_agents.append(agent['id']) - - agents = quantum.list_agents(agent_type=L3_AGENT) - routers = {} - for agent in agents['agents']: - if not agent['alive']: - log('L3 Agent %s down' % agent['id']) - for router in \ - quantum.list_routers_on_l3_agent( - agent['id'])['routers']: - routers[router['id']] = agent['id'] - else: - if agent['host'].split('.')[0] in partner_gateways: - l3_agents.append(agent['id']) - - if len(dhcp_agents) == 0 or len(l3_agents) == 0: - log('Unable to relocate resources, there are %s dhcp_agents and %s \ - l3_agents in this cluster' % (len(dhcp_agents), len(l3_agents))) - return - - index = 0 - for router_id in routers: - agent = index % len(l3_agents) - log('Moving router %s from %s to %s' % - (router_id, routers[router_id], l3_agents[agent])) - quantum.remove_router_from_l3_agent(l3_agent=routers[router_id], - router_id=router_id) - quantum.add_router_to_l3_agent(l3_agent=l3_agents[agent], - body={'router_id': router_id}) - index += 1 - - index = 0 - for network_id in networks: - agent = index % len(dhcp_agents) - log('Moving network %s from %s to %s' % - (network_id, networks[network_id], dhcp_agents[agent])) - quantum.remove_network_from_dhcp_agent(dhcp_agent=networks[network_id], - network_id=network_id) - quantum.add_network_to_dhcp_agent(dhcp_agent=dhcp_agents[agent], - body={'network_id': network_id}) - index += 1 - def services(): ''' Returns a list of services associate with this charm ''' @@ -862,12 +775,12 @@ def configure_ovs(): portmaps = DataPortContext()() bridgemaps = parse_bridge_mappings(config('bridge-mappings')) - for provider, br in bridgemaps.iteritems(): + for br in bridgemaps.values(): add_bridge(br) if not portmaps: continue - for port, _br in portmaps.iteritems(): + for port, _br in portmaps.items(): if _br == br: add_bridge_port(br, port, promisc=True) @@ -908,13 +821,13 @@ def remove_file(path): def install_legacy_ha_files(force=False): - for f, p in LEGACY_FILES_MAP.iteritems(): + for f, p in LEGACY_FILES_MAP.items(): srcfile = os.path.join(LEGACY_HA_TEMPLATE_FILES, f) copy_file(srcfile, p['path'], p.get('permissions', None), force=force) def remove_legacy_ha_files(): - for f, p in LEGACY_FILES_MAP.iteritems(): + for f, p in LEGACY_FILES_MAP.items(): remove_file(os.path.join(p['path'], f)) diff --git a/test-requirements.txt b/test-requirements.txt index 9edd4bbf..db21f458 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,12 +5,12 @@ coverage>=3.6 mock>=1.2 flake8>=2.2.4,<=2.4.1 os-testr>=0.4.1 -charm-tools>=2.0.0 +charm-tools>=2.0.0;python_version=='2.7' requests==2.6.0 # BEGIN: Amulet OpenStack Charm Helper Requirements # Liberty client lower constraints -amulet>=1.14.3,<2.0 -bundletester>=0.6.1,<1.0 +amulet>=1.14.3,<2.0;python_version=='2.7' +bundletester>=0.6.1,<1.0;python_version=='2.7' python-ceilometerclient>=1.5.0 python-cinderclient>=1.4.0 python-glanceclient>=1.1.0 diff --git a/tox.ini b/tox.ini index 6d44f4b9..5a89e6da 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ # This file is managed centrally by release-tools and should not be modified # within individual charm repos. [tox] -envlist = pep8,py27 +envlist = pep8,py27,py35 skipsdist = True [testenv] @@ -20,6 +20,7 @@ passenv = HOME TERM AMULET_* CS_API_* basepython = python2.7 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt +commands = /bin/true [testenv:py35] basepython = python3.5 diff --git a/unit_tests/__init__.py b/unit_tests/__init__.py index c9fd5a96..c9c8fd1a 100644 --- a/unit_tests/__init__.py +++ b/unit_tests/__init__.py @@ -1,4 +1,5 @@ import sys +sys.path.append('unit_tests') sys.path.append('actions') sys.path.append('hooks') diff --git a/unit_tests/test_neutron_contexts.py b/unit_tests/test_neutron_contexts.py index 23a55ff1..6baeeb1d 100644 --- a/unit_tests/test_neutron_contexts.py +++ b/unit_tests/test_neutron_contexts.py @@ -1,18 +1,19 @@ + +import io + +from contextlib import contextmanager + from mock import ( - Mock, MagicMock, patch ) import neutron_contexts -import sys -from contextlib import contextmanager from test_utils import ( CharmTestCase ) TO_PATCH = [ - 'apt_install', 'config', 'eligible_leader', 'unit_get', @@ -27,14 +28,14 @@ def patch_open(): Yields the mock for "open" and "file", respectively.''' mock_open = MagicMock(spec=open) - mock_file = MagicMock(spec=file) + mock_file = MagicMock(spec=io.FileIO) @contextmanager def stub_open(*args, **kwargs): mock_open(*args, **kwargs) yield mock_file - with patch('__builtin__.open', stub_open): + with patch('builtins.open', stub_open): yield mock_open, mock_file @@ -323,54 +324,6 @@ class TestSharedSecret(CharmTestCase): neutron_contexts.SHARED_SECRET.format('neutron'), 'r') -class TestHostIP(CharmTestCase): - - def setUp(self): - super(TestHostIP, self).setUp(neutron_contexts, - TO_PATCH) - self.config.side_effect = self.test_config.get - self.network_get_primary_address.side_effect = NotImplementedError - # Save and inject - self.mods = {'dns': None, 'dns.resolver': None} - for mod in self.mods: - if mod not in sys.modules: - sys.modules[mod] = Mock() - else: - del self.mods[mod] - - def tearDown(self): - super(TestHostIP, self).tearDown() - # Cleanup - for mod in self.mods.keys(): - del sys.modules[mod] - - def test_get_host_ip_already_ip(self): - self.assertEqual(neutron_contexts.get_host_ip('10.5.0.1'), - '10.5.0.1') - - def test_get_host_ip_noarg(self): - self.unit_get.return_value = "10.5.0.1" - self.assertEqual(neutron_contexts.get_host_ip(), - '10.5.0.1') - - @patch('dns.resolver.query') - def test_get_host_ip_hostname_unresolvable(self, _query): - class NXDOMAIN(Exception): - pass - _query.side_effect = NXDOMAIN() - self.assertRaises(NXDOMAIN, neutron_contexts.get_host_ip, - 'missing.example.com') - - @patch('dns.resolver.query') - def test_get_host_ip_hostname_resolvable(self, _query): - data = MagicMock() - data.address = '10.5.0.1' - _query.return_value = [data] - self.assertEqual(neutron_contexts.get_host_ip('myhost.example.com'), - '10.5.0.1') - _query.assert_called_with('myhost.example.com', 'A') - - class TestMisc(CharmTestCase): def setUp(self): diff --git a/unit_tests/test_neutron_hooks.py b/unit_tests/test_neutron_hooks.py index 53b8f0cc..77fcbaec 100644 --- a/unit_tests/test_neutron_hooks.py +++ b/unit_tests/test_neutron_hooks.py @@ -83,7 +83,7 @@ class TestQuantumHooks(CharmTestCase): self.valid_plugin.return_value = True _pkgs = ['foo', 'bar'] self.filter_installed_packages.return_value = _pkgs - self._call_hook('install.real') + self._call_hook('install') self.configure_installation_source.assert_called_with( 'cloud:precise-havana' ) @@ -99,7 +99,7 @@ class TestQuantumHooks(CharmTestCase): def test_install_hook_precise_nocloudarchive(self): self.test_config.set('openstack-origin', 'distro') - self._call_hook('install.real') + self._call_hook('install') self.configure_installation_source.assert_called_with( 'cloud:precise-icehouse' ) @@ -107,7 +107,7 @@ class TestQuantumHooks(CharmTestCase): @patch('sys.exit') def test_install_hook_invalid_plugin(self, _exit): self.valid_plugin.return_value = False - self._call_hook('install.real') + self._call_hook('install') self.assertTrue(self.log.called) _exit.assert_called_with(1) @@ -132,7 +132,7 @@ class TestQuantumHooks(CharmTestCase): projects_yaml = yaml.dump(openstack_origin_git) self.test_config.set('openstack-origin', repo) self.test_config.set('openstack-origin-git', projects_yaml) - self._call_hook('install.real') + self._call_hook('install') self.configure_installation_source.assert_called_with( 'cloud:trusty-juno' ) diff --git a/unit_tests/test_neutron_utils.py b/unit_tests/test_neutron_utils.py index eb6d02bf..100d167c 100644 --- a/unit_tests/test_neutron_utils.py +++ b/unit_tests/test_neutron_utils.py @@ -1,5 +1,3 @@ -import collections - from mock import MagicMock, call, patch, ANY import charmhelpers.core.hookenv as hookenv @@ -31,8 +29,6 @@ TO_PATCH = [ 'service_running', 'NetworkServiceContext', 'ExternalPortContext', - 'unit_private_ip', - 'relations_of_type', 'render', 'service_stop', 'determine_dkms_package', @@ -58,20 +54,11 @@ openstack_origin_git = \ class TestNeutronUtils(CharmTestCase): - def assertDictEqual(self, d1, d2, msg=None): # assertEqual uses for dicts - for k, v1 in d1.iteritems(): - self.assertIn(k, d2, msg) - v2 = d2[k] - if(isinstance(v1, collections.Iterable) and - not isinstance(v1, basestring)): - self.assertItemsEqual(v1, v2, msg) - else: - self.assertEqual(v1, v2, msg) - def setUp(self): super(TestNeutronUtils, self).setUp(neutron_utils, TO_PATCH) self.headers_package.return_value = 'linux-headers-2.6.18' self._set_distrib_codename('trusty') + self.maxDiff = None def tearDown(self): # Reset cached cache @@ -286,7 +273,7 @@ class TestNeutronUtils(CharmTestCase): ]) calls = [call('br1', 'eth0.100', promisc=True), call('br1', 'eth0.200', promisc=True)] - self.add_bridge_port.assert_has_calls(calls) + self.add_bridge_port.assert_has_calls(calls, any_order=True) @patch.object(neutron_utils, 'register_configs') @patch('charmhelpers.contrib.openstack.templating.OSConfigRenderer') @@ -373,12 +360,12 @@ class TestNeutronUtils(CharmTestCase): mock_get_packages.return_value = ['neutron-vpn-agent'] self.os_release.return_value = 'icehouse' ex_map = { - neutron_utils.NEUTRON_CONF: ['neutron-dhcp-agent', - 'neutron-metadata-agent', + neutron_utils.NEUTRON_CONF: ['neutron-lbaas-agent', 'neutron-plugin-openvswitch-agent', + 'neutron-vpn-agent', + 'neutron-dhcp-agent', 'neutron-metering-agent', - 'neutron-lbaas-agent', - 'neutron-vpn-agent'], + 'neutron-metadata-agent'], neutron_utils.NEUTRON_DNSMASQ_CONF: ['neutron-dhcp-agent'], neutron_utils.NEUTRON_LBAAS_AGENT_CONF: ['neutron-lbaas-agent'], @@ -409,8 +396,7 @@ class TestNeutronUtils(CharmTestCase): neutron_utils.NOVA_API_METADATA_AA_PROFILE_PATH: ['nova-api-metadata'], } - - self.assertDictEqual(neutron_utils.restart_map(), ex_map) + self.assertEqual(neutron_utils.restart_map(), ex_map) @patch.object(neutron_utils, 'get_packages') def test_restart_map_ovs_mitaka(self, mock_get_packages): @@ -418,12 +404,12 @@ class TestNeutronUtils(CharmTestCase): mock_get_packages.return_value = ['neutron-vpn-agent'] self.os_release.return_value = 'mitaka' ex_map = { - neutron_utils.NEUTRON_CONF: ['neutron-dhcp-agent', - 'neutron-metadata-agent', - 'neutron-openvswitch-agent', + neutron_utils.NEUTRON_CONF: ['neutron-lbaas-agent', + 'neutron-vpn-agent', + 'neutron-dhcp-agent', 'neutron-metering-agent', - 'neutron-lbaas-agent', - 'neutron-vpn-agent'], + 'neutron-metadata-agent', + 'neutron-openvswitch-agent'], neutron_utils.NEUTRON_DNSMASQ_CONF: ['neutron-dhcp-agent'], neutron_utils.NEUTRON_LBAAS_AGENT_CONF: ['neutron-lbaas-agent'], @@ -453,7 +439,7 @@ class TestNeutronUtils(CharmTestCase): neutron_utils.NOVA_API_METADATA_AA_PROFILE_PATH: ['nova-api-metadata'], } - self.assertEqual(ex_map, neutron_utils.restart_map()) + self.assertEqual(neutron_utils.restart_map(), ex_map) @patch.object(neutron_utils, 'get_packages') def test_restart_map_ovs_newton(self, mock_get_packages): @@ -461,12 +447,12 @@ class TestNeutronUtils(CharmTestCase): mock_get_packages.return_value = ['neutron-vpn-agent'] self.os_release.return_value = 'newton' ex_map = { - neutron_utils.NEUTRON_CONF: ['neutron-dhcp-agent', - 'neutron-metadata-agent', - 'neutron-openvswitch-agent', - 'neutron-metering-agent', + neutron_utils.NEUTRON_CONF: ['neutron-vpn-agent', 'neutron-lbaasv2-agent', - 'neutron-vpn-agent'], + 'neutron-dhcp-agent', + 'neutron-metering-agent', + 'neutron-metadata-agent', + 'neutron-openvswitch-agent'], neutron_utils.NEUTRON_DNSMASQ_CONF: ['neutron-dhcp-agent'], neutron_utils.NEUTRON_LBAAS_AGENT_CONF: ['neutron-lbaasv2-agent'], @@ -483,8 +469,20 @@ class TestNeutronUtils(CharmTestCase): neutron_utils.NOVA_CONF: ['nova-api-metadata'], neutron_utils.EXT_PORT_CONF: ['ext-port'], neutron_utils.PHY_NIC_MTU_CONF: ['os-charm-phy-nic-mtu'], + neutron_utils.NEUTRON_DHCP_AA_PROFILE_PATH: ['neutron-dhcp-agent'], + neutron_utils.NEUTRON_OVS_AA_PROFILE_PATH: + ['neutron-openvswitch-agent'], + neutron_utils.NEUTRON_L3_AA_PROFILE_PATH: ['neutron-vpn-agent'], + neutron_utils.NEUTRON_LBAASV2_AA_PROFILE_PATH: + ['neutron-lbaasv2-agent'], + neutron_utils.NEUTRON_METADATA_AA_PROFILE_PATH: + ['neutron-metadata-agent'], + neutron_utils.NEUTRON_METERING_AA_PROFILE_PATH: + ['neutron-metering-agent'], + neutron_utils.NOVA_API_METADATA_AA_PROFILE_PATH: + ['nova-api-metadata'], } - self.assertEqual(ex_map, neutron_utils.restart_map()) + self.assertEqual(neutron_utils.restart_map(), ex_map) @patch.object(neutron_utils, 'get_packages') def test_restart_map_ovs_post_trusty(self, mock_get_packages): @@ -493,7 +491,7 @@ class TestNeutronUtils(CharmTestCase): mock_get_packages.return_value = ['neutron-l3-agent'] self.os_release.return_value = 'diablo' rmap = neutron_utils.restart_map() - for services in rmap.itervalues(): + for services in rmap.values(): self.assertFalse('neutron-vpn-agent' in services) @patch.object(neutron_utils, 'get_packages') @@ -502,11 +500,11 @@ class TestNeutronUtils(CharmTestCase): mock_get_packages.return_value = ['neutron-vpn-agent'] self.os_release.return_value = 'icehouse' ex_map = { - neutron_utils.NEUTRON_CONF: ['neutron-dhcp-agent', - 'neutron-metadata-agent', + neutron_utils.NEUTRON_CONF: ['neutron-lbaas-agent', + 'neutron-vpn-agent', + 'neutron-dhcp-agent', 'neutron-metering-agent', - 'neutron-lbaas-agent', - 'neutron-vpn-agent'], + 'neutron-metadata-agent'], neutron_utils.NEUTRON_DNSMASQ_CONF: ['neutron-dhcp-agent'], neutron_utils.NEUTRON_LBAAS_AGENT_CONF: ['neutron-lbaas-agent'], @@ -522,7 +520,6 @@ class TestNeutronUtils(CharmTestCase): neutron_utils.EXT_PORT_CONF: ['ext-port'], neutron_utils.PHY_NIC_MTU_CONF: ['os-charm-phy-nic-mtu'], neutron_utils.NEUTRON_DHCP_AA_PROFILE_PATH: ['neutron-dhcp-agent'], - neutron_utils.NEUTRON_L3_AA_PROFILE_PATH: ['neutron-vpn-agent'], neutron_utils.NEUTRON_LBAAS_AA_PROFILE_PATH: ['neutron-lbaas-agent'], neutron_utils.NEUTRON_METADATA_AA_PROFILE_PATH: @@ -533,7 +530,7 @@ class TestNeutronUtils(CharmTestCase): ['nova-api-metadata'], } - self.assertDictEqual(neutron_utils.restart_map(), ex_map) + self.assertEqual(neutron_utils.restart_map(), ex_map) @patch.object(neutron_utils, 'get_packages') def test_restart_map_ovs_odl_newton(self, mock_get_packages): @@ -541,11 +538,11 @@ class TestNeutronUtils(CharmTestCase): mock_get_packages.return_value = ['neutron-vpn-agent'] self.os_release.return_value = 'newton' ex_map = { - neutron_utils.NEUTRON_CONF: ['neutron-dhcp-agent', - 'neutron-metadata-agent', - 'neutron-metering-agent', + neutron_utils.NEUTRON_CONF: ['neutron-vpn-agent', 'neutron-lbaasv2-agent', - 'neutron-vpn-agent'], + 'neutron-dhcp-agent', + 'neutron-metering-agent', + 'neutron-metadata-agent'], neutron_utils.NEUTRON_DNSMASQ_CONF: ['neutron-dhcp-agent'], neutron_utils.NEUTRON_LBAAS_AGENT_CONF: ['neutron-lbaasv2-agent'], @@ -561,7 +558,6 @@ class TestNeutronUtils(CharmTestCase): neutron_utils.EXT_PORT_CONF: ['ext-port'], neutron_utils.PHY_NIC_MTU_CONF: ['os-charm-phy-nic-mtu'], neutron_utils.NEUTRON_DHCP_AA_PROFILE_PATH: ['neutron-dhcp-agent'], - neutron_utils.NEUTRON_L3_AA_PROFILE_PATH: ['neutron-vpn-agent'], neutron_utils.NEUTRON_LBAASV2_AA_PROFILE_PATH: ['neutron-lbaasv2-agent'], neutron_utils.NEUTRON_METADATA_AA_PROFILE_PATH: @@ -746,125 +742,6 @@ class DummyExternalPortContext(): def __call__(self): return self.return_value -agents_all_alive = { - 'DHCP Agent': { - 'agents': [ - {'alive': True, - 'host': 'cluster1-machine1.internal', - 'id': '3e3550f2-38cc-11e3-9617-3c970e8b1cf7'}, - {'alive': True, - 'host': 'cluster1-machine2.internal', - 'id': '53d6eefc-38cc-11e3-b3c8-3c970e8b1cf7'}, - {'alive': True, - 'host': 'cluster2-machine1.internal', - 'id': '92b8b6bc-38ce-11e3-8537-3c970e8b1cf7'}, - {'alive': True, - 'host': 'cluster2-machine3.internal', - 'id': 'ebdcc950-51c8-11e3-a804-1c6f65b044df'}, - ] - }, - 'L3 Agent': { - 'agents': [ - {'alive': True, - 'host': 'cluster1-machine1.internal', - 'id': '7128198e-38ce-11e3-ba78-3c970e8b1cf7'}, - {'alive': True, - 'host': 'cluster1-machine2.internal', - 'id': '72453824-38ce-11e3-938e-3c970e8b1cf7'}, - {'alive': True, - 'host': 'cluster2-machine1.internal', - 'id': '84a04126-38ce-11e3-9449-3c970e8b1cf7'}, - {'alive': True, - 'host': 'cluster2-machine3.internal', - 'id': '00f4268a-51c9-11e3-9177-1c6f65b044df'}, - ] - } -} - -agents_some_dead_cl1 = { - 'DHCP Agent': { - 'agents': [ - {'alive': False, - 'host': 'cluster1-machine1.internal', - 'id': '3e3550f2-38cc-11e3-9617-3c970e8b1cf7'}, - {'alive': True, - 'host': 'cluster2-machine1.internal', - 'id': '53d6eefc-38cc-11e3-b3c8-3c970e8b1cf7'}, - {'alive': True, - 'host': 'cluster2-machine2.internal', - 'id': '92b8b6bc-38ce-11e3-8537-3c970e8b1cf7'}, - {'alive': True, - 'host': 'cluster2-machine3.internal', - 'id': 'ebdcc950-51c8-11e3-a804-1c6f65b044df'}, - ] - }, - 'L3 Agent': { - 'agents': [ - {'alive': False, - 'host': 'cluster1-machine1.internal', - 'id': '7128198e-38ce-11e3-ba78-3c970e8b1cf7'}, - {'alive': True, - 'host': 'cluster2-machine1.internal', - 'id': '72453824-38ce-11e3-938e-3c970e8b1cf7'}, - {'alive': True, - 'host': 'cluster2-machine2.internal', - 'id': '84a04126-38ce-11e3-9449-3c970e8b1cf7'}, - {'alive': True, - 'host': 'cluster2-machine3.internal', - 'id': '00f4268a-51c9-11e3-9177-1c6f65b044df'}, - ] - } -} - -agents_some_dead_cl2 = { - 'DHCP Agent': { - 'agents': [ - {'alive': True, - 'host': 'cluster1-machine1.internal', - 'id': '3e3550f2-38cc-11e3-9617-3c970e8b1cf7'}, - {'alive': True, - 'host': 'cluster2-machine1.internal', - 'id': '53d6eefc-38cc-11e3-b3c8-3c970e8b1cf7'}, - {'alive': False, - 'host': 'cluster2-machine2.internal', - 'id': '92b8b6bc-38ce-11e3-8537-3c970e8b1cf7'}, - {'alive': True, - 'host': 'cluster2-machine3.internal', - 'id': 'ebdcc950-51c8-11e3-a804-1c6f65b044df'}, - ] - }, - 'L3 Agent': { - 'agents': [ - {'alive': True, - 'host': 'cluster1-machine1.internal', - 'id': '7128198e-38ce-11e3-ba78-3c970e8b1cf7'}, - {'alive': True, - 'host': 'cluster2-machine1.internal', - 'id': '72453824-38ce-11e3-938e-3c970e8b1cf7'}, - {'alive': False, - 'host': 'cluster2-machine2.internal', - 'id': '84a04126-38ce-11e3-9449-3c970e8b1cf7'}, - {'alive': True, - 'host': 'cluster2-machine3.internal', - 'id': '00f4268a-51c9-11e3-9177-1c6f65b044df'}, - ] - } -} - -dhcp_agent_networks = { - 'networks': [ - {'id': 'foo'}, - {'id': 'bar'} - ] -} - -l3_agent_routers = { - 'routers': [ - {'id': 'baz'}, - {'id': 'bong'} - ] -} - cluster1 = ['cluster1-machine1.internal'] cluster2 = ['cluster2-machine1.internal', 'cluster2-machine2.internal' 'cluster2-machine3.internal'] @@ -882,86 +759,6 @@ class TestNeutronAgentReallocation(CharmTestCase): # Reset cached cache hookenv.cache = {} - def test_no_network_context(self): - self.NetworkServiceContext.return_value = \ - DummyNetworkServiceContext(return_value=None) - neutron_utils.reassign_agent_resources() - self.assertTrue(self.log.called) - - @patch('neutronclient.v2_0.client.Client') - def test_no_down_agents(self, _client): - self.NetworkServiceContext.return_value = \ - DummyNetworkServiceContext(return_value=network_context) - dummy_client = MagicMock() - dummy_client.list_agents.side_effect = agents_all_alive.itervalues() - _client.return_value = dummy_client - neutron_utils.reassign_agent_resources() - dummy_client.add_router_to_l3_agent.assert_not_called() - dummy_client.remove_router_from_l3_agent.assert_not_called() - dummy_client.add_network_to_dhcp_agent.assert_not_called() - dummy_client.remove_network_from_dhcp_agent.assert_not_called() - - @patch('neutronclient.v2_0.client.Client') - def test_agents_down_relocation_required(self, _client): - self.NetworkServiceContext.return_value = \ - DummyNetworkServiceContext(return_value=network_context) - dummy_client = MagicMock() - dummy_client.list_agents.side_effect = \ - agents_some_dead_cl2.itervalues() - dummy_client.list_networks_on_dhcp_agent.return_value = \ - dhcp_agent_networks - dummy_client.list_routers_on_l3_agent.return_value = \ - l3_agent_routers - _client.return_value = dummy_client - self.unit_private_ip.return_value = 'cluster2-machine1.internal' - self.relations_of_type.return_value = \ - [{'private-address': 'cluster2-machine3.internal'}] - neutron_utils.reassign_agent_resources() - - # Ensure routers removed from dead l3 agent - dummy_client.remove_router_from_l3_agent.assert_has_calls( - [call(l3_agent='84a04126-38ce-11e3-9449-3c970e8b1cf7', - router_id='bong'), - call(l3_agent='84a04126-38ce-11e3-9449-3c970e8b1cf7', - router_id='baz')], any_order=True) - # and re-assigned across the remaining two live agents - dummy_client.add_router_to_l3_agent.assert_has_calls( - [call(l3_agent='00f4268a-51c9-11e3-9177-1c6f65b044df', - body={'router_id': 'baz'}), - call(l3_agent='72453824-38ce-11e3-938e-3c970e8b1cf7', - body={'router_id': 'bong'})], any_order=True) - # Ensure networks removed from dead dhcp agent - dummy_client.remove_network_from_dhcp_agent.assert_has_calls( - [call(dhcp_agent='92b8b6bc-38ce-11e3-8537-3c970e8b1cf7', - network_id='foo'), - call(dhcp_agent='92b8b6bc-38ce-11e3-8537-3c970e8b1cf7', - network_id='bar')], any_order=True) - # and re-assigned across the remaining two live agents - dummy_client.add_network_to_dhcp_agent.assert_has_calls( - [call(dhcp_agent='53d6eefc-38cc-11e3-b3c8-3c970e8b1cf7', - body={'network_id': 'foo'}), - call(dhcp_agent='ebdcc950-51c8-11e3-a804-1c6f65b044df', - body={'network_id': 'bar'})], any_order=True) - - @patch('neutronclient.v2_0.client.Client') - def test_agents_down_relocation_impossible(self, _client): - self.NetworkServiceContext.return_value = \ - DummyNetworkServiceContext(return_value=network_context) - dummy_client = MagicMock() - dummy_client.list_agents.side_effect = \ - agents_some_dead_cl1.itervalues() - dummy_client.list_networks_on_dhcp_agent.return_value = \ - dhcp_agent_networks - dummy_client.list_routers_on_l3_agent.return_value = \ - l3_agent_routers - _client.return_value = dummy_client - self.unit_private_ip.return_value = 'cluster1-machine1.internal' - self.relations_of_type.return_value = [] - neutron_utils.reassign_agent_resources() - self.assertTrue(self.log.called) - assert not dummy_client.remove_router_from_l3_agent.called - assert not dummy_client.remove_network_from_dhcp_agent.called - @patch.object(neutron_utils, 'git_install_requested') @patch.object(neutron_utils, 'git_clone_and_install') @patch.object(neutron_utils, 'git_post_install') @@ -1002,62 +799,62 @@ class TestNeutronAgentReallocation(CharmTestCase): self.assertEqual(add_user_to_group.call_args_list, expected) expected = [ call('/etc/neutron', owner='neutron', - group='neutron', perms=0755, force=False), + group='neutron', perms=0o755, force=False), call('/etc/neutron/rootwrap.d', owner='neutron', - group='neutron', perms=0755, force=False), + group='neutron', perms=0o755, force=False), call('/etc/neutron/plugins', owner='neutron', - group='neutron', perms=0755, force=False), + group='neutron', perms=0o755, force=False), call('/etc/nova', owner='neutron', - group='neutron', perms=0755, force=False), + group='neutron', perms=0o755, force=False), call('/var/lib/neutron', owner='neutron', - group='neutron', perms=0755, force=False), + group='neutron', perms=0o755, force=False), call('/var/lib/neutron/lock', owner='neutron', - group='neutron', perms=0755, force=False), + group='neutron', perms=0o755, force=False), call('/var/log/neutron', owner='neutron', - group='neutron', perms=0755, force=False), + group='neutron', perms=0o755, force=False), call('/var/lib/nova', owner='neutron', - group='neutron', perms=0755, force=False), + group='neutron', perms=0o755, force=False), call('/var/log/nova', owner='neutron', - group='neutron', perms=0755, force=False), + group='neutron', perms=0o755, force=False), ] self.assertEqual(mkdir.call_args_list, expected) expected = [ call('/var/log/neutron/bigswitch-agent.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), call('/var/log/neutron/dhcp-agent.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), call('/var/log/neutron/l3-agent.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), call('/var/log/neutron/lbaas-agent.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), call('/var/log/neutron/ibm-agent.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), call('/var/log/neutron/linuxbridge-agent.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), call('/var/log/neutron/metadata-agent.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), call('/var/log/neutron/metering_agent.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), call('/var/log/neutron/mlnx-agent.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), call('/var/log/neutron/nec-agent.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), call('/var/log/neutron/nvsd-agent.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), call('/var/log/neutron/openflow-agent.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), call('/var/log/neutron/openvswitch-agent.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), call('/var/log/neutron/ovs-cleanup.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), call('/var/log/neutron/ryu-agent.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), call('/var/log/neutron/server.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), call('/var/log/neutron/sriov-agent.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), call('/var/log/neutron/vpn_agent.log', '', owner='neutron', - group='neutron', perms=0644), + group='neutron', perms=0o644), ] self.assertEqual(write_file.call_args_list, expected) diff --git a/unit_tests/test_utils.py b/unit_tests/test_utils.py index 2676c8cf..1b5cddea 100644 --- a/unit_tests/test_utils.py +++ b/unit_tests/test_utils.py @@ -36,7 +36,7 @@ def get_default_config(): ''' default_config = {} config = load_config() - for k, v in config.iteritems(): + for k, v in config.items(): if 'default' in v: default_config[k] = v['default'] else: