Update for Python 3 execution

Refactor codebase and unit tests to default to execution
under Python 3.

Drop install shim as Python 3 is always present >= trusty.

Drop legacy dhcp and network reassignment code from charm as
a) this relies on a py3 neutronclient (not supported on older
releases) and b) this function was superceeded by the ha-legacy-mode
and then neutron router and network HA built in functionality.

Use charmhelper provided get_host_ip as this superceeds the in
charm version of this function.

Change-Id: I0b28bf0851d44e85b1e856cbd97b71099faa76ae

.gitignore

@@ -13,3 +13,4 @@ wily
 xenial
 .unit-state.db
 func-results.json
+__pycache__

actions/actions.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 import os
 import sys

actions/git_reinstall.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 import sys
 import traceback
 

actions/openstack_upgrade.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 import sys
 
 sys.path.append('hooks/')

hooks/install

@@ -1,20 +0,0 @@
-#!/bin/bash
-# Wrapper to deal with newer Ubuntu versions that don't have py2 installed
-# by default.
-
-declare -a DEPS=('apt' 'netaddr' 'netifaces' 'pip' 'yaml')
-
-check_and_install() {
-    pkg="${1}-${2}"
-    if ! dpkg -s ${pkg} 2>&1 > /dev/null; then
-        apt-get -y install ${pkg}
-    fi
-}
-
-PYTHON="python"
-
-for dep in ${DEPS[@]}; do
-    check_and_install ${PYTHON} ${dep}
-done
-
-exec ./hooks/install.real

hooks/install

@@ -0,0 +1 @@
+neutron_hooks.py

hooks/install.real

@@ -1 +0,0 @@
-neutron_hooks.py

hooks/neutron_contexts.py

@@ -1,16 +1,11 @@
 # vim: set ts=4:et
 import os
 import uuid
-import socket
 from charmhelpers.core.hookenv import (
     config,
     unit_get,
-    cached,
     network_get_primary_address,
 )
-from charmhelpers.fetch import (
-    apt_install,
-)
 from charmhelpers.contrib.openstack.context import (
     OSContextGenerator,
     NeutronAPIContext,
@@ -21,6 +16,7 @@ from charmhelpers.contrib.hahelpers.cluster import(
 )
 from charmhelpers.contrib.network.ip import (
     get_address_in_network,
+    get_host_ip,
 )
 
 NEUTRON_ML2_PLUGIN = "ml2"
@@ -149,24 +145,6 @@ class NeutronGatewayContext(NeutronAPIContext):
         return ctxt
 
 
-@cached
-def get_host_ip(hostname=None):
-    try:
-        import dns.resolver
-    except ImportError:
-        apt_install('python-dnspython', fatal=True)
-        import dns.resolver
-    hostname = hostname or unit_get('private-address')
-    try:
-        # Test to see if already an IPv4 address
-        socket.inet_aton(hostname)
-        return hostname
-    except socket.error:
-        answers = dns.resolver.query(hostname, 'A')
-        if answers:
-            return answers[0].address
-
-
 SHARED_SECRET = "/etc/{}/secret.txt"
 
 

hooks/neutron_hooks.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 
 from base64 import b64decode
 
@@ -25,7 +25,6 @@ from charmhelpers.core.host import (
 )
 from charmhelpers.contrib.hahelpers.cluster import(
     get_hacluster_config,
-    eligible_leader
 )
 from charmhelpers.contrib.hahelpers.apache import(
     install_ca_cert
@@ -64,7 +63,6 @@ from neutron_utils import (
     remove_legacy_ha_files,
     install_legacy_ha_files,
     cleanup_ovs_netns,
-    reassign_agent_resources,
     stop_neutron_ha_monitor_daemon,
     use_l3ha,
     NEUTRON_COMMON,
@@ -77,7 +75,7 @@ hooks = Hooks()
 CONFIGS = register_configs()
 
 
-@hooks.hook('install.real')
+@hooks.hook('install')
 @harden()
 def install():
     status_set('maintenance', 'Executing pre-install')
@@ -260,9 +258,6 @@ def cluster_departed():
         log('Unable to re-assign agent resources for failed nodes with n1kv',
             level=WARNING)
         return
-    if not config('ha-legacy-mode') and eligible_leader(None):
-        reassign_agent_resources()
-        CONFIGS.write_all()
 
 
 @hooks.hook('cluster-relation-broken')

hooks/neutron_utils.py

@@ -23,8 +23,6 @@ from charmhelpers.core.hookenv import (
     INFO,
     ERROR,
     config,
-    relations_of_type,
-    unit_private_ip,
     is_relation_made,
     relation_ids,
 )
@@ -51,7 +49,6 @@ from charmhelpers.contrib.openstack.utils import (
     git_install_requested,
     git_pip_venv_dir,
     git_src_dir,
-    get_hostname,
     make_assess_status_func,
     os_release,
     pause_unit,
@@ -683,7 +680,7 @@ def stop_services():
     plugin = config('plugin')
     config_files = resolve_config_files(plugin, release)
     svcs = set()
-    for ctxt in config_files[config('plugin')].itervalues():
+    for ctxt in config_files[config('plugin')].values():
         for svc in ctxt['services']:
             svcs.add(remap_service(svc))
     for svc in svcs:
@@ -705,7 +702,7 @@ def restart_map(release=None):
     config_files = resolve_config_files(plugin, release)
     _map = {}
     enable_vpn_agent = 'neutron-vpn-agent' in get_packages()
-    for f, ctxt in config_files[plugin].iteritems():
+    for f, ctxt in config_files[plugin].items():
         svcs = set()
         for svc in ctxt['services']:
             svcs.add(remap_service(svc))
@@ -725,90 +722,6 @@ def restart_map(release=None):
 INT_BRIDGE = "br-int"
 EXT_BRIDGE = "br-ex"
 
-DHCP_AGENT = "DHCP Agent"
-L3_AGENT = "L3 Agent"
-
-
-# TODO: make work with neutron
-def reassign_agent_resources():
-    ''' Use agent scheduler API to detect down agents and re-schedule '''
-    env = NetworkServiceContext()()
-    if not env:
-        log('Unable to re-assign resources at this time')
-        return
-    try:
-        from quantumclient.v2_0 import client
-    except ImportError:
-        ''' Try to import neutronclient instead for havana+ '''
-        from neutronclient.v2_0 import client
-
-    auth_url = '%(auth_protocol)s://%(keystone_host)s:%(auth_port)s/v2.0' % env
-    quantum = client.Client(username=env['service_username'],
-                            password=env['service_password'],
-                            tenant_name=env['service_tenant'],
-                            auth_url=auth_url,
-                            region_name=env['region'])
-
-    partner_gateways = [unit_private_ip().split('.')[0]]
-    for partner_gateway in relations_of_type(reltype='cluster'):
-        gateway_hostname = get_hostname(partner_gateway['private-address'])
-        partner_gateways.append(gateway_hostname.partition('.')[0])
-
-    agents = quantum.list_agents(agent_type=DHCP_AGENT)
-    dhcp_agents = []
-    l3_agents = []
-    networks = {}
-    for agent in agents['agents']:
-        if not agent['alive']:
-            log('DHCP Agent %s down' % agent['id'])
-            for network in \
-                    quantum.list_networks_on_dhcp_agent(
-                        agent['id'])['networks']:
-                networks[network['id']] = agent['id']
-        else:
-            if agent['host'].partition('.')[0] in partner_gateways:
-                dhcp_agents.append(agent['id'])
-
-    agents = quantum.list_agents(agent_type=L3_AGENT)
-    routers = {}
-    for agent in agents['agents']:
-        if not agent['alive']:
-            log('L3 Agent %s down' % agent['id'])
-            for router in \
-                    quantum.list_routers_on_l3_agent(
-                        agent['id'])['routers']:
-                routers[router['id']] = agent['id']
-        else:
-            if agent['host'].split('.')[0] in partner_gateways:
-                l3_agents.append(agent['id'])
-
-    if len(dhcp_agents) == 0 or len(l3_agents) == 0:
-        log('Unable to relocate resources, there are %s dhcp_agents and %s \
-             l3_agents in this cluster' % (len(dhcp_agents), len(l3_agents)))
-        return
-
-    index = 0
-    for router_id in routers:
-        agent = index % len(l3_agents)
-        log('Moving router %s from %s to %s' %
-            (router_id, routers[router_id], l3_agents[agent]))
-        quantum.remove_router_from_l3_agent(l3_agent=routers[router_id],
-                                            router_id=router_id)
-        quantum.add_router_to_l3_agent(l3_agent=l3_agents[agent],
-                                       body={'router_id': router_id})
-        index += 1
-
-    index = 0
-    for network_id in networks:
-        agent = index % len(dhcp_agents)
-        log('Moving network %s from %s to %s' %
-            (network_id, networks[network_id], dhcp_agents[agent]))
-        quantum.remove_network_from_dhcp_agent(dhcp_agent=networks[network_id],
-                                               network_id=network_id)
-        quantum.add_network_to_dhcp_agent(dhcp_agent=dhcp_agents[agent],
-                                          body={'network_id': network_id})
-        index += 1
-
 
 def services():
     ''' Returns a list of services associate with this charm '''
@@ -862,12 +775,12 @@ def configure_ovs():
 
         portmaps = DataPortContext()()
         bridgemaps = parse_bridge_mappings(config('bridge-mappings'))
-        for provider, br in bridgemaps.iteritems():
+        for br in bridgemaps.values():
             add_bridge(br)
             if not portmaps:
                 continue
 
-            for port, _br in portmaps.iteritems():
+            for port, _br in portmaps.items():
                 if _br == br:
                     add_bridge_port(br, port, promisc=True)
 
@@ -908,13 +821,13 @@ def remove_file(path):
 
 
 def install_legacy_ha_files(force=False):
-    for f, p in LEGACY_FILES_MAP.iteritems():
+    for f, p in LEGACY_FILES_MAP.items():
         srcfile = os.path.join(LEGACY_HA_TEMPLATE_FILES, f)
         copy_file(srcfile, p['path'], p.get('permissions', None), force=force)
 
 
 def remove_legacy_ha_files():
-    for f, p in LEGACY_FILES_MAP.iteritems():
+    for f, p in LEGACY_FILES_MAP.items():
         remove_file(os.path.join(p['path'], f))
 
 

test-requirements.txt

@@ -5,12 +5,12 @@ coverage>=3.6
 mock>=1.2
 flake8>=2.2.4,<=2.4.1
 os-testr>=0.4.1
-charm-tools>=2.0.0
+charm-tools>=2.0.0;python_version=='2.7'
 requests==2.6.0
 # BEGIN: Amulet OpenStack Charm Helper Requirements
 # Liberty client lower constraints
-amulet>=1.14.3,<2.0
+amulet>=1.14.3,<2.0;python_version=='2.7'
-bundletester>=0.6.1,<1.0
+bundletester>=0.6.1,<1.0;python_version=='2.7'
 python-ceilometerclient>=1.5.0
 python-cinderclient>=1.4.0
 python-glanceclient>=1.1.0

tox.ini

@@ -2,7 +2,7 @@
 # This file is managed centrally by release-tools and should not be modified
 # within individual charm repos.
 [tox]
-envlist = pep8,py27
+envlist = pep8,py27,py35
 skipsdist = True
 
 [testenv]
@@ -20,6 +20,7 @@ passenv = HOME TERM AMULET_* CS_API_*
 basepython = python2.7
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
+commands = /bin/true
 
 [testenv:py35]
 basepython = python3.5

unit_tests/__init__.py

@@ -1,4 +1,5 @@
 import sys
 
+sys.path.append('unit_tests')
 sys.path.append('actions')
 sys.path.append('hooks')

unit_tests/test_neutron_contexts.py

@@ -1,18 +1,19 @@
+
+import io
+
+from contextlib import contextmanager
+
 from mock import (
-    Mock,
     MagicMock,
     patch
 )
 import neutron_contexts
-import sys
-from contextlib import contextmanager
 
 from test_utils import (
     CharmTestCase
 )
 
 TO_PATCH = [
-    'apt_install',
     'config',
     'eligible_leader',
     'unit_get',
@@ -27,14 +28,14 @@ def patch_open():
 
     Yields the mock for "open" and "file", respectively.'''
     mock_open = MagicMock(spec=open)
-    mock_file = MagicMock(spec=file)
+    mock_file = MagicMock(spec=io.FileIO)
 
     @contextmanager
     def stub_open(*args, **kwargs):
         mock_open(*args, **kwargs)
         yield mock_file
 
-    with patch('__builtin__.open', stub_open):
+    with patch('builtins.open', stub_open):
         yield mock_open, mock_file
 
 
@@ -323,54 +324,6 @@ class TestSharedSecret(CharmTestCase):
                 neutron_contexts.SHARED_SECRET.format('neutron'), 'r')
 
 
-class TestHostIP(CharmTestCase):
-
-    def setUp(self):
-        super(TestHostIP, self).setUp(neutron_contexts,
-                                      TO_PATCH)
-        self.config.side_effect = self.test_config.get
-        self.network_get_primary_address.side_effect = NotImplementedError
-        # Save and inject
-        self.mods = {'dns': None, 'dns.resolver': None}
-        for mod in self.mods:
-            if mod not in sys.modules:
-                sys.modules[mod] = Mock()
-            else:
-                del self.mods[mod]
-
-    def tearDown(self):
-        super(TestHostIP, self).tearDown()
-        # Cleanup
-        for mod in self.mods.keys():
-            del sys.modules[mod]
-
-    def test_get_host_ip_already_ip(self):
-        self.assertEqual(neutron_contexts.get_host_ip('10.5.0.1'),
-                         '10.5.0.1')
-
-    def test_get_host_ip_noarg(self):
-        self.unit_get.return_value = "10.5.0.1"
-        self.assertEqual(neutron_contexts.get_host_ip(),
-                         '10.5.0.1')
-
-    @patch('dns.resolver.query')
-    def test_get_host_ip_hostname_unresolvable(self, _query):
-        class NXDOMAIN(Exception):
-            pass
-        _query.side_effect = NXDOMAIN()
-        self.assertRaises(NXDOMAIN, neutron_contexts.get_host_ip,
-                          'missing.example.com')
-
-    @patch('dns.resolver.query')
-    def test_get_host_ip_hostname_resolvable(self, _query):
-        data = MagicMock()
-        data.address = '10.5.0.1'
-        _query.return_value = [data]
-        self.assertEqual(neutron_contexts.get_host_ip('myhost.example.com'),
-                         '10.5.0.1')
-        _query.assert_called_with('myhost.example.com', 'A')
-
-
 class TestMisc(CharmTestCase):
 
     def setUp(self):

unit_tests/test_neutron_hooks.py

@@ -83,7 +83,7 @@ class TestQuantumHooks(CharmTestCase):
         self.valid_plugin.return_value = True
         _pkgs = ['foo', 'bar']
         self.filter_installed_packages.return_value = _pkgs
-        self._call_hook('install.real')
+        self._call_hook('install')
         self.configure_installation_source.assert_called_with(
             'cloud:precise-havana'
         )
@@ -99,7 +99,7 @@ class TestQuantumHooks(CharmTestCase):
 
     def test_install_hook_precise_nocloudarchive(self):
         self.test_config.set('openstack-origin', 'distro')
-        self._call_hook('install.real')
+        self._call_hook('install')
         self.configure_installation_source.assert_called_with(
             'cloud:precise-icehouse'
         )
@@ -107,7 +107,7 @@ class TestQuantumHooks(CharmTestCase):
     @patch('sys.exit')
     def test_install_hook_invalid_plugin(self, _exit):
         self.valid_plugin.return_value = False
-        self._call_hook('install.real')
+        self._call_hook('install')
         self.assertTrue(self.log.called)
         _exit.assert_called_with(1)
 
@@ -132,7 +132,7 @@ class TestQuantumHooks(CharmTestCase):
         projects_yaml = yaml.dump(openstack_origin_git)
         self.test_config.set('openstack-origin', repo)
         self.test_config.set('openstack-origin-git', projects_yaml)
-        self._call_hook('install.real')
+        self._call_hook('install')
         self.configure_installation_source.assert_called_with(
             'cloud:trusty-juno'
         )

unit_tests/test_neutron_utils.py

@@ -1,5 +1,3 @@
-import collections
-
 from mock import MagicMock, call, patch, ANY
 
 import charmhelpers.core.hookenv as hookenv
@@ -31,8 +29,6 @@ TO_PATCH = [
     'service_running',
     'NetworkServiceContext',
     'ExternalPortContext',
-    'unit_private_ip',
-    'relations_of_type',
     'render',
     'service_stop',
     'determine_dkms_package',
@@ -58,20 +54,11 @@ openstack_origin_git = \
 
 class TestNeutronUtils(CharmTestCase):
 
-    def assertDictEqual(self, d1, d2, msg=None):  # assertEqual uses for dicts
-        for k, v1 in d1.iteritems():
-            self.assertIn(k, d2, msg)
-            v2 = d2[k]
-            if(isinstance(v1, collections.Iterable) and
-               not isinstance(v1, basestring)):
-                self.assertItemsEqual(v1, v2, msg)
-            else:
-                self.assertEqual(v1, v2, msg)
-
     def setUp(self):
         super(TestNeutronUtils, self).setUp(neutron_utils, TO_PATCH)
         self.headers_package.return_value = 'linux-headers-2.6.18'
         self._set_distrib_codename('trusty')
+        self.maxDiff = None
 
     def tearDown(self):
         # Reset cached cache
@@ -286,7 +273,7 @@ class TestNeutronUtils(CharmTestCase):
         ])
         calls = [call('br1', 'eth0.100', promisc=True),
                  call('br1', 'eth0.200', promisc=True)]
-        self.add_bridge_port.assert_has_calls(calls)
+        self.add_bridge_port.assert_has_calls(calls, any_order=True)
 
     @patch.object(neutron_utils, 'register_configs')
     @patch('charmhelpers.contrib.openstack.templating.OSConfigRenderer')
@@ -373,12 +360,12 @@ class TestNeutronUtils(CharmTestCase):
         mock_get_packages.return_value = ['neutron-vpn-agent']
         self.os_release.return_value = 'icehouse'
         ex_map = {
-            neutron_utils.NEUTRON_CONF: ['neutron-dhcp-agent',
+            neutron_utils.NEUTRON_CONF: ['neutron-lbaas-agent',
-                                         'neutron-metadata-agent',
                                          'neutron-plugin-openvswitch-agent',
+                                         'neutron-vpn-agent',
+                                         'neutron-dhcp-agent',
                                          'neutron-metering-agent',
-                                         'neutron-lbaas-agent',
+                                         'neutron-metadata-agent'],
-                                         'neutron-vpn-agent'],
             neutron_utils.NEUTRON_DNSMASQ_CONF: ['neutron-dhcp-agent'],
             neutron_utils.NEUTRON_LBAAS_AGENT_CONF:
             ['neutron-lbaas-agent'],
@@ -409,8 +396,7 @@ class TestNeutronUtils(CharmTestCase):
             neutron_utils.NOVA_API_METADATA_AA_PROFILE_PATH:
             ['nova-api-metadata'],
         }
-
+        self.assertEqual(neutron_utils.restart_map(), ex_map)
-        self.assertDictEqual(neutron_utils.restart_map(), ex_map)
 
     @patch.object(neutron_utils, 'get_packages')
     def test_restart_map_ovs_mitaka(self, mock_get_packages):
@@ -418,12 +404,12 @@ class TestNeutronUtils(CharmTestCase):
         mock_get_packages.return_value = ['neutron-vpn-agent']
         self.os_release.return_value = 'mitaka'
         ex_map = {
-            neutron_utils.NEUTRON_CONF: ['neutron-dhcp-agent',
+            neutron_utils.NEUTRON_CONF: ['neutron-lbaas-agent',
-                                         'neutron-metadata-agent',
+                                         'neutron-vpn-agent',
-                                         'neutron-openvswitch-agent',
+                                         'neutron-dhcp-agent',
                                          'neutron-metering-agent',
-                                         'neutron-lbaas-agent',
+                                         'neutron-metadata-agent',
-                                         'neutron-vpn-agent'],
+                                         'neutron-openvswitch-agent'],
             neutron_utils.NEUTRON_DNSMASQ_CONF: ['neutron-dhcp-agent'],
             neutron_utils.NEUTRON_LBAAS_AGENT_CONF:
             ['neutron-lbaas-agent'],
@@ -453,7 +439,7 @@ class TestNeutronUtils(CharmTestCase):
             neutron_utils.NOVA_API_METADATA_AA_PROFILE_PATH:
             ['nova-api-metadata'],
         }
-        self.assertEqual(ex_map, neutron_utils.restart_map())
+        self.assertEqual(neutron_utils.restart_map(), ex_map)
 
     @patch.object(neutron_utils, 'get_packages')
     def test_restart_map_ovs_newton(self, mock_get_packages):
@@ -461,12 +447,12 @@ class TestNeutronUtils(CharmTestCase):
         mock_get_packages.return_value = ['neutron-vpn-agent']
         self.os_release.return_value = 'newton'
         ex_map = {
-            neutron_utils.NEUTRON_CONF: ['neutron-dhcp-agent',
+            neutron_utils.NEUTRON_CONF: ['neutron-vpn-agent',
-                                         'neutron-metadata-agent',
-                                         'neutron-openvswitch-agent',
-                                         'neutron-metering-agent',
                                          'neutron-lbaasv2-agent',
-                                         'neutron-vpn-agent'],
+                                         'neutron-dhcp-agent',
+                                         'neutron-metering-agent',
+                                         'neutron-metadata-agent',
+                                         'neutron-openvswitch-agent'],
             neutron_utils.NEUTRON_DNSMASQ_CONF: ['neutron-dhcp-agent'],
             neutron_utils.NEUTRON_LBAAS_AGENT_CONF:
             ['neutron-lbaasv2-agent'],
@@ -483,8 +469,20 @@ class TestNeutronUtils(CharmTestCase):
             neutron_utils.NOVA_CONF: ['nova-api-metadata'],
             neutron_utils.EXT_PORT_CONF: ['ext-port'],
             neutron_utils.PHY_NIC_MTU_CONF: ['os-charm-phy-nic-mtu'],
+            neutron_utils.NEUTRON_DHCP_AA_PROFILE_PATH: ['neutron-dhcp-agent'],
+            neutron_utils.NEUTRON_OVS_AA_PROFILE_PATH:
+                ['neutron-openvswitch-agent'],
+            neutron_utils.NEUTRON_L3_AA_PROFILE_PATH: ['neutron-vpn-agent'],
+            neutron_utils.NEUTRON_LBAASV2_AA_PROFILE_PATH:
+            ['neutron-lbaasv2-agent'],
+            neutron_utils.NEUTRON_METADATA_AA_PROFILE_PATH:
+            ['neutron-metadata-agent'],
+            neutron_utils.NEUTRON_METERING_AA_PROFILE_PATH:
+            ['neutron-metering-agent'],
+            neutron_utils.NOVA_API_METADATA_AA_PROFILE_PATH:
+            ['nova-api-metadata'],
         }
-        self.assertEqual(ex_map, neutron_utils.restart_map())
+        self.assertEqual(neutron_utils.restart_map(), ex_map)
 
     @patch.object(neutron_utils, 'get_packages')
     def test_restart_map_ovs_post_trusty(self, mock_get_packages):
@@ -493,7 +491,7 @@ class TestNeutronUtils(CharmTestCase):
         mock_get_packages.return_value = ['neutron-l3-agent']
         self.os_release.return_value = 'diablo'
         rmap = neutron_utils.restart_map()
-        for services in rmap.itervalues():
+        for services in rmap.values():
             self.assertFalse('neutron-vpn-agent' in services)
 
     @patch.object(neutron_utils, 'get_packages')
@@ -502,11 +500,11 @@ class TestNeutronUtils(CharmTestCase):
         mock_get_packages.return_value = ['neutron-vpn-agent']
         self.os_release.return_value = 'icehouse'
         ex_map = {
-            neutron_utils.NEUTRON_CONF: ['neutron-dhcp-agent',
+            neutron_utils.NEUTRON_CONF: ['neutron-lbaas-agent',
-                                         'neutron-metadata-agent',
+                                         'neutron-vpn-agent',
+                                         'neutron-dhcp-agent',
                                          'neutron-metering-agent',
-                                         'neutron-lbaas-agent',
+                                         'neutron-metadata-agent'],
-                                         'neutron-vpn-agent'],
             neutron_utils.NEUTRON_DNSMASQ_CONF: ['neutron-dhcp-agent'],
             neutron_utils.NEUTRON_LBAAS_AGENT_CONF:
             ['neutron-lbaas-agent'],
@@ -522,7 +520,6 @@ class TestNeutronUtils(CharmTestCase):
             neutron_utils.EXT_PORT_CONF: ['ext-port'],
             neutron_utils.PHY_NIC_MTU_CONF: ['os-charm-phy-nic-mtu'],
             neutron_utils.NEUTRON_DHCP_AA_PROFILE_PATH: ['neutron-dhcp-agent'],
-            neutron_utils.NEUTRON_L3_AA_PROFILE_PATH: ['neutron-vpn-agent'],
             neutron_utils.NEUTRON_LBAAS_AA_PROFILE_PATH:
             ['neutron-lbaas-agent'],
             neutron_utils.NEUTRON_METADATA_AA_PROFILE_PATH:
@@ -533,7 +530,7 @@ class TestNeutronUtils(CharmTestCase):
             ['nova-api-metadata'],
         }
 
-        self.assertDictEqual(neutron_utils.restart_map(), ex_map)
+        self.assertEqual(neutron_utils.restart_map(), ex_map)
 
     @patch.object(neutron_utils, 'get_packages')
     def test_restart_map_ovs_odl_newton(self, mock_get_packages):
@@ -541,11 +538,11 @@ class TestNeutronUtils(CharmTestCase):
         mock_get_packages.return_value = ['neutron-vpn-agent']
         self.os_release.return_value = 'newton'
         ex_map = {
-            neutron_utils.NEUTRON_CONF: ['neutron-dhcp-agent',
+            neutron_utils.NEUTRON_CONF: ['neutron-vpn-agent',
-                                         'neutron-metadata-agent',
-                                         'neutron-metering-agent',
                                          'neutron-lbaasv2-agent',
-                                         'neutron-vpn-agent'],
+                                         'neutron-dhcp-agent',
+                                         'neutron-metering-agent',
+                                         'neutron-metadata-agent'],
             neutron_utils.NEUTRON_DNSMASQ_CONF: ['neutron-dhcp-agent'],
             neutron_utils.NEUTRON_LBAAS_AGENT_CONF:
             ['neutron-lbaasv2-agent'],
@@ -561,7 +558,6 @@ class TestNeutronUtils(CharmTestCase):
             neutron_utils.EXT_PORT_CONF: ['ext-port'],
             neutron_utils.PHY_NIC_MTU_CONF: ['os-charm-phy-nic-mtu'],
             neutron_utils.NEUTRON_DHCP_AA_PROFILE_PATH: ['neutron-dhcp-agent'],
-            neutron_utils.NEUTRON_L3_AA_PROFILE_PATH: ['neutron-vpn-agent'],
             neutron_utils.NEUTRON_LBAASV2_AA_PROFILE_PATH:
             ['neutron-lbaasv2-agent'],
             neutron_utils.NEUTRON_METADATA_AA_PROFILE_PATH:
@@ -746,125 +742,6 @@ class DummyExternalPortContext():
     def __call__(self):
         return self.return_value
 
-agents_all_alive = {
-    'DHCP Agent': {
-        'agents': [
-            {'alive': True,
-             'host': 'cluster1-machine1.internal',
-             'id': '3e3550f2-38cc-11e3-9617-3c970e8b1cf7'},
-            {'alive': True,
-             'host': 'cluster1-machine2.internal',
-             'id': '53d6eefc-38cc-11e3-b3c8-3c970e8b1cf7'},
-            {'alive': True,
-             'host': 'cluster2-machine1.internal',
-             'id': '92b8b6bc-38ce-11e3-8537-3c970e8b1cf7'},
-            {'alive': True,
-             'host': 'cluster2-machine3.internal',
-             'id': 'ebdcc950-51c8-11e3-a804-1c6f65b044df'},
-        ]
-    },
-    'L3 Agent': {
-        'agents': [
-            {'alive': True,
-             'host': 'cluster1-machine1.internal',
-             'id': '7128198e-38ce-11e3-ba78-3c970e8b1cf7'},
-            {'alive': True,
-             'host': 'cluster1-machine2.internal',
-             'id': '72453824-38ce-11e3-938e-3c970e8b1cf7'},
-            {'alive': True,
-             'host': 'cluster2-machine1.internal',
-             'id': '84a04126-38ce-11e3-9449-3c970e8b1cf7'},
-            {'alive': True,
-             'host': 'cluster2-machine3.internal',
-             'id': '00f4268a-51c9-11e3-9177-1c6f65b044df'},
-        ]
-    }
-}
-
-agents_some_dead_cl1 = {
-    'DHCP Agent': {
-        'agents': [
-            {'alive': False,
-             'host': 'cluster1-machine1.internal',
-             'id': '3e3550f2-38cc-11e3-9617-3c970e8b1cf7'},
-            {'alive': True,
-             'host': 'cluster2-machine1.internal',
-             'id': '53d6eefc-38cc-11e3-b3c8-3c970e8b1cf7'},
-            {'alive': True,
-             'host': 'cluster2-machine2.internal',
-             'id': '92b8b6bc-38ce-11e3-8537-3c970e8b1cf7'},
-            {'alive': True,
-             'host': 'cluster2-machine3.internal',
-             'id': 'ebdcc950-51c8-11e3-a804-1c6f65b044df'},
-        ]
-    },
-    'L3 Agent': {
-        'agents': [
-            {'alive': False,
-             'host': 'cluster1-machine1.internal',
-             'id': '7128198e-38ce-11e3-ba78-3c970e8b1cf7'},
-            {'alive': True,
-             'host': 'cluster2-machine1.internal',
-             'id': '72453824-38ce-11e3-938e-3c970e8b1cf7'},
-            {'alive': True,
-             'host': 'cluster2-machine2.internal',
-             'id': '84a04126-38ce-11e3-9449-3c970e8b1cf7'},
-            {'alive': True,
-             'host': 'cluster2-machine3.internal',
-             'id': '00f4268a-51c9-11e3-9177-1c6f65b044df'},
-        ]
-    }
-}
-
-agents_some_dead_cl2 = {
-    'DHCP Agent': {
-        'agents': [
-            {'alive': True,
-             'host': 'cluster1-machine1.internal',
-             'id': '3e3550f2-38cc-11e3-9617-3c970e8b1cf7'},
-            {'alive': True,
-             'host': 'cluster2-machine1.internal',
-             'id': '53d6eefc-38cc-11e3-b3c8-3c970e8b1cf7'},
-            {'alive': False,
-             'host': 'cluster2-machine2.internal',
-             'id': '92b8b6bc-38ce-11e3-8537-3c970e8b1cf7'},
-            {'alive': True,
-             'host': 'cluster2-machine3.internal',
-             'id': 'ebdcc950-51c8-11e3-a804-1c6f65b044df'},
-        ]
-    },
-    'L3 Agent': {
-        'agents': [
-            {'alive': True,
-             'host': 'cluster1-machine1.internal',
-             'id': '7128198e-38ce-11e3-ba78-3c970e8b1cf7'},
-            {'alive': True,
-             'host': 'cluster2-machine1.internal',
-             'id': '72453824-38ce-11e3-938e-3c970e8b1cf7'},
-            {'alive': False,
-             'host': 'cluster2-machine2.internal',
-             'id': '84a04126-38ce-11e3-9449-3c970e8b1cf7'},
-            {'alive': True,
-             'host': 'cluster2-machine3.internal',
-             'id': '00f4268a-51c9-11e3-9177-1c6f65b044df'},
-        ]
-    }
-}
-
-dhcp_agent_networks = {
-    'networks': [
-        {'id': 'foo'},
-        {'id': 'bar'}
-    ]
-}
-
-l3_agent_routers = {
-    'routers': [
-        {'id': 'baz'},
-        {'id': 'bong'}
-    ]
-}
-
 cluster1 = ['cluster1-machine1.internal']
 cluster2 = ['cluster2-machine1.internal', 'cluster2-machine2.internal'
             'cluster2-machine3.internal']
@@ -882,86 +759,6 @@ class TestNeutronAgentReallocation(CharmTestCase):
         # Reset cached cache
         hookenv.cache = {}
 
-    def test_no_network_context(self):
-        self.NetworkServiceContext.return_value = \
-            DummyNetworkServiceContext(return_value=None)
-        neutron_utils.reassign_agent_resources()
-        self.assertTrue(self.log.called)
-
-    @patch('neutronclient.v2_0.client.Client')
-    def test_no_down_agents(self, _client):
-        self.NetworkServiceContext.return_value = \
-            DummyNetworkServiceContext(return_value=network_context)
-        dummy_client = MagicMock()
-        dummy_client.list_agents.side_effect = agents_all_alive.itervalues()
-        _client.return_value = dummy_client
-        neutron_utils.reassign_agent_resources()
-        dummy_client.add_router_to_l3_agent.assert_not_called()
-        dummy_client.remove_router_from_l3_agent.assert_not_called()
-        dummy_client.add_network_to_dhcp_agent.assert_not_called()
-        dummy_client.remove_network_from_dhcp_agent.assert_not_called()
-
-    @patch('neutronclient.v2_0.client.Client')
-    def test_agents_down_relocation_required(self, _client):
-        self.NetworkServiceContext.return_value = \
-            DummyNetworkServiceContext(return_value=network_context)
-        dummy_client = MagicMock()
-        dummy_client.list_agents.side_effect = \
-            agents_some_dead_cl2.itervalues()
-        dummy_client.list_networks_on_dhcp_agent.return_value = \
-            dhcp_agent_networks
-        dummy_client.list_routers_on_l3_agent.return_value = \
-            l3_agent_routers
-        _client.return_value = dummy_client
-        self.unit_private_ip.return_value = 'cluster2-machine1.internal'
-        self.relations_of_type.return_value = \
-            [{'private-address': 'cluster2-machine3.internal'}]
-        neutron_utils.reassign_agent_resources()
-
-        # Ensure routers removed from dead l3 agent
-        dummy_client.remove_router_from_l3_agent.assert_has_calls(
-            [call(l3_agent='84a04126-38ce-11e3-9449-3c970e8b1cf7',
-                  router_id='bong'),
-             call(l3_agent='84a04126-38ce-11e3-9449-3c970e8b1cf7',
-                  router_id='baz')], any_order=True)
-        # and re-assigned across the remaining two live agents
-        dummy_client.add_router_to_l3_agent.assert_has_calls(
-            [call(l3_agent='00f4268a-51c9-11e3-9177-1c6f65b044df',
-                  body={'router_id': 'baz'}),
-             call(l3_agent='72453824-38ce-11e3-938e-3c970e8b1cf7',
-                  body={'router_id': 'bong'})], any_order=True)
-        # Ensure networks removed from dead dhcp agent
-        dummy_client.remove_network_from_dhcp_agent.assert_has_calls(
-            [call(dhcp_agent='92b8b6bc-38ce-11e3-8537-3c970e8b1cf7',
-                  network_id='foo'),
-             call(dhcp_agent='92b8b6bc-38ce-11e3-8537-3c970e8b1cf7',
-                  network_id='bar')], any_order=True)
-        # and re-assigned across the remaining two live agents
-        dummy_client.add_network_to_dhcp_agent.assert_has_calls(
-            [call(dhcp_agent='53d6eefc-38cc-11e3-b3c8-3c970e8b1cf7',
-                  body={'network_id': 'foo'}),
-             call(dhcp_agent='ebdcc950-51c8-11e3-a804-1c6f65b044df',
-                  body={'network_id': 'bar'})], any_order=True)
-
-    @patch('neutronclient.v2_0.client.Client')
-    def test_agents_down_relocation_impossible(self, _client):
-        self.NetworkServiceContext.return_value = \
-            DummyNetworkServiceContext(return_value=network_context)
-        dummy_client = MagicMock()
-        dummy_client.list_agents.side_effect = \
-            agents_some_dead_cl1.itervalues()
-        dummy_client.list_networks_on_dhcp_agent.return_value = \
-            dhcp_agent_networks
-        dummy_client.list_routers_on_l3_agent.return_value = \
-            l3_agent_routers
-        _client.return_value = dummy_client
-        self.unit_private_ip.return_value = 'cluster1-machine1.internal'
-        self.relations_of_type.return_value = []
-        neutron_utils.reassign_agent_resources()
-        self.assertTrue(self.log.called)
-        assert not dummy_client.remove_router_from_l3_agent.called
-        assert not dummy_client.remove_network_from_dhcp_agent.called
-
     @patch.object(neutron_utils, 'git_install_requested')
     @patch.object(neutron_utils, 'git_clone_and_install')
     @patch.object(neutron_utils, 'git_post_install')
@@ -1002,62 +799,62 @@ class TestNeutronAgentReallocation(CharmTestCase):
         self.assertEqual(add_user_to_group.call_args_list, expected)
         expected = [
             call('/etc/neutron', owner='neutron',
-                 group='neutron', perms=0755, force=False),
+                 group='neutron', perms=0o755, force=False),
             call('/etc/neutron/rootwrap.d', owner='neutron',
-                 group='neutron', perms=0755, force=False),
+                 group='neutron', perms=0o755, force=False),
             call('/etc/neutron/plugins', owner='neutron',
-                 group='neutron', perms=0755, force=False),
+                 group='neutron', perms=0o755, force=False),
             call('/etc/nova', owner='neutron',
-                 group='neutron', perms=0755, force=False),
+                 group='neutron', perms=0o755, force=False),
             call('/var/lib/neutron', owner='neutron',
-                 group='neutron', perms=0755, force=False),
+                 group='neutron', perms=0o755, force=False),
             call('/var/lib/neutron/lock', owner='neutron',
-                 group='neutron', perms=0755, force=False),
+                 group='neutron', perms=0o755, force=False),
             call('/var/log/neutron', owner='neutron',
-                 group='neutron', perms=0755, force=False),
+                 group='neutron', perms=0o755, force=False),
             call('/var/lib/nova', owner='neutron',
-                 group='neutron', perms=0755, force=False),
+                 group='neutron', perms=0o755, force=False),
             call('/var/log/nova', owner='neutron',
-                 group='neutron', perms=0755, force=False),
+                 group='neutron', perms=0o755, force=False),
         ]
         self.assertEqual(mkdir.call_args_list, expected)
         expected = [
             call('/var/log/neutron/bigswitch-agent.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
             call('/var/log/neutron/dhcp-agent.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
             call('/var/log/neutron/l3-agent.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
             call('/var/log/neutron/lbaas-agent.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
             call('/var/log/neutron/ibm-agent.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
             call('/var/log/neutron/linuxbridge-agent.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
             call('/var/log/neutron/metadata-agent.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
             call('/var/log/neutron/metering_agent.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
             call('/var/log/neutron/mlnx-agent.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
             call('/var/log/neutron/nec-agent.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
             call('/var/log/neutron/nvsd-agent.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
             call('/var/log/neutron/openflow-agent.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
             call('/var/log/neutron/openvswitch-agent.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
             call('/var/log/neutron/ovs-cleanup.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
             call('/var/log/neutron/ryu-agent.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
             call('/var/log/neutron/server.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
             call('/var/log/neutron/sriov-agent.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
             call('/var/log/neutron/vpn_agent.log', '', owner='neutron',
-                 group='neutron', perms=0644),
+                 group='neutron', perms=0o644),
         ]
         self.assertEqual(write_file.call_args_list, expected)
 

unit_tests/test_utils.py

@@ -36,7 +36,7 @@ def get_default_config():
     '''
     default_config = {}
     config = load_config()
-    for k, v in config.iteritems():
+    for k, v in config.items():
         if 'default' in v:
             default_config[k] = v['default']
         else: