openstack/charm-neutron-api
Code Issues Proposed changes

Update charm to use Python 3

Browse Source 
Updates across the charm and unit tests to switch to
execution under Python 3.

Note that the changes are not backwards compatible
with Python 2.

Refactor use of neutronclient python module to simply
wrap the neutron binary, using the yaml output format
to avoid the requirement for a Python 3 module on
older OpenStack release versions.

Change-Id: Ic26b0dd19a76552481939325963a6c21585dee3c
This commit is contained in:
James Page 2017-11-05 09:04:50 +11:00
parent c6513b0dc1
commit 5f22e621c2
16 changed files with 99 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -9,3 +9,4 @@ tags
trusty/
xenial/
.stestr
__pycache__

2
actions/actions.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python3
#
# Copyright 2016 Canonical Ltd
#

2
actions/git_reinstall.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python3
#
# Copyright 2016 Canonical Ltd
#

2
actions/openstack_upgrade.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python3
#
# Copyright 2016 Canonical Ltd
#

20
hooks/install
View File

@ -1,20 +0,0 @@
#!/bin/bash
# Wrapper to deal with newer Ubuntu versions that don't have py2 installed
# by default.
declare -a DEPS=('apt' 'netaddr' 'netifaces' 'pip' 'yaml')
check_and_install() {
pkg="${1}-${2}"
if ! dpkg -s ${pkg} 2>&1 > /dev/null; then
apt-get -y install ${pkg}
fi
}
PYTHON="python"
for dep in ${DEPS[@]}; do
check_and_install ${PYTHON} ${dep}
done
exec ./hooks/install.real

1
hooks/install Symbolic link
View File

@ -0,0 +1 @@
neutron_api_hooks.py

1
hooks/install.real
View File

@ -1 +0,0 @@
neutron_api_hooks.py

2
hooks/neutron_api_context.py
View File

@ -349,7 +349,7 @@ class NeutronCCContext(context.NeutronContext):
cmp_release = CompareOpenStackReleases(release)
if config('neutron-plugin') in ['vsp']:
_config = config()
for k, v in _config.iteritems():
for k, v in _config.items():
if k.startswith('vsd'):
ctxt[k.replace('-', '_')] = v
for rid in relation_ids('vsd-rest-api'):

5
hooks/neutron_api_hooks.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python3
#
# Copyright 2016 Canonical Ltd
#
@ -186,8 +186,7 @@ def configure_https():
identity_joined(rid=rid)
@hooks.hook('install.real')
@hooks.hook()
@hooks.hook('install')
@harden()
def install():
status_set('maintenance', 'Executing pre-install')

50
hooks/neutron_api_utils.py
View File

@ -20,6 +20,7 @@ import shutil
import subprocess
import uuid
import glob
import yaml
from base64 import b64encode
from charmhelpers.contrib.openstack import context, templating
from charmhelpers.contrib.openstack.neutron import (
@ -511,14 +512,14 @@ def register_configs(release=None):
release = release or os_release('neutron-common')
configs = templating.OSConfigRenderer(templates_dir=TEMPLATES,
openstack_release=release)
for cfg, rscs in resource_map().iteritems():
for cfg, rscs in resource_map().items():
configs.register(cfg, rscs['contexts'])
return configs
def restart_map():
return OrderedDict([(cfg, v['services'])
for cfg, v in resource_map().iteritems()
for cfg, v in resource_map().items()
if v['services']])
@ -686,22 +687,43 @@ def setup_ipv6():
apt_install('haproxy/trusty-backports', fatal=True)
class FakeNeutronClient(object):
'''Fake wrapper for Neutron Client'''
def __init__(self, username, password, tenant_name,
auth_url, region_name):
self.env = {
'OS_USERNAME': username,
'OS_PASSWORD': password,
'OS_TENANT_NAME': tenant_name,
'OS_AUTH_URL': auth_url,
'OS_REGION': region_name,
}
def list_routers(self):
cmd = ['neutron', 'router-list', '-f', 'yaml']
try:
routers = subprocess.check_output(
cmd, env=self.env).decode('UTF-8')
return {'routers': yaml.load(routers)}
except subprocess.CalledProcessError:
return {'routers': []}
def get_neutron_client():
''' Return a neutron client if possible '''
env = neutron_api_context.IdentityServiceContext()()
if not env:
log('Unable to check resources at this time')
return
return None
auth_url = '%(auth_protocol)s://%(auth_host)s:%(auth_port)s/v2.0' % env
# Late import to avoid install hook failures when pkg hasnt been installed
from neutronclient.v2_0 import client
neutron_client = client.Client(username=env['admin_user'],
password=env['admin_password'],
tenant_name=env['admin_tenant_name'],
auth_url=auth_url,
region_name=env['region'])
return neutron_client
auth_url = '{auth_protocol}://{auth_host}:{auth_port}/v2.0'.format(**env)
return FakeNeutronClient(username=env['admin_user'],
password=env['admin_password'],
tenant_name=env['admin_tenant_name'],
auth_url=auth_url,
region_name=env['region'])
def router_feature_present(feature):
@ -758,10 +780,10 @@ def git_pre_install():
add_user_to_group('neutron', 'neutron')
for d in dirs:
mkdir(d, owner='neutron', group='neutron', perms=0755, force=False)
mkdir(d, owner='neutron', group='neutron', perms=0o755, force=False)
for l in logs:
write_file(l, '', owner='neutron', group='neutron', perms=0600)
write_file(l, '', owner='neutron', group='neutron', perms=0o600)
def git_post_install(projects_yaml):

6
test-requirements.txt
View File

@ -5,12 +5,12 @@ coverage>=3.6
mock>=1.2
flake8>=2.2.4,<=2.4.1
os-testr>=0.4.1
charm-tools>=2.0.0
charm-tools>=2.0.0;python_version=='2.7'
requests==2.6.0
# BEGIN: Amulet OpenStack Charm Helper Requirements
# Liberty client lower constraints
amulet>=1.14.3,<2.0
bundletester>=0.6.1,<1.0
amulet>=1.14.3,<2.0;python_version=='2.7'
bundletester>=0.6.1,<1.0;python_version=='2.7'
python-ceilometerclient>=1.5.0
python-cinderclient>=1.4.0
python-glanceclient>=1.1.0

3
tox.ini
View File

@ -2,7 +2,7 @@
# This file is managed centrally by release-tools and should not be modified
# within individual charm repos.
[tox]
envlist = pep8,py27
envlist = pep8,py27,py35
skipsdist = True
[testenv]
@ -20,6 +20,7 @@ passenv = HOME TERM AMULET_* CS_API_*
basepython = python2.7
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
commands = /bin/true
[testenv:py35]
basepython = python3.5

1
unit_tests/__init__.py
View File

@ -16,3 +16,4 @@ import sys
sys.path.append('actions/')
sys.path.append('hooks/')
sys.path.append('unit_tests')

48
unit_tests/test_neutron_api_context.py
View File

@ -63,10 +63,8 @@ class GeneralTests(CharmTestCase):
def test_get_tenant_network_types_unsupported(self):
self.test_config.set('overlay-network-type', 'tokenring')
with self.assertRaises(ValueError) as _exceptctxt:
with self.assertRaises(ValueError):
context._get_tenant_network_types()
self.assertEqual(_exceptctxt.exception.message,
'Unsupported overlay-network-type tokenring')
def test_get_tenant_network_types_default(self):
self.test_config.set('overlay-network-type', 'gre vxlan')
@ -92,20 +90,14 @@ class GeneralTests(CharmTestCase):
def test_get_tenant_network_types_unsupported_default(self):
self.test_config.set('overlay-network-type', '')
self.test_config.set('default-tenant-network-type', 'whizzy')
with self.assertRaises(ValueError) as _exceptctxt:
with self.assertRaises(ValueError):
context._get_tenant_network_types()
self.assertEqual(_exceptctxt.exception.message,
'Unsupported or unconfigured '
'default-tenant-network-type whizzy')
def test_get_tenant_network_types_unconfigured_default(self):
self.test_config.set('overlay-network-type', 'gre')
self.test_config.set('default-tenant-network-type', 'vxlan')
with self.assertRaises(ValueError) as _exceptctxt:
with self.assertRaises(ValueError):
context._get_tenant_network_types()
self.assertEqual(_exceptctxt.exception.message,
'Unsupported or unconfigured '
'default-tenant-network-type vxlan')
def test_get_l3ha(self):
self.test_config.set('enable-l3ha', True)
@ -317,7 +309,7 @@ class HAProxyContextTest(CharmTestCase):
def test_context_No_peers(self, _log, _rids, _mkdir):
_rids.return_value = []
hap_ctxt = context.HAProxyContext()
with patch('__builtin__.__import__'):
with patch('builtins.__import__'):
self.assertTrue('units' not in hap_ctxt())
@patch.object(charmhelpers.contrib.openstack.context, 'mkdir')
@ -333,8 +325,8 @@ class HAProxyContextTest(CharmTestCase):
@patch.object(charmhelpers.contrib.openstack.context, 'relation_ids')
@patch.object(charmhelpers.contrib.openstack.context, 'log')
@patch.object(charmhelpers.contrib.openstack.context, 'kv')
@patch('__builtin__.__import__')
@patch('__builtin__.open')
@patch('builtins.__import__')
@patch('builtins.open')
def test_context_peers(self, _open, _import, _kv, _log, _rids, _runits,
_rget, _uget, _lunit, _config,
_get_address_in_network, _get_netmask_for_address,
@ -414,7 +406,7 @@ class NeutronCCContextTest(CharmTestCase):
@patch.object(context.NeutronCCContext, 'network_manager')
@patch.object(context.NeutronCCContext, 'plugin')
@patch('__builtin__.__import__')
@patch('builtins.__import__')
def test_neutroncc_context_no_setting(self, _import, plugin, nm):
plugin.return_value = None
ctxt_data = {
@ -502,7 +494,7 @@ class NeutronCCContextTest(CharmTestCase):
@patch.object(context.NeutronCCContext, 'network_manager')
@patch.object(context.NeutronCCContext, 'plugin')
@patch('__builtin__.__import__')
@patch('builtins.__import__')
def test_neutroncc_context_vxlan(self, _import, plugin, nm):
plugin.return_value = None
self.test_config.set('flat-network-providers', 'physnet2 physnet3')
@ -550,7 +542,7 @@ class NeutronCCContextTest(CharmTestCase):
@patch.object(context.NeutronCCContext, 'network_manager')
@patch.object(context.NeutronCCContext, 'plugin')
@patch('__builtin__.__import__')
@patch('builtins.__import__')
def test_neutroncc_context_l3ha(self, _import, plugin, nm):
plugin.return_value = None
self.test_config.set('enable-l3ha', True)
@ -602,7 +594,7 @@ class NeutronCCContextTest(CharmTestCase):
@patch.object(context.NeutronCCContext, 'network_manager')
@patch.object(context.NeutronCCContext, 'plugin')
@patch('__builtin__.__import__')
@patch('builtins.__import__')
def test_neutroncc_context_l3ha_l3_agents(self, _import, plugin, nm):
plugin.return_value = None
self.os_release.return_value = 'juno'
@ -616,7 +608,7 @@ class NeutronCCContextTest(CharmTestCase):
@patch.object(context.NeutronCCContext, 'network_manager')
@patch.object(context.NeutronCCContext, 'plugin')
@patch('__builtin__.__import__')
@patch('builtins.__import__')
def test_neutroncc_context_sriov(self, _import, plugin, nm):
plugin.return_value = None
self.test_config.set('enable-sriov', True)
@ -660,7 +652,7 @@ class NeutronCCContextTest(CharmTestCase):
@patch.object(context.NeutronCCContext, 'network_manager')
@patch.object(context.NeutronCCContext, 'plugin')
@patch('__builtin__.__import__')
@patch('builtins.__import__')
def test_neutroncc_context_unsupported_overlay(self, _import, plugin, nm):
plugin.return_value = None
self.test_config.set('overlay-network-type', 'bobswitch')
@ -669,7 +661,7 @@ class NeutronCCContextTest(CharmTestCase):
@patch.object(context.NeutronCCContext, 'network_manager')
@patch.object(context.NeutronCCContext, 'plugin')
@patch('__builtin__.__import__')
@patch('builtins.__import__')
def test_neutroncc_context_api_rel(self, _import, plugin, nm):
nova_url = 'http://127.0.0.10'
plugin.return_value = None
@ -697,7 +689,7 @@ class NeutronCCContextTest(CharmTestCase):
@patch.object(context.NeutronCCContext, 'network_manager')
@patch.object(context.NeutronCCContext, 'plugin')
@patch('__builtin__.__import__')
@patch('builtins.__import__')
def test_neutroncc_context_nsx(self, _import, plugin, nm):
plugin.return_value = 'nsx'
self.os_release.return_value = 'havana'
@ -712,12 +704,12 @@ class NeutronCCContextTest(CharmTestCase):
'nsx_tz_uuid': 'tzuuid',
'nsx_username': 'bob',
}
for key in expect.iterkeys():
for key in expect.keys():
self.assertEqual(napi_ctxt[key], expect[key])
@patch.object(context.NeutronCCContext, 'network_manager')
@patch.object(context.NeutronCCContext, 'plugin')
@patch('__builtin__.__import__')
@patch('builtins.__import__')
def test_neutroncc_context_nuage(self, _import, plugin, nm):
plugin.return_value = 'vsp'
self.os_release.return_value = 'havana'
@ -732,12 +724,12 @@ class NeutronCCContextTest(CharmTestCase):
'vsd_base_uri': '/nuage/api/v1_0',
'vsd_netpart_name': 'foo-enterprise',
}
for key in expect.iterkeys():
for key in expect.keys():
self.assertEqual(napi_ctxt[key], expect[key])
@patch.object(context.NeutronCCContext, 'network_manager')
@patch.object(context.NeutronCCContext, 'plugin')
@patch('__builtin__.__import__')
@patch('builtins.__import__')
def test_neutroncc_context_qos(self, _import, plugin, nm):
plugin.return_value = None
self.os_release.return_value = 'mitaka'
@ -749,12 +741,12 @@ class NeutronCCContextTest(CharmTestCase):
'extension_drivers': 'qos',
'service_plugins': service_plugins,
}
for key in expect.iterkeys():
for key in expect.keys():
self.assertEqual(napi_ctxt[key], expect[key])
@patch.object(context.NeutronCCContext, 'network_manager')
@patch.object(context.NeutronCCContext, 'plugin')
@patch('__builtin__.__import__')
@patch('builtins.__import__')
def test_neutroncc_context_service_plugins(self, _import, plugin, nm):
plugin.return_value = None
self.test_config.set('enable-qos', False)

28
unit_tests/test_neutron_api_hooks.py
View File

@ -151,7 +151,7 @@ class NeutronAPIHooksTests(CharmTestCase):
_port_calls = [call(port) for port in _ports]
self.determine_packages.return_value = _pkgs
self.determine_ports.return_value = _ports
self._call_hook('install.real')
self._call_hook('install')
self.configure_installation_source.assert_called_with(
'distro'
)
@ -174,7 +174,7 @@ class NeutronAPIHooksTests(CharmTestCase):
_port_calls = [call(port) for port in _ports]
self.determine_packages.return_value = _pkgs
self.determine_ports.return_value = _ports
self._call_hook('install.real')
self._call_hook('install')
self.configure_installation_source.assert_called_with(
'distro'
)
@ -210,7 +210,7 @@ class NeutronAPIHooksTests(CharmTestCase):
self.test_config.set('openstack-origin', repo)
self.test_config.set('openstack-origin-git', projects_yaml)
codename.return_value = 'juno'
self._call_hook('install.real')
self._call_hook('install')
self.assertTrue(self.execd_preinstall.called)
self.configure_installation_source.assert_called_with(repo)
self.apt_update.assert_called_with(fatal=True)
@ -251,22 +251,16 @@ class NeutronAPIHooksTests(CharmTestCase):
self.neutron_ready.return_value = True
self.dvr_router_present.return_value = True
self.get_dvr.return_value = False
with self.assertRaises(Exception) as context:
with self.assertRaises(Exception):
self._call_hook('config-changed')
self.assertEqual(context.exception.message,
'Cannot disable dvr while dvr enabled routers exist.'
' Please remove any distributed routers')
def test_config_changed_nol3ha_harouters(self):
self.neutron_ready.return_value = True
self.dvr_router_present.return_value = False
self.l3ha_router_present.return_value = True
self.get_l3ha.return_value = False
with self.assertRaises(Exception) as context:
with self.assertRaises(Exception):
self._call_hook('config-changed')
self.assertEqual(context.exception.message,
'Cannot disable Router HA while ha enabled routers'
' exist. Please remove any ha routers')
@patch.object(utils, 'get_os_codename_install_source')
@patch.object(hooks, 'configure_https')
@ -372,12 +366,8 @@ class NeutronAPIHooksTests(CharmTestCase):
def test_db_joined_with_postgresql(self):
self.is_relation_made.return_value = True
with self.assertRaises(Exception) as context:
with self.assertRaises(Exception):
hooks.db_joined()
self.assertEqual(context.exception.message,
'Attempting to associate a mysql database when there '
'is already associated a postgresql one')
def test_postgresql_db_joined(self):
self.unit_get.return_value = 'myhostname'
@ -389,12 +379,8 @@ class NeutronAPIHooksTests(CharmTestCase):
def test_postgresql_joined_with_db(self):
self.is_relation_made.return_value = True
with self.assertRaises(Exception) as context:
with self.assertRaises(Exception):
hooks.pgsql_neutron_db_joined()
self.assertEqual(context.exception.message,
'Attempting to associate a postgresql database when'
' there is already associated a mysql one')
@patch.object(hooks, 'neutron_plugin_api_subordinate_relation_joined')
@patch.object(hooks, 'conditional_neutron_migration')

48
unit_tests/test_neutron_api_utils.py
View File

@ -127,7 +127,7 @@ class TestNeutronAPIUtils(CharmTestCase):
pkg_list = nutils.determine_packages()
expect = deepcopy(nutils.BASE_PACKAGES)
expect.extend(['neutron-server', 'neutron-plugin-ml2'])
self.assertItemsEqual(pkg_list, expect)
self.assertEqual(sorted(pkg_list), sorted(expect))
@patch.object(nutils, 'git_install_requested')
def test_determine_vsp_packages(self, git_requested):
@ -141,7 +141,7 @@ class TestNeutronAPIUtils(CharmTestCase):
expect = deepcopy(nutils.BASE_PACKAGES)
expect.extend(['neutron-server', 'neutron-plugin-nuage',
'python-nuagenetlib', 'nuage-neutron'])
self.assertItemsEqual(pkg_list, expect)
self.assertEqual(sorted(pkg_list), sorted(expect))
@patch.object(nutils, 'git_install_requested')
def test_determine_packages_kilo(self, git_requested):
@ -153,7 +153,7 @@ class TestNeutronAPIUtils(CharmTestCase):
expect.extend(['neutron-server', 'neutron-plugin-ml2',
'python-networking-hyperv'])
expect.extend(nutils.KILO_PACKAGES)
self.assertItemsEqual(pkg_list, expect)
self.assertEqual(sorted(pkg_list), sorted(expect))
@patch.object(nutils, 'git_install_requested')
def test_determine_packages_noplugin(self, git_requested):
@ -164,12 +164,12 @@ class TestNeutronAPIUtils(CharmTestCase):
pkg_list = nutils.determine_packages()
expect = deepcopy(nutils.BASE_PACKAGES)
expect.extend(['neutron-server'])
self.assertItemsEqual(pkg_list, expect)
self.assertEqual(sorted(pkg_list), sorted(expect))
def test_determine_ports(self):
self.os_release.return_value = 'havana'
port_list = nutils.determine_ports()
self.assertItemsEqual(port_list, [9696])
self.assertEqual(port_list, [9696])
@patch.object(nutils, 'manage_plugin')
@patch('os.path.exists')
@ -232,26 +232,14 @@ class TestNeutronAPIUtils(CharmTestCase):
_restart_map = nutils.restart_map()
ML2CONF = "/etc/neutron/plugins/ml2/ml2_conf.ini"
expect = OrderedDict([
(nutils.NEUTRON_CONF, {
'services': ['neutron-server'],
}),
(nutils.NEUTRON_DEFAULT, {
'services': ['neutron-server'],
}),
(nutils.API_PASTE_INI, {
'services': ['neutron-server'],
}),
(ML2CONF, {
'services': ['neutron-server'],
}),
(nutils.APACHE_CONF, {
'services': ['apache2'],
}),
(nutils.HAPROXY_CONF, {
'services': ['haproxy'],
}),
(nutils.NEUTRON_CONF, ['neutron-server']),
(nutils.NEUTRON_DEFAULT, ['neutron-server']),
(nutils.API_PASTE_INI, ['neutron-server']),
(nutils.APACHE_CONF, ['apache2']),
(nutils.HAPROXY_CONF, ['haproxy']),
(ML2CONF, ['neutron-server']),
])
self.assertItemsEqual(_restart_map, expect)
self.assertEqual(_restart_map, expect)
@patch('os.path.exists')
def test_register_configs(self, mock_path_exists):
@ -275,7 +263,7 @@ class TestNeutronAPIUtils(CharmTestCase):
'/etc/neutron/plugins/ml2/ml2_conf.ini',
'/etc/apache2/sites-available/openstack_https_frontend',
'/etc/haproxy/haproxy.cfg']
self.assertItemsEqual(_regconfs.configs, confs)
self.assertEqual(sorted(_regconfs.configs), sorted(confs))
@patch('os.path.isfile')
def test_keystone_ca_cert_b64_no_cert_file(self, _isfile):
@ -390,7 +378,7 @@ class TestNeutronAPIUtils(CharmTestCase):
self.assertFalse(migrate_neutron_db.called)
@patch.object(ncontext, 'IdentityServiceContext')
@patch('neutronclient.v2_0.client.Client')
@patch.object(nutils, 'FakeNeutronClient')
def test_get_neutron_client(self, nclient, IdentityServiceContext):
creds = {
'auth_protocol': 'http',
@ -533,16 +521,16 @@ class TestNeutronAPIUtils(CharmTestCase):
add_user_to_group.assert_called_with('neutron', 'neutron')
expected = [
call('/var/lib/neutron', owner='neutron',
group='neutron', perms=0755, force=False),
group='neutron', perms=0o755, force=False),
call('/var/lib/neutron/lock', owner='neutron',
group='neutron', perms=0755, force=False),
group='neutron', perms=0o755, force=False),
call('/var/log/neutron', owner='neutron',
group='neutron', perms=0755, force=False),
group='neutron', perms=0o755, force=False),
]
self.assertEqual(mkdir.call_args_list, expected)
expected = [
call('/var/log/neutron/server.log', '', owner='neutron',
group='neutron', perms=0600),
group='neutron', perms=0o600),
]
self.assertEqual(write_file.call_args_list, expected)

7
unit_tests/test_utils.py
View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import io
import os
import logging
import unittest
@ -54,7 +55,7 @@ def get_default_config():
'''
default_config = {}
config = load_config()
for k, v in config.iteritems():
for k, v in config.items():
if 'default' in v:
default_config[k] = v['default']
else:
@ -128,12 +129,12 @@ def patch_open():
Yields the mock for "open" and "file", respectively.'''
mock_open = MagicMock(spec=open)
mock_file = MagicMock(spec=file)
mock_file = MagicMock(spec=io.FileIO)
@contextmanager
def stub_open(*args, **kwargs):
mock_open(*args, **kwargs)
yield mock_file
with patch('__builtin__.open', stub_open):
with patch('builtins.open', stub_open):
yield mock_open, mock_file