From 73afa34c38c2143d4dca7c4504443ecd1d088f82 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 19 Sep 2018 13:41:16 +0200 Subject: [PATCH] py3: Switch to Python 3 for OpenStack Rocky Switch to using python3-{nova,neutron} at OpenStack Rocky; purge previously installed Python 2 modules on upgrade. Switch default test target to bionic-rocky. Change-Id: I0a2b48bfc8c950efea3e83c74ec3eb6dd43796c4 --- .../charmhelpers/contrib/hahelpers/apache.py | 14 +-- .../contrib/openstack/amulet/utils.py | 104 +++++++++++++----- .../charmhelpers/contrib/openstack/context.py | 4 - hooks/charmhelpers/core/host.py | 26 ++++- hooks/neutron_utils.py | 36 ++++++ .../contrib/openstack/amulet/utils.py | 104 +++++++++++++----- tests/charmhelpers/core/host.py | 26 ++++- tox.ini | 2 +- unit_tests/test_neutron_utils.py | 67 +++++++++++ 9 files changed, 308 insertions(+), 75 deletions(-) diff --git a/hooks/charmhelpers/contrib/hahelpers/apache.py b/hooks/charmhelpers/contrib/hahelpers/apache.py index 605a1bec..2c1e371e 100644 --- a/hooks/charmhelpers/contrib/hahelpers/apache.py +++ b/hooks/charmhelpers/contrib/hahelpers/apache.py @@ -23,8 +23,8 @@ # import os -import subprocess +from charmhelpers.core import host from charmhelpers.core.hookenv import ( config as config_get, relation_get, @@ -83,14 +83,4 @@ def retrieve_ca_cert(cert_file): def install_ca_cert(ca_cert): - if ca_cert: - cert_file = ('/usr/local/share/ca-certificates/' - 'keystone_juju_ca_cert.crt') - old_cert = retrieve_ca_cert(cert_file) - if old_cert and old_cert == ca_cert: - log("CA cert is the same as installed version", level=INFO) - else: - log("Installing new CA cert", level=INFO) - with open(cert_file, 'wb') as crt: - crt.write(ca_cert) - subprocess.check_call(['update-ca-certificates', '--fresh']) + host.install_ca_cert(ca_cert, 'keystone_juju_ca_cert') diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index 936b4036..10dbe59a 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -680,18 +680,30 @@ class OpenStackAmuletUtils(AmuletUtils): nova.flavors.create(name, ram, vcpus, disk, flavorid, ephemeral, swap, rxtx_factor, is_public) - def create_cirros_image(self, glance, image_name): - """Download the latest cirros image and upload it to glance, - validate and return a resource pointer. + def glance_create_image(self, glance, image_name, image_url, + download_dir='tests', + hypervisor_type='qemu', + disk_format='qcow2', + architecture='x86_64', + container_format='bare'): + """Download an image and upload it to glance, validate its status + and return an image object pointer. KVM defaults, can override for + LXD. - :param glance: pointer to authenticated glance connection + :param glance: pointer to authenticated glance api connection :param image_name: display name for new image + :param image_url: url to retrieve + :param download_dir: directory to store downloaded image file + :param hypervisor_type: glance image hypervisor property + :param disk_format: glance image disk format + :param architecture: glance image architecture property + :param container_format: glance image container format :returns: glance image pointer """ - self.log.debug('Creating glance cirros image ' - '({})...'.format(image_name)) + self.log.debug('Creating glance image ({}) from ' + '{}...'.format(image_name, image_url)) - # Download cirros image + # Download image http_proxy = os.getenv('AMULET_HTTP_PROXY') self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) if http_proxy: @@ -700,31 +712,33 @@ class OpenStackAmuletUtils(AmuletUtils): else: opener = urllib.FancyURLopener() - f = opener.open('http://download.cirros-cloud.net/version/released') - version = f.read().strip() - cirros_img = 'cirros-{}-x86_64-disk.img'.format(version) - local_path = os.path.join('tests', cirros_img) - - if not os.path.exists(local_path): - cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net', - version, cirros_img) - opener.retrieve(cirros_url, local_path) - f.close() + abs_file_name = os.path.join(download_dir, image_name) + if not os.path.exists(abs_file_name): + opener.retrieve(image_url, abs_file_name) + # Create glance image + glance_properties = { + 'architecture': architecture, + 'hypervisor_type': hypervisor_type + } # Create glance image if float(glance.version) < 2.0: - with open(local_path) as fimage: - image = glance.images.create(name=image_name, is_public=True, - disk_format='qcow2', - container_format='bare', - data=fimage) + with open(abs_file_name) as f: + image = glance.images.create( + name=image_name, + is_public=True, + disk_format=disk_format, + container_format=container_format, + properties=glance_properties, + data=f) else: image = glance.images.create( name=image_name, - disk_format="qcow2", visibility="public", - container_format="bare") - glance.images.upload(image.id, open(local_path, 'rb')) + disk_format=disk_format, + container_format=container_format) + glance.images.upload(image.id, open(abs_file_name, 'rb')) + glance.images.update(image.id, **glance_properties) # Wait for image to reach active status img_id = image.id @@ -753,15 +767,49 @@ class OpenStackAmuletUtils(AmuletUtils): val_img_stat, val_img_cfmt, val_img_dfmt)) if val_img_name == image_name and val_img_stat == 'active' \ - and val_img_pub is True and val_img_cfmt == 'bare' \ - and val_img_dfmt == 'qcow2': + and val_img_pub is True and val_img_cfmt == container_format \ + and val_img_dfmt == disk_format: self.log.debug(msg_attr) else: - msg = ('Volume validation failed, {}'.format(msg_attr)) + msg = ('Image validation failed, {}'.format(msg_attr)) amulet.raise_status(amulet.FAIL, msg=msg) return image + def create_cirros_image(self, glance, image_name): + """Download the latest cirros image and upload it to glance, + validate and return a resource pointer. + + :param glance: pointer to authenticated glance connection + :param image_name: display name for new image + :returns: glance image pointer + """ + # /!\ DEPRECATION WARNING + self.log.warn('/!\\ DEPRECATION WARNING: use ' + 'glance_create_image instead of ' + 'create_cirros_image.') + + self.log.debug('Creating glance cirros image ' + '({})...'.format(image_name)) + + # Get cirros image URL + http_proxy = os.getenv('AMULET_HTTP_PROXY') + self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) + if http_proxy: + proxies = {'http': http_proxy} + opener = urllib.FancyURLopener(proxies) + else: + opener = urllib.FancyURLopener() + + f = opener.open('http://download.cirros-cloud.net/version/released') + version = f.read().strip() + cirros_img = 'cirros-{}-x86_64-disk.img'.format(version) + cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net', + version, cirros_img) + f.close() + + return self.glance_create_image(glance, image_name, cirros_url) + def delete_image(self, glance, image): """Delete the specified image.""" diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index 92cb742e..3e4e82a7 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -1523,10 +1523,6 @@ class NeutronAPIContext(OSContextGenerator): 'rel_key': 'enable-nsg-logging', 'default': False, }, - 'nsg_log_output_base': { - 'rel_key': 'nsg-log-output-base', - 'default': None, - }, } ctxt = self.get_neutron_options({}) for rid in relation_ids('neutron-plugin-api'): diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index e9fd38a0..0ebfdbd1 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -34,7 +34,7 @@ import six from contextlib import contextmanager from collections import OrderedDict -from .hookenv import log, DEBUG, local_unit +from .hookenv import log, INFO, DEBUG, local_unit, charm_name from .fstab import Fstab from charmhelpers.osplatform import get_platform @@ -1040,3 +1040,27 @@ def modulo_distribution(modulo=3, wait=30, non_zero_wait=False): return modulo * wait else: return calculated_wait_time + + +def install_ca_cert(ca_cert, name=None): + """ + Install the given cert as a trusted CA. + + The ``name`` is the stem of the filename where the cert is written, and if + not provided, it will default to ``juju-{charm_name}``. + + If the cert is empty or None, or is unchanged, nothing is done. + """ + if not ca_cert: + return + if not isinstance(ca_cert, bytes): + ca_cert = ca_cert.encode('utf8') + if not name: + name = 'juju-{}'.format(charm_name()) + cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name) + new_hash = hashlib.md5(ca_cert).hexdigest() + if file_hash(cert_file) == new_hash: + return + log("Installing new CA cert at: {}".format(cert_file), level=INFO) + write_file(cert_file, ca_cert) + subprocess.check_call(['update-ca-certificates', '--fresh']) diff --git a/hooks/neutron_utils.py b/hooks/neutron_utils.py index 1417e8b7..91b00630 100644 --- a/hooks/neutron_utils.py +++ b/hooks/neutron_utils.py @@ -25,6 +25,9 @@ from charmhelpers.fetch import ( apt_upgrade, apt_update, apt_install, + apt_autoremove, + apt_purge, + filter_missing_packages, ) from charmhelpers.contrib.network.ovs import ( add_bridge, @@ -176,6 +179,21 @@ GATEWAY_PKGS = { ], } +PURGE_PACKAGES = [ + 'python-mysqldb', + 'python-psycopg2', + 'python-oslo.config', + 'python-nova', + 'python-neutron', + 'python-neutron-fwaas', +] + +PY3_PACKAGES = [ + 'python3-nova', + 'python3-neutron', + 'python3-neutron-fwaas', +] + EARLY_PACKAGES = { OVS: ['openvswitch-datapath-dkms'], NSX: [], @@ -253,9 +271,21 @@ def get_packages(): packages.append('neutron-lbaasv2-agent') packages.extend(determine_l3ha_packages()) + if cmp_os_source >= 'rocky': + packages = [p for p in packages if not p.startswith('python-')] + packages.extend(PY3_PACKAGES) + return packages +def get_purge_packages(): + '''Return a list of packages to purge for the current OS release''' + cmp_os_source = CompareOpenStackReleases(os_release('neutron-common')) + if cmp_os_source >= 'rocky': + return PURGE_PACKAGES + return [] + + def determine_l3ha_packages(): if use_l3ha(): return L3HA_PACKAGES @@ -706,12 +736,18 @@ def do_openstack_upgrade(configs): apt_update(fatal=True) apt_upgrade(options=dpkg_opts, fatal=True, dist=True) + # The cached version of os_release will now be invalid as the pkg version # should have changed during the upgrade. reset_os_release() apt_install(get_early_packages(), fatal=True) apt_install(get_packages(), fatal=True) + installed_packages = filter_missing_packages(get_purge_packages()) + if installed_packages: + apt_purge(installed_packages, fatal=True) + apt_autoremove(purge=True, fatal=True) + def configure_ovs(): if config('plugin') in [OVS, OVS_ODL]: diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index 936b4036..10dbe59a 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -680,18 +680,30 @@ class OpenStackAmuletUtils(AmuletUtils): nova.flavors.create(name, ram, vcpus, disk, flavorid, ephemeral, swap, rxtx_factor, is_public) - def create_cirros_image(self, glance, image_name): - """Download the latest cirros image and upload it to glance, - validate and return a resource pointer. + def glance_create_image(self, glance, image_name, image_url, + download_dir='tests', + hypervisor_type='qemu', + disk_format='qcow2', + architecture='x86_64', + container_format='bare'): + """Download an image and upload it to glance, validate its status + and return an image object pointer. KVM defaults, can override for + LXD. - :param glance: pointer to authenticated glance connection + :param glance: pointer to authenticated glance api connection :param image_name: display name for new image + :param image_url: url to retrieve + :param download_dir: directory to store downloaded image file + :param hypervisor_type: glance image hypervisor property + :param disk_format: glance image disk format + :param architecture: glance image architecture property + :param container_format: glance image container format :returns: glance image pointer """ - self.log.debug('Creating glance cirros image ' - '({})...'.format(image_name)) + self.log.debug('Creating glance image ({}) from ' + '{}...'.format(image_name, image_url)) - # Download cirros image + # Download image http_proxy = os.getenv('AMULET_HTTP_PROXY') self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) if http_proxy: @@ -700,31 +712,33 @@ class OpenStackAmuletUtils(AmuletUtils): else: opener = urllib.FancyURLopener() - f = opener.open('http://download.cirros-cloud.net/version/released') - version = f.read().strip() - cirros_img = 'cirros-{}-x86_64-disk.img'.format(version) - local_path = os.path.join('tests', cirros_img) - - if not os.path.exists(local_path): - cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net', - version, cirros_img) - opener.retrieve(cirros_url, local_path) - f.close() + abs_file_name = os.path.join(download_dir, image_name) + if not os.path.exists(abs_file_name): + opener.retrieve(image_url, abs_file_name) + # Create glance image + glance_properties = { + 'architecture': architecture, + 'hypervisor_type': hypervisor_type + } # Create glance image if float(glance.version) < 2.0: - with open(local_path) as fimage: - image = glance.images.create(name=image_name, is_public=True, - disk_format='qcow2', - container_format='bare', - data=fimage) + with open(abs_file_name) as f: + image = glance.images.create( + name=image_name, + is_public=True, + disk_format=disk_format, + container_format=container_format, + properties=glance_properties, + data=f) else: image = glance.images.create( name=image_name, - disk_format="qcow2", visibility="public", - container_format="bare") - glance.images.upload(image.id, open(local_path, 'rb')) + disk_format=disk_format, + container_format=container_format) + glance.images.upload(image.id, open(abs_file_name, 'rb')) + glance.images.update(image.id, **glance_properties) # Wait for image to reach active status img_id = image.id @@ -753,15 +767,49 @@ class OpenStackAmuletUtils(AmuletUtils): val_img_stat, val_img_cfmt, val_img_dfmt)) if val_img_name == image_name and val_img_stat == 'active' \ - and val_img_pub is True and val_img_cfmt == 'bare' \ - and val_img_dfmt == 'qcow2': + and val_img_pub is True and val_img_cfmt == container_format \ + and val_img_dfmt == disk_format: self.log.debug(msg_attr) else: - msg = ('Volume validation failed, {}'.format(msg_attr)) + msg = ('Image validation failed, {}'.format(msg_attr)) amulet.raise_status(amulet.FAIL, msg=msg) return image + def create_cirros_image(self, glance, image_name): + """Download the latest cirros image and upload it to glance, + validate and return a resource pointer. + + :param glance: pointer to authenticated glance connection + :param image_name: display name for new image + :returns: glance image pointer + """ + # /!\ DEPRECATION WARNING + self.log.warn('/!\\ DEPRECATION WARNING: use ' + 'glance_create_image instead of ' + 'create_cirros_image.') + + self.log.debug('Creating glance cirros image ' + '({})...'.format(image_name)) + + # Get cirros image URL + http_proxy = os.getenv('AMULET_HTTP_PROXY') + self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) + if http_proxy: + proxies = {'http': http_proxy} + opener = urllib.FancyURLopener(proxies) + else: + opener = urllib.FancyURLopener() + + f = opener.open('http://download.cirros-cloud.net/version/released') + version = f.read().strip() + cirros_img = 'cirros-{}-x86_64-disk.img'.format(version) + cirros_url = 'http://{}/{}/{}'.format('download.cirros-cloud.net', + version, cirros_img) + f.close() + + return self.glance_create_image(glance, image_name, cirros_url) + def delete_image(self, glance, image): """Delete the specified image.""" diff --git a/tests/charmhelpers/core/host.py b/tests/charmhelpers/core/host.py index e9fd38a0..0ebfdbd1 100644 --- a/tests/charmhelpers/core/host.py +++ b/tests/charmhelpers/core/host.py @@ -34,7 +34,7 @@ import six from contextlib import contextmanager from collections import OrderedDict -from .hookenv import log, DEBUG, local_unit +from .hookenv import log, INFO, DEBUG, local_unit, charm_name from .fstab import Fstab from charmhelpers.osplatform import get_platform @@ -1040,3 +1040,27 @@ def modulo_distribution(modulo=3, wait=30, non_zero_wait=False): return modulo * wait else: return calculated_wait_time + + +def install_ca_cert(ca_cert, name=None): + """ + Install the given cert as a trusted CA. + + The ``name`` is the stem of the filename where the cert is written, and if + not provided, it will default to ``juju-{charm_name}``. + + If the cert is empty or None, or is unchanged, nothing is done. + """ + if not ca_cert: + return + if not isinstance(ca_cert, bytes): + ca_cert = ca_cert.encode('utf8') + if not name: + name = 'juju-{}'.format(charm_name()) + cert_file = '/usr/local/share/ca-certificates/{}.crt'.format(name) + new_hash = hashlib.md5(ca_cert).hexdigest() + if file_hash(cert_file) == new_hash: + return + log("Installing new CA cert at: {}".format(cert_file), level=INFO) + write_file(cert_file, ca_cert) + subprocess.check_call(['update-ca-certificates', '--fresh']) diff --git a/tox.ini b/tox.ini index c5765bae..b7a79be6 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,7 @@ basepython = python2.7 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = - bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-queens --no-destroy + bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-rocky --no-destroy [testenv:func27-dfs] # Charm Functional Test diff --git a/unit_tests/test_neutron_utils.py b/unit_tests/test_neutron_utils.py index e44606db..55e58df2 100644 --- a/unit_tests/test_neutron_utils.py +++ b/unit_tests/test_neutron_utils.py @@ -21,6 +21,9 @@ TO_PATCH = [ 'apt_update', 'apt_upgrade', 'apt_install', + 'apt_autoremove', + 'apt_purge', + 'filter_missing_packages', 'configure_installation_source', 'log', 'add_bridge', @@ -148,6 +151,33 @@ class TestNeutronUtils(CharmTestCase): self.assertFalse('python-mysqldb' in packages) self.assertTrue('python-pymysql' in packages) + def test_get_packages_ovs_rocky(self): + self.config.return_value = 'ovs' + self.os_release.return_value = 'rocky' + packages = neutron_utils.get_packages() + self.assertEqual( + len(packages), + len([p for p in packages if not p.startswith('python-')]) + ) + + def test_get_purge_packages_ovs(self): + self.config.return_value = 'ovs' + self.os_release.return_value = 'queens' + self.assertEqual([], neutron_utils.get_purge_packages()) + + def test_get_purge_packages_ovs_rocky(self): + self.config.return_value = 'ovs' + self.os_release.return_value = 'rocky' + self.assertEqual([ + 'python-mysqldb', + 'python-psycopg2', + 'python-oslo.config', + 'python-nova', + 'python-neutron', + 'python-neutron-fwaas'], + neutron_utils.get_purge_packages() + ) + def test_get_packages_ovsodl_icehouse(self): self.config.return_value = 'ovs-odl' self.os_release.return_value = 'icehouse' @@ -297,6 +327,7 @@ class TestNeutronUtils(CharmTestCase): self.test_config.set('plugin', 'ovs') self.get_os_codename_install_source.return_value = 'havana' self.os_release.return_value = 'havana' + self.filter_missing_packages.side_effect = lambda x: x neutron_utils.do_openstack_upgrade(mock_configs) mock_register_configs.assert_called_with('havana') self.assertTrue(self.log.called) @@ -311,6 +342,42 @@ class TestNeutronUtils(CharmTestCase): self.configure_installation_source.assert_called_with( 'cloud:precise-havana' ) + self.apt_purge.assert_not_called() + self.apt_autoremove.assert_not_called() + + @patch.object(neutron_utils, 'register_configs') + @patch('charmhelpers.contrib.openstack.templating.OSConfigRenderer') + def test_do_openstack_upgrade_rocky(self, mock_renderer, + mock_register_configs): + mock_configs = MagicMock() + mock_register_configs.return_value = mock_configs + self.config.side_effect = self.test_config.get + self.is_relation_made.return_value = False + self.test_config.set('openstack-origin', 'cloud:bionic-rocky') + self.test_config.set('plugin', 'ovs') + self.get_os_codename_install_source.return_value = 'rocky' + self.os_release.return_value = 'rocky' + self.filter_missing_packages.side_effect = lambda x: x + neutron_utils.do_openstack_upgrade(mock_configs) + mock_register_configs.assert_called_with('rocky') + self.assertTrue(self.log.called) + self.apt_update.assert_called_with(fatal=True) + dpkg_opts = [ + '--option', 'Dpkg::Options::=--force-confnew', + '--option', 'Dpkg::Options::=--force-confdef', + ] + self.apt_upgrade.assert_called_with( + options=dpkg_opts, fatal=True, dist=True + ) + self.apt_purge.assert_called_with( + neutron_utils.PURGE_PACKAGES, fatal=True + ) + self.apt_autoremove.assert_called_with( + purge=True, fatal=True + ) + self.configure_installation_source.assert_called_with( + 'cloud:bionic-rocky' + ) @patch('charmhelpers.contrib.openstack.templating.OSConfigRenderer') def test_register_configs_ovs(self, mock_renderer):