py3: Switch packages to Python 3 for train and later

Switch package install to Python 3 for OpenStack Train and later.

When upgrading, remove any python-* packages that were explicitly
installated and then autoremove --purge any dependencies that are
no longer required.

This patch also includes the following related changes:
  * Use the common files package, swift, rather than python-swift
    when using a package name to determine release.

Also add OS_* to tox.ini to allow functional tests to execute.

Change-Id: I121af845bf11c22052479a497b196a4670021256

hooks/swift_storage_hooks.py

@@ -39,9 +39,9 @@ _add_path(_root)
 
 
 from lib.swift_storage_utils import (
-    PACKAGES,
     RESTART_MAP,
     SWIFT_SVCS,
+    determine_packages,
     do_openstack_upgrade,
     ensure_swift_directories,
     fetch_swift_rings,
@@ -103,6 +103,7 @@ from charmhelpers.contrib.openstack.utils import (
     is_unit_paused_set,
     set_unit_paused,
     set_unit_upgrading,
+    get_os_codename_install_source,
 )
 from charmhelpers.contrib.network.ip import (
     get_relation_ip,
@@ -200,10 +201,13 @@ def initialize_ufw():
 def install():
     status_set('maintenance', 'Executing pre-install')
     execd_preinstall()
-    configure_installation_source(config('openstack-origin'))
+    src = config('openstack-origin')
+    configure_installation_source(src)
     status_set('maintenance', 'Installing apt packages')
     apt_update()
-    apt_install(PACKAGES, fatal=True)
+    rel = get_os_codename_install_source(src)
+    pkgs = determine_packages(rel)
+    apt_install(pkgs, fatal=True)
     initialize_ufw()
     ensure_swift_directories()
 
@@ -262,7 +266,9 @@ def install_vaultlocker():
 @harden()
 def upgrade_charm():
     initialize_ufw()
-    apt_install(filter_installed_packages(PACKAGES), fatal=True)
+    rel = get_os_codename_install_source(config('openstack-origin'))
+    pkgs = determine_packages(rel)
+    apt_install(pkgs, fatal=True)
     update_nrpe_config()
     ensure_devs_tracked()
 

lib/swift_storage_utils.py

@@ -24,7 +24,11 @@ from lib.swift_storage_context import (
 
 from charmhelpers.fetch import (
     apt_upgrade,
-    apt_update
+    apt_update,
+    apt_install,
+    apt_purge,
+    apt_autoremove,
+    filter_missing_packages,
 )
 
 from charmhelpers.core.unitdata import (
@@ -57,6 +61,7 @@ from charmhelpers.core.hookenv import (
     ingress_address,
     storage_list,
     storage_get,
+    status_set,
 )
 
 from charmhelpers.contrib.network import ufw
@@ -77,6 +82,9 @@ from charmhelpers.contrib.openstack.utils import (
     get_os_codename_install_source,
     get_os_codename_package,
     save_script_rc as _save_script_rc,
+    CompareOpenStackReleases,
+    reset_os_release,
+    os_release,
 )
 
 from charmhelpers.contrib.openstack import (
@@ -93,9 +101,29 @@ import charmhelpers.contrib.openstack.vaultlocker as vaultlocker
 from charmhelpers.core.unitdata import kv
 
 PACKAGES = [
-    'swift', 'swift-account', 'swift-container', 'swift-object',
+    'gdisk',
-    'xfsprogs', 'gdisk', 'lvm2', 'python-jinja2', 'python-psutil',
+    'lvm2',
+    'swift',
+    'swift-account',
+    'swift-container',
+    'swift-object',
+    'python-jinja2',
+    'python-psutil',
     'ufw',
+    'xfsprogs',
+]
+
+PY3_PACKAGES = [
+    'python3-jinja2',
+    'python3-psutil',
+    'python3-six',
+    'python3-swift',
+]
+
+PURGE_PACKAGES = [
+    'python-jinja2',
+    'python-psutil',
+    'python-swift',
 ]
 
 VERSION_PACKAGE = 'swift-account'
@@ -163,7 +191,7 @@ def ensure_swift_directories():
 
 
 def register_configs():
-    release = get_os_codename_package('python-swift', fatal=False) or 'essex'
+    release = get_os_codename_package('swift', fatal=False) or 'essex'
     configs = templating.OSConfigRenderer(templates_dir=TEMPLATES,
                                           openstack_release=release)
     configs.register('/etc/swift/swift.conf',
@@ -181,6 +209,43 @@ def register_configs():
     return configs
 
 
+def determine_packages(release):
+    """Determine what packages are needed for a given OpenStack release."""
+    cmp_openstack = CompareOpenStackReleases(release)
+    pkgs = PACKAGES[:]
+    if cmp_openstack >= 'train':
+        pkgs = [p for p in pkgs if not p.startswith('python-')]
+        pkgs.extend(PY3_PACKAGES)
+    return pkgs
+
+
+def determine_purge_packages():
+    '''
+    Determine list of packages that where previously installed which are no
+    longer needed.
+
+    :returns: list of package names
+    '''
+    cmp_openstack = CompareOpenStackReleases(os_release('swift'))
+    if cmp_openstack >= 'train':
+        return PURGE_PACKAGES
+    return []
+
+
+def remove_old_packages():
+    '''Purge any packages that need to be removed.
+
+    :returns: bool Whether packages were removed.
+    '''
+    installed_packages = filter_missing_packages(determine_purge_packages())
+    if installed_packages:
+        log('Removing apt packages')
+        status_set('maintenance', 'Removing apt packages')
+        apt_purge(installed_packages, fatal=True)
+        apt_autoremove(purge=True, fatal=True)
+    return bool(installed_packages)
+
+
 def swift_init(target, action, fatal=False):
     '''
     Call swift-init on a specific target with given action, potentially
@@ -204,6 +269,10 @@ def do_openstack_upgrade(configs):
     ]
     apt_update()
     apt_upgrade(options=dpkg_opts, fatal=True, dist=True)
+    reset_os_release()
+    apt_install(packages=determine_packages(new_os_rel),
+                options=dpkg_opts, fatal=True)
+    remove_old_packages()
     configs.set_release(openstack_release=new_os_rel)
     configs.write_all()
     if not is_paused():

tox.ini

@@ -19,8 +19,7 @@ setenv = VIRTUAL_ENV={envdir}
 install_command =
   pip install {opts} {packages}
 commands = stestr run --slowest {posargs}
-whitelist_externals = juju
+passenv = HOME TERM AMULET_* CS_API_* OS_* TEST_*
-passenv = HOME TERM AMULET_* CS_* OS_* TEST_*
 
 [testenv:py3]
 basepython = python3

unit_tests/test_swift_storage_relations.py

@@ -169,11 +169,12 @@ class SwiftStorageRelationsTests(CharmTestCase):
     @patch.object(hooks, 'add_ufw_gre_rule', lambda *args: None)
     @patch.object(hooks, 'ensure_devs_tracked')
     def test_upgrade_charm(self, mock_ensure_devs_tracked):
-        self.filter_installed_packages.return_value = [
-            'python-psutil']
         hooks.upgrade_charm()
-        self.apt_install.assert_called_with([
+        self.apt_install.assert_called_with(
-            'python-psutil'], fatal=True)
+            ['gdisk', 'lvm2', 'swift', 'swift-account',
+             'swift-container', 'swift-object', 'python-jinja2',
+             'python-psutil', 'ufw', 'xfsprogs'],
+            fatal=True)
         self.assertTrue(self.update_nrpe_config.called)
         self.assertTrue(mock_ensure_devs_tracked.called)
 

unit_tests/test_swift_storage_utils.py

@@ -25,6 +25,7 @@ import lib.swift_storage_utils as swift_utils
 TO_PATCH = [
     'apt_update',
     'apt_upgrade',
+    'apt_install',
     'log',
     'config',
     'configure_installation_source',
@@ -52,6 +53,8 @@ TO_PATCH = [
     'relation_ids',
     'vaultlocker',
     'kv',
+    'reset_os_release',
+    'CompareOpenStackReleases',
 ]
 
 
@@ -477,13 +480,15 @@ class SwiftStorageUtilsTests(CharmTestCase):
         ]
         self.assertEqual(ex, configs.register.call_args_list)
 
-    def test_do_upgrade(self):
+    @patch.object(swift_utils, 'remove_old_packages')
+    def test_do_upgrade_queens(self, mock_remove_old_packages):
         self.is_paused.return_value = False
-        self.test_config.set('openstack-origin', 'cloud:precise-grizzly')
+        self.test_config.set('openstack-origin', 'cloud:bionic-queens')
-        self.get_os_codename_install_source.return_value = 'grizzly'
+        self.get_os_codename_install_source.return_value = 'queens'
+        self.CompareOpenStackReleases.return_value = 'queens'
         swift_utils.do_openstack_upgrade(MagicMock())
         self.configure_installation_source.assert_called_with(
-            'cloud:precise-grizzly'
+            'cloud:bionic-queens'
         )
         dpkg_opts = [
             '--option', 'Dpkg::Options::=--force-confnew',
@@ -494,6 +499,49 @@ class SwiftStorageUtilsTests(CharmTestCase):
             options=dpkg_opts,
             fatal=True, dist=True
         )
+        self.apt_install.assert_called_with(
+            options=dpkg_opts,
+            packages=['gdisk', 'lvm2', 'swift', 'swift-account',
+                      'swift-container', 'swift-object', 'python-jinja2',
+                      'python-psutil', 'ufw', 'xfsprogs'],
+            fatal=True
+        )
+        self.assertTrue(mock_remove_old_packages.called)
+        self.assertTrue(self.reset_os_release.called)
+        services = (swift_utils.ACCOUNT_SVCS + swift_utils.CONTAINER_SVCS +
+                    swift_utils.OBJECT_SVCS)
+        for service in services:
+            self.assertIn(call(service), self.service_restart.call_args_list)
+
+    @patch.object(swift_utils, 'remove_old_packages')
+    def test_do_upgrade_train(self, mock_remove_old_packages):
+        self.is_paused.return_value = False
+        self.test_config.set('openstack-origin', 'cloud:bionic-train')
+        self.get_os_codename_install_source.return_value = 'train'
+        self.CompareOpenStackReleases.return_value = 'train'
+        swift_utils.do_openstack_upgrade(MagicMock())
+        self.configure_installation_source.assert_called_with(
+            'cloud:bionic-train'
+        )
+        dpkg_opts = [
+            '--option', 'Dpkg::Options::=--force-confnew',
+            '--option', 'Dpkg::Options::=--force-confdef',
+        ]
+        self.assertTrue(self.apt_update.called)
+        self.apt_upgrade.assert_called_with(
+            options=dpkg_opts,
+            fatal=True, dist=True
+        )
+        self.apt_install.assert_called_with(
+            options=dpkg_opts,
+            packages=['gdisk', 'lvm2', 'swift', 'swift-account',
+                      'swift-container', 'swift-object', 'ufw', 'xfsprogs',
+                      'python3-jinja2', 'python3-psutil', 'python3-six',
+                      'python3-swift'],
+            fatal=True
+        )
+        self.assertTrue(mock_remove_old_packages.called)
+        self.assertTrue(self.reset_os_release.called)
         services = (swift_utils.ACCOUNT_SVCS + swift_utils.CONTAINER_SVCS +
                     swift_utils.OBJECT_SVCS)
         for service in services: