py3: Switch to using Python 3 for rocky or later

Switch package install to Python 3 for OpenStack Rocky or later.

When upgrading, remove any python-* packages that where explicitly
installated and then autoremove --purge any dependencies that are
no longer required.

Change-Id: I1355382846aeb6d0ad29be32b9e6fcf47d318290

hooks/horizon_utils.py

@@ -15,6 +15,7 @@
 # vim: set ts=4:et
 
 from collections import OrderedDict
+from copy import deepcopy
 import os
 import subprocess
 import time
@@ -51,7 +52,10 @@ from charmhelpers.fetch import (
     apt_upgrade,
     apt_update,
     add_source,
-    apt_install
+    apt_install,
+    apt_purge,
+    apt_autoremove,
+    filter_missing_packages,
 )
 
 import hooks.horizon_contexts as horizon_contexts
@@ -66,6 +70,18 @@ BASE_PACKAGES = [
     'python-novaclient',
 ]
 
+PY3_PACKAGES = [
+    'python3-django-horizon',
+    'python3-designate-dashboard',
+    'python3-heat-dashboard',
+    'python3-neutron-lbaas-dashboard',
+    'python3-keystoneclient',
+    'python3-novaclient',
+    'python3-memcache',
+    'python3-pymysql',
+    'libapache2-mod-wsgi-py3',
+]
+
 VERSION_PACKAGE = 'openstack-dashboard'
 
 REQUIRED_INTERFACES = {
@@ -241,20 +257,44 @@ def enable_ssl():
 
 def determine_packages():
     """Determine packages to install"""
-    packages = BASE_PACKAGES
-    release = get_os_codename_install_source(config('openstack-origin'))
+    packages = deepcopy(BASE_PACKAGES)
+    release = CompareOpenStackReleases(os_release('openstack-dashboard'))
     # Really should be handled as a dep in the openstack-dashboard package
-    if CompareOpenStackReleases(release) >= 'mitaka':
+    if release >= 'mitaka':
         packages.append('python-pymysql')
-    if (CompareOpenStackReleases(release) >= 'ocata' and
-            CompareOpenStackReleases(release) < 'rocky'):
+    if release >= 'ocata' and release < 'rocky':
         packages.append('python-neutron-lbaas-dashboard')
-    if CompareOpenStackReleases(release) >= 'queens':
+    if release >= 'queens':
         packages.append('python-designate-dashboard')
         packages.append('python-heat-dashboard')
+    if release >= 'rocky':
+        packages = [p for p in packages if not p.startswith('python-')]
+        packages.extend(PY3_PACKAGES)
     return list(set(packages))
 
 
+def determine_purge_packages():
+    """
+    Determine list of packages that where previously installed which are no
+    longer needed.
+
+    :returns: list of package names
+    """
+    release = CompareOpenStackReleases(os_release('openstack-dashboard'))
+    if release >= 'rocky':
+        pkgs = [p for p in BASE_PACKAGES if p.startswith('python-')]
+        pkgs.extend([
+            'python-django-horizon',
+            'python-django-openstack-auth',
+            'python-pymysql',
+            'python-neutron-lbaas-dashboard',
+            'python-designate-dashboard',
+            'python-heat-dashboard',
+        ])
+        return pkgs
+    return []
+
+
 def do_openstack_upgrade(configs):
     """
     Perform an upgrade.  Takes care of upgrading packages, rewriting
@@ -278,6 +318,11 @@ def do_openstack_upgrade(configs):
     reset_os_release()
     apt_install(determine_packages(), fatal=True)
 
+    installed_pkgs = filter_missing_packages(determine_purge_packages())
+    if installed_pkgs:
+        apt_purge(installed_pkgs, fatal=True)
+        apt_autoremove(purge=True, fatal=True)
+
     # set CONFIGS to load templates from new release
     configs.set_release(openstack_release=new_os_rel)
 
@@ -415,12 +460,20 @@ def _pause_resume_helper(f, configs):
 
 
 def db_migration():
-    if cmp_pkgrevno('python-django', '1.9') >= 0:
+    release = CompareOpenStackReleases(os_release('openstack-dashboard'))
+    if release >= 'rocky':
+        python = 'python3'
+        python_django = 'python3-django'
+    else:
+        python = 'python2'
+        python_django = 'python-django'
+    if cmp_pkgrevno(python_django, '1.9') >= 0:
         # syncdb was removed in django 1.9
         subcommand = 'migrate'
     else:
         subcommand = 'syncdb'
-    cmd = ['/usr/share/openstack-dashboard/manage.py', subcommand, '--noinput']
+    cmd = [python, '/usr/share/openstack-dashboard/manage.py', subcommand,
+           '--noinput']
     subprocess.check_call(cmd)
 
 

tox.ini

@@ -64,7 +64,7 @@ basepython = python2.7
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 commands =
-    bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-queens --no-destroy
+    bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-rocky --no-destroy
 
 [testenv:func27-dfs]
 # Charm Functional Test

unit_tests/test_horizon_utils.py

@@ -41,10 +41,8 @@ class TestHorizonUtils(CharmTestCase):
     def setUp(self):
         super(TestHorizonUtils, self).setUp(horizon_utils, TO_PATCH)
 
-    @patch.object(horizon_utils, 'get_os_codename_install_source')
-    def test_determine_packages(self,
-                                _get_os_codename_install_source):
-        _get_os_codename_install_source.return_value = 'icehouse'
+    def test_determine_packages(self):
+        horizon_utils.os_release.return_value = 'icehouse'
         self.assertEqual(
             sorted(horizon_utils.determine_packages()),
             sorted([
@@ -56,11 +54,48 @@ class TestHorizonUtils(CharmTestCase):
                 'openstack-dashboard',
                 'memcached']))
 
-    @patch.object(horizon_utils, 'get_os_codename_install_source')
-    def test_determine_packages_mitaka(self, _get_os_codename_install_source):
-        _get_os_codename_install_source.return_value = 'mitaka'
+    def test_determine_packages_mitaka(self):
+        horizon_utils.os_release.return_value = 'mitaka'
         self.assertTrue('python-pymysql' in horizon_utils.determine_packages())
 
+    def test_determine_packages_queens(self):
+        horizon_utils.os_release.return_value = 'queens'
+        self.assertEqual(
+            sorted(horizon_utils.determine_packages()),
+            sorted(horizon_utils.BASE_PACKAGES +
+                   ['python-pymysql',
+                    'python-neutron-lbaas-dashboard',
+                    'python-designate-dashboard',
+                    'python-heat-dashboard']))
+
+    def test_determine_packages_rocky(self):
+        horizon_utils.os_release.return_value = 'rocky'
+        self.assertEqual(
+            sorted(horizon_utils.determine_packages()),
+            sorted([p for p in horizon_utils.BASE_PACKAGES
+                    if not p.startswith('python-')] +
+                   horizon_utils.PY3_PACKAGES)
+        )
+
+    def test_determine_purge_packages(self):
+        'Ensure no packages are identified for purge prior to rocky'
+        horizon_utils.os_release.return_value = 'queens'
+        self.assertEqual(horizon_utils.determine_purge_packages(), [])
+
+    def test_determine_purge_packages_rocky(self):
+        'Ensure python packages are identified for purge at rocky'
+        horizon_utils.os_release.return_value = 'rocky'
+        self.assertEqual(
+            horizon_utils.determine_purge_packages(),
+            [p for p in horizon_utils.BASE_PACKAGES
+             if p.startswith('python-')] +
+            ['python-django-horizon',
+             'python-django-openstack-auth',
+             'python-pymysql',
+             'python-neutron-lbaas-dashboard',
+             'python-designate-dashboard',
+             'python-heat-dashboard'])
+
     @patch('subprocess.call')
     def test_enable_ssl(self, _call):
         horizon_utils.enable_ssl()
@@ -104,6 +139,7 @@ class TestHorizonUtils(CharmTestCase):
     def test_do_openstack_upgrade(self, determine_packages):
         self.config.return_value = 'cloud:precise-havana'
         self.get_os_codename_install_source.return_value = 'havana'
+        horizon_utils.os_release.return_value = 'icehouse'
         configs = MagicMock()
         determine_packages.return_value = ['testpkg']
         horizon_utils.do_openstack_upgrade(configs)
@@ -234,15 +270,26 @@ class TestHorizonUtils(CharmTestCase):
     @patch('subprocess.check_call')
     def test_db_migration(self, mock_subprocess):
         self.cmp_pkgrevno.return_value = -1
+        horizon_utils.os_release.return_value = 'mitaka'
         horizon_utils.db_migration()
         mock_subprocess.assert_called_with(
-            ['/usr/share/openstack-dashboard/manage.py',
+            ['python2', '/usr/share/openstack-dashboard/manage.py',
              'syncdb', '--noinput'])
 
     @patch('subprocess.check_call')
-    def test_db_migration_bionic_and_beyond(self, mock_subprocess):
+    def test_db_migration_bionic_and_beyond_queens(self, mock_subprocess):
         self.cmp_pkgrevno.return_value = 0
+        horizon_utils.os_release.return_value = 'queens'
         horizon_utils.db_migration()
         mock_subprocess.assert_called_with(
-            ['/usr/share/openstack-dashboard/manage.py',
+            ['python2', '/usr/share/openstack-dashboard/manage.py',
+             'migrate', '--noinput'])
+
+    @patch('subprocess.check_call')
+    def test_db_migration_bionic_and_beyond_rocky(self, mock_subprocess):
+        self.cmp_pkgrevno.return_value = 0
+        horizon_utils.os_release.return_value = 'rocky'
+        horizon_utils.db_migration()
+        mock_subprocess.assert_called_with(
+            ['python3', '/usr/share/openstack-dashboard/manage.py',
              'migrate', '--noinput'])