Migrate charm to python3 only runtime

Change-Id: I921f3e2fc8250ca7250ca7dfc47e12644ce4ef98

.zuul.yaml

@@ -1,4 +1,3 @@
 - project:
     templates:
-      - python-charm-jobs
-      - openstack-python35-jobs-nonvoting
+      - python35-charm-jobs

charm-helpers-hooks.yaml

@@ -1,5 +1,5 @@
 repo: https://github.com/juju/charm-helpers
-destination: hooks/charmhelpers
+destination: charmhelpers
 include:
     - core
     - osplatform

hooks/cinder_hooks.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python3
 #
 # Copyright 2016 Canonical Ltd
 #
@@ -14,60 +14,72 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
 import sys
-import json
 import uuid
 
-from cinder_utils import (
-    register_configs,
-    restart_map,
-    scrub_old_style_ceph,
-    PACKAGES,
-    REQUIRED_INTERFACES,
-    VERSION_PACKAGE,
-    CEPH_CONF,
-)
-from cinder_contexts import (
-    CephSubordinateContext,
-    ceph_config_file,
-)
-from charmhelpers.contrib.openstack.context import CephContext
 
+_path = os.path.dirname(os.path.realpath(__file__))
+_root = os.path.abspath(os.path.join(_path, '..'))
+
+
+def _add_path(path):
+    if path not in sys.path:
+        sys.path.insert(1, path)
+
+
+_add_path(_root)
+
+from charmhelpers.contrib.openstack.alternatives import remove_alternative
+from charmhelpers.contrib.openstack.context import CephContext
+from charmhelpers.contrib.openstack.utils import (
+    clear_unit_paused,
+    clear_unit_upgrading,
+    os_application_version_set,
+    set_os_workload_status,
+    set_unit_paused,
+    set_unit_upgrading,
+)
+from charmhelpers.contrib.storage.linux.ceph import (
+    CephBrokerRq,
+    delete_keyring,
+    ensure_ceph_keyring,
+    is_request_complete,
+    send_request_if_needed,
+)
 from charmhelpers.core.hookenv import (
-    Hooks,
-    UnregisteredHookError,
     config,
-    service_name,
-    relation_set,
-    relation_ids,
-    status_set,
-    log,
+    Hooks,
+    is_leader,
     leader_get,
     leader_set,
-    is_leader,
+    log,
+    relation_ids,
+    relation_set,
+    service_name,
+    status_set,
+    UnregisteredHookError,
 )
-from charmhelpers.fetch import apt_install, apt_update
 from charmhelpers.core.host import (
     restart_on_change,
     service_restart,
 )
-from charmhelpers.contrib.openstack.alternatives import remove_alternative
-from charmhelpers.contrib.storage.linux.ceph import (
-    send_request_if_needed,
-    is_request_complete,
-    ensure_ceph_keyring,
-    CephBrokerRq,
-    delete_keyring,
-)
+from charmhelpers.fetch import apt_install, apt_update
 from charmhelpers.payload.execd import execd_preinstall
-from charmhelpers.contrib.openstack.utils import (
-    set_os_workload_status,
-    os_application_version_set,
-    set_unit_paused,
-    set_unit_upgrading,
-    clear_unit_paused,
-    clear_unit_upgrading,
+
+from cinder_contexts import (
+    ceph_config_file,
+    CephSubordinateContext,
+)
+from cinder_utils import (
+    CEPH_CONF,
+    PACKAGES,
+    register_configs,
+    REQUIRED_INTERFACES,
+    restart_map,
+    scrub_old_style_ceph,
+    VERSION_PACKAGE,
 )
 
 

hooks/cinder_utils.py

@@ -12,31 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
-
 import os
 import re
-
 from collections import OrderedDict
 from tempfile import NamedTemporaryFile
 
-
+from charmhelpers.contrib.openstack import (
+    context,
+    templating,
+)
+from charmhelpers.contrib.openstack.alternatives import install_alternative
+from charmhelpers.contrib.openstack.utils import get_os_codename_package
 from charmhelpers.core.hookenv import (
+    hook_name,
     relation_ids,
     service_name,
-    hook_name,
 )
-
-from charmhelpers.contrib.openstack import (
-    templating,
-    context,
-)
-
-from charmhelpers.contrib.openstack.utils import (
-    get_os_codename_package,
-)
-
-from charmhelpers.contrib.openstack.alternatives import install_alternative
 from charmhelpers.core.host import mkdir
 
 
@@ -89,7 +80,7 @@ def register_configs():
         # lower priority that both of these but thats OK
         if not os.path.exists(ceph_config_file()):
             # touch file for pre-templated generation
-            open(ceph_config_file(), 'w').close()
+            open(ceph_config_file(), 'wt').close()
         install_alternative(os.path.basename(CEPH_CONF),
                             CEPH_CONF, ceph_config_file())
         CONFIG_FILES[ceph_config_file()] = {
@@ -113,7 +104,7 @@ def restart_map():
                     that should be restarted when file changes.
     '''
     _map = []
-    for f, ctxt in CONFIG_FILES.iteritems():
+    for f, ctxt in CONFIG_FILES.items():
         svcs = []
         for svc in ctxt['services']:
             svcs.append(svc)
@@ -130,11 +121,11 @@ def scrub_old_style_ceph():
     # NOTE: purge any CEPH_ARGS data from /etc/environment
     env_file = '/etc/environment'
     ceph_match = re.compile("^CEPH_ARGS.*").search
-    with open(env_file, 'r') as input_file:
-        with NamedTemporaryFile(mode='w',
+    with open(env_file, 'rt') as input_file:
+        with NamedTemporaryFile(mode='wt',
                                 delete=False,
                                 dir=os.path.dirname(env_file)) as outfile:
             for line in input_file:
                 if not ceph_match(line):
                     print(line, end='', file=outfile)
-            os.rename(outfile.name, input_file.name)
+    os.rename(outfile.name, input_file.name)

hooks/install

@@ -11,7 +11,7 @@ check_and_install() {
     fi
 }
 
-PYTHON="python"
+PYTHON="python3"
 
 for dep in ${DEPS[@]}; do
     check_and_install ${PYTHON} ${dep}

tests/basic_deployment.py

@@ -730,6 +730,12 @@ class CinderCephBasicDeployment(OpenStackAmuletDeployment):
     def test_500_ceph_alternatives_cleanup(self):
         """Check ceph alternatives are removed when ceph-mon
         relation is broken"""
+        # Skip this test if release is less than xenial_ocata as in that case
+        # cinder HAS a relation with ceph directly and this test would fail
+        if self._get_openstack_release() < self.xenial_ocata:
+            u.log.debug("No checking 500 ceph alternatives as "
+                        "/etc/ceph/ceph.conf will exist.")
+            return
         u.log.debug('Checking ceph alternatives are removed '
                     'upon broken ceph-mon relation')
         ceph_dir = self.cinder_ceph_sentry.directory_listing('/etc/ceph')

tox.ini

@@ -2,7 +2,7 @@
 # This file is managed centrally by release-tools and should not be modified
 # within individual charm repos.
 [tox]
-envlist = pep8,py27
+envlist = pep8,py3{5,6}
 skipsdist = True
 
 [testenv]
@@ -20,6 +20,7 @@ passenv = HOME TERM AMULET_* CS_API_*
 basepython = python2.7
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
+commands = /bin/true
 
 [testenv:py35]
 basepython = python3.5

unit_tests/__init__.py

@@ -12,5 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import sys
-sys.path.append('hooks')
+
+_path = os.path.dirname(os.path.realpath(__file__))
+_actions = os.path.abspath(os.path.join(_path, '../actions'))
+_hooks = os.path.abspath(os.path.join(_path, '../hooks'))
+_charmhelpers = os.path.abspath(os.path.join(_path, '../charmhelpers'))
+_unit_tests = os.path.abspath(os.path.join(_path, '../unit_tests'))
+
+
+def _add_path(path):
+    if path not in sys.path:
+        sys.path.insert(1, path)
+
+_add_path(_actions)
+_add_path(_hooks)
+_add_path(_charmhelpers)
+_add_path(_unit_tests)

unit_tests/test_cinder_hooks.py

@@ -168,12 +168,13 @@ class TestCinderHooks(CharmTestCase):
     def test_ceph_broken(self, mock_config):
         self.CONFIGS.complete_contexts.return_value = ['ceph']
         self.service_name.return_value = 'cinder-ceph'
-        hooks.hooks.execute(['hooks/ceph-relation-changed'])
-        hooks.hooks.execute(['hooks/ceph-relation-broken'])
+        with patch.object(hooks, 'CEPH_CONF', new="/some/random/file"):
+            hooks.hooks.execute(['hooks/ceph-relation-changed'])
+            hooks.hooks.execute(['hooks/ceph-relation-broken'])
         self.delete_keyring.assert_called_with(service='cinder-ceph')
         self.assertTrue(self.CONFIGS.write_all.called)
         self.remove_alternative.assert_called_with(
-            os.path.basename(self.CEPH_CONF),
+            os.path.basename("/some/random/file"),
             self.ceph_config_file())
 
     @patch('charmhelpers.core.hookenv.config')

unit_tests/test_utils.py

@@ -49,7 +49,7 @@ def get_default_config():
     '''
     default_config = {}
     config = load_config()
-    for k, v in config.iteritems():
+    for k, v in config.items():
         if 'default' in v:
             default_config[k] = v['default']
         else: