Convert charm to Python 3

Change-Id: Ib7cc06b3b42f26f725a9ea79f09189cc72952d29
This commit is contained in:
Alex Kavanagh 2019-03-13 18:18:51 +00:00
parent 94a35649c4
commit 02b406b6f3
82 changed files with 139 additions and 92 deletions

View File

@ -1,4 +1,3 @@
- project:
templates:
- python-charm-jobs
- openstack-python35-jobs-nonvoting
- python35-charm-jobs

View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python3
#
# Copyright 2016 Canonical Ltd
#
@ -14,11 +14,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import os
import subprocess
import sys
import traceback
sys.path.append('hooks/')
_path = os.path.dirname(os.path.realpath(__file__))
_hooks = os.path.abspath(os.path.join(_path, '../hooks'))
_root = os.path.abspath(os.path.join(_path, '..'))
def _add_path(path):
if path not in sys.path:
sys.path.insert(1, path)
_add_path(_hooks)
_add_path(_root)
from charmhelpers.core.hookenv import (
action_fail,
action_get,
@ -50,7 +65,7 @@ def status(args):
cmd = ['crm', 'status', '--inactive']
try:
result = subprocess.check_output(cmd)
result = subprocess.check_output(cmd).decode('utf-8')
action_set({'result': result})
except subprocess.CalledProcessError as e:
log("ERROR: Failed call to crm resource status. "

View File

@ -1,5 +1,5 @@
repo: https://github.com/juju/charm-helpers
destination: hooks/charmhelpers
destination: charmhelpers
include:
- core
- cli

View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python3
#
# Copyright 2016 Canonical Ltd
#
@ -20,6 +20,18 @@ import shutil
import socket
import sys
_path = os.path.dirname(os.path.realpath(__file__))
_root = os.path.abspath(os.path.join(_path, '..'))
def _add_path(path):
if path not in sys.path:
sys.path.insert(1, path)
_add_path(_root)
import pcmk
from charmhelpers.core.hookenv import (
@ -200,7 +212,7 @@ def migrate_maas_dns():
write_maas_dns_address(resource, res_ipaddr)
@hooks.hook()
@hooks.hook('upgrade-charm.real')
def upgrade_charm():
install()
migrate_maas_dns()
@ -269,10 +281,10 @@ def ha_relation_changed():
return
if True in [ra.startswith('ocf:openstack')
for ra in resources.itervalues()]:
for ra in resources.values()]:
apt_install('openstack-resource-agents')
if True in [ra.startswith('ocf:ceph')
for ra in resources.itervalues()]:
for ra in resources.values()]:
apt_install('ceph-resource-agents')
if True in [ra.startswith('ocf:maas')
@ -328,7 +340,7 @@ def ha_relation_changed():
pcmk.commit('crm -w -F configure delete %s' % res_name)
log('Configuring Resources: %s' % (resources), level=DEBUG)
for res_name, res_type in resources.iteritems():
for res_name, res_type in resources.items():
# disable the service we are going to put in HA
if res_type.split(':')[0] == "lsb":
disable_lsb_services(res_type.split(':')[1])
@ -367,7 +379,7 @@ def ha_relation_changed():
raise Exception(msg)
log('Configuring Groups: %s' % (groups), level=DEBUG)
for grp_name, grp_params in groups.iteritems():
for grp_name, grp_params in groups.items():
if not pcmk.crm_opt_exists(grp_name):
cmd = ('crm -w -F configure group %s %s' %
(grp_name, grp_params))
@ -375,14 +387,14 @@ def ha_relation_changed():
log('%s' % cmd, level=DEBUG)
log('Configuring Master/Slave (ms): %s' % (ms), level=DEBUG)
for ms_name, ms_params in ms.iteritems():
for ms_name, ms_params in ms.items():
if not pcmk.crm_opt_exists(ms_name):
cmd = 'crm -w -F configure ms %s %s' % (ms_name, ms_params)
pcmk.commit(cmd)
log('%s' % cmd, level=DEBUG)
log('Configuring Orders: %s' % (orders), level=DEBUG)
for ord_name, ord_params in orders.iteritems():
for ord_name, ord_params in orders.items():
if not pcmk.crm_opt_exists(ord_name):
cmd = 'crm -w -F configure order %s %s' % (ord_name,
ord_params)
@ -390,7 +402,7 @@ def ha_relation_changed():
log('%s' % cmd, level=DEBUG)
log('Configuring Clones: %s' % clones, level=DEBUG)
for cln_name, cln_params in clones.iteritems():
for cln_name, cln_params in clones.items():
if not pcmk.crm_opt_exists(cln_name):
cmd = 'crm -w -F configure clone %s %s' % (cln_name,
cln_params)
@ -402,7 +414,7 @@ def ha_relation_changed():
# need to exist otherwise constraint creation will fail.
log('Configuring Colocations: %s' % colocations, level=DEBUG)
for col_name, col_params in colocations.iteritems():
for col_name, col_params in colocations.items():
if not pcmk.crm_opt_exists(col_name):
cmd = 'crm -w -F configure colocation %s %s' % (col_name,
col_params)
@ -410,14 +422,14 @@ def ha_relation_changed():
log('%s' % cmd, level=DEBUG)
log('Configuring Locations: %s' % locations, level=DEBUG)
for loc_name, loc_params in locations.iteritems():
for loc_name, loc_params in locations.items():
if not pcmk.crm_opt_exists(loc_name):
cmd = 'crm -w -F configure location %s %s' % (loc_name,
loc_params)
pcmk.commit(cmd)
log('%s' % cmd, level=DEBUG)
for res_name, res_type in resources.iteritems():
for res_name, res_type in resources.items():
if len(init_services) != 0 and res_name in init_services:
# Checks that the resources are running and started.
# Ensure that clones are excluded as the resource is

View File

@ -1,20 +1,4 @@
#!/bin/bash -e
# Wrapper to deal with newer Ubuntu versions that don't have py2 installed
# by default.
declare -a DEPS=('apt' 'netaddr' 'netifaces' 'pip' 'yaml' 'dnspython')
check_and_install() {
pkg="${1}-${2}"
if ! dpkg -s ${pkg} 2>&1 > /dev/null; then
apt-get -y install ${pkg}
fi
}
PYTHON="python"
for dep in ${DEPS[@]}; do
check_and_install ${PYTHON} ${dep}
done
./hooks/install_deps
exec ./hooks/install.real

18
hooks/install_deps Executable file
View File

@ -0,0 +1,18 @@
#!/bin/bash -e
# Wrapper to deal with newer Ubuntu versions that don't have py2 installed
# by default.
declare -a DEPS=('apt' 'netaddr' 'netifaces' 'pip' 'yaml' 'dnspython')
check_and_install() {
pkg="${1}-${2}"
if ! dpkg -s ${pkg} 2>&1 > /dev/null; then
apt-get -y install ${pkg}
fi
}
PYTHON="python3"
for dep in ${DEPS[@]}; do
check_and_install ${PYTHON} ${dep}
done

View File

@ -72,7 +72,7 @@ class MAASHelper(object):
try:
cmd = ['maas-cli', MAAS_PROFILE_NAME, 'nodes', 'list']
out = subprocess.check_output(cmd)
out = subprocess.check_output(cmd).decode('utf-8')
except subprocess.CalledProcessError:
log('Could not get node inventory from MAAS.', ERROR)
return False

View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import commands
import hashlib
import re
import subprocess
@ -22,7 +21,7 @@ import time
import xml.etree.ElementTree as etree
from distutils.version import StrictVersion
from StringIO import StringIO
from io import StringIO
from charmhelpers.core import unitdata
from charmhelpers.core.hookenv import (
log,
@ -47,7 +46,7 @@ def wait_for_pcmk(retries=12, sleep=10):
for i in range(retries):
if crm_up:
return True
output = commands.getstatusoutput("crm node list")[1]
output = subprocess.getstatusoutput("crm node list")[1]
crm_up = hostname in output
time.sleep(sleep)
if not crm_up:
@ -61,7 +60,7 @@ def commit(cmd):
def is_resource_present(resource):
status = commands.getstatusoutput("crm resource status %s" % resource)[0]
status = subprocess.getstatusoutput("crm resource status %s" % resource)[0]
if status != 0:
return False
@ -87,7 +86,7 @@ def online(node=None):
def crm_opt_exists(opt_name):
output = commands.getstatusoutput("crm configure show")[1]
output = subprocess.getstatusoutput("crm configure show")[1]
if opt_name in output:
return True
@ -95,7 +94,8 @@ def crm_opt_exists(opt_name):
def crm_res_running(opt_name):
(_, output) = commands.getstatusoutput("crm resource status %s" % opt_name)
(_, output) = subprocess.getstatusoutput(
"crm resource status %s" % opt_name)
if output.startswith("resource %s is running" % opt_name):
return True
@ -104,7 +104,7 @@ def crm_res_running(opt_name):
def list_nodes():
cmd = ['crm', 'node', 'list']
out = subprocess.check_output(cmd)
out = subprocess.check_output(cmd).decode('utf-8')
nodes = []
for line in str(out).split('\n'):
if line != '':
@ -185,18 +185,19 @@ def get_property(name):
# crmsh >= 2.3 renamed show-property to get-property, 2.3.x is
# available since zesty
if crm_version() >= StrictVersion('2.3.0'):
output = subprocess.check_output(['crm', 'configure',
'get-property', name],
universal_newlines=True)
output = subprocess.check_output(
['crm', 'configure', 'get-property', name],
universal_newlines=True)
elif crm_version() < StrictVersion('2.2.0'):
# before 2.2.0 there is no method to get a property
output = subprocess.check_output(['crm', 'configure', 'show', 'xml'],
universal_newlines=True)
return get_property_from_xml(name, output)
else:
output = subprocess.check_output(['crm', 'configure',
'show-property', name],
universal_newlines=True)
output = subprocess .check_output(
['crm', 'configure', 'show-property', name],
universal_newlines=True)
return output
@ -207,9 +208,9 @@ def set_property(name, value):
:param name: property name
:param value: new value
"""
subprocess.check_output(['crm', 'configure',
'property', '%s=%s' % (name, value)],
universal_newlines=True)
subprocess.check_call(['crm', 'configure',
'property', '%s=%s' % (name, value)],
universal_newlines=True)
def crm_version():
@ -218,11 +219,8 @@ def crm_version():
"""
ver = subprocess.check_output(['crm', '--version'],
universal_newlines=True)
r = re.compile(r'.*(\d\.\d\.\d).*')
matched = r.match(ver)
if not matched:
raise ValueError('error parsin crm version: %s' % ver)
else:
@ -245,12 +243,12 @@ def crm_update_resource(res_name, res_type, res_params=None, force=False):
return 0
with tempfile.NamedTemporaryFile() as f:
f.write('primitive {} {}'.format(res_name, res_type))
f.write('primitive {} {}'.format(res_name, res_type).encode('ascii'))
if res_params:
f.write(' \\\n\t{}'.format(res_params))
f.write(' \\\n\t{}'.format(res_params).encode('ascii'))
else:
f.write('\n')
f.write('\n'.encode('ascii'))
f.flush()
f.seek(0)
@ -281,6 +279,7 @@ def resource_checksum(res_name, res_type, res_params=None):
"""
m = hashlib.md5()
m.update(res_type)
m.update(res_params)
m.update(res_type.encode('utf-8'))
if res_params is not None:
m.update(res_params.encode('utf-8'))
return m.hexdigest()

View File

@ -1 +0,0 @@
hooks.py

7
hooks/upgrade-charm Normal file
View File

@ -0,0 +1,7 @@
#!/bin/bash
# Wrapper to ensure that old python bytecode isn't hanging around
# after we upgrade the charm with newer libraries
rm -rf **/*.pyc
./hooks/install_deps
exec ./hooks/upgrade-charm.real

1
hooks/upgrade-charm.real Symbolic link
View File

@ -0,0 +1 @@
hooks.py

View File

@ -124,7 +124,7 @@ class MAASConfigIncomplete(Exception):
def disable_upstart_services(*services):
for service in services:
with open("/etc/init/{}.override".format(service), "w") as override:
with open("/etc/init/{}.override".format(service), "wt") as override:
override.write("manual")
@ -214,7 +214,7 @@ def get_corosync_id(unit_name):
def nulls(data):
"""Returns keys of values that are null (but not bool)"""
return [k for k in data.iterkeys()
return [k for k in data.keys()
if not isinstance(data[k], bool) and not data[k]]
@ -295,7 +295,7 @@ def get_corosync_conf():
return conf
missing = [k for k, v in conf.iteritems() if v is None]
missing = [k for k, v in conf.items() if v is None]
log('Missing required configuration: %s' % missing)
return None
@ -404,12 +404,12 @@ def get_ipv6_addr():
for rid in relation_ids('ha'):
for unit in related_units(rid):
resources = parse_data(rid, unit, 'resources')
for res in resources.itervalues():
for res in resources.values():
if 'ocf:heartbeat:IPv6addr' in res:
res_params = parse_data(rid, unit, 'resource_params')
res_p = res_params.get(res)
if res_p:
for k, v in res_p.itervalues():
for k, v in res_p.values():
if utils.is_ipv6(v):
log("Excluding '%s' from address list" % v,
level=DEBUG)
@ -766,7 +766,9 @@ def is_in_standby_mode(node_name):
@param node_name: The name of the node to check
@returns boolean - True if node_name is in standby mode
"""
out = subprocess.check_output(['crm', 'node', 'status', node_name])
out = (subprocess
.check_output(['crm', 'node', 'status', node_name])
.decode('utf-8'))
root = ET.fromstring(out)
standby_mode = False
@ -807,7 +809,7 @@ def node_has_resources(node_name):
@param node_name: The name of the node to check
@returns boolean - True if node_name has resources
"""
out = subprocess.check_output(['crm_mon', '-X'])
out = subprocess.check_output(['crm_mon', '-X']).decode('utf-8')
root = ET.fromstring(out)
has_resources = False
for resource in root.iter('resource'):
@ -936,7 +938,7 @@ def ocf_file_exists(res_name, resources,
@return: boolean - True if the ocf resource exists
"""
res_type = None
for key, val in resources.iteritems():
for key, val in resources.items():
if res_name == key:
if len(val.split(':')) > 2:
res_type = val.split(':')[1]
@ -955,7 +957,7 @@ def kill_legacy_ocf_daemon_process(res_name):
ocf_name = res_name.replace('res_', '').replace('_', '-')
reg_expr = '([0-9]+)\s+[^0-9]+{}'.format(ocf_name)
cmd = ['ps', '-eo', 'pid,cmd']
ps = subprocess.check_output(cmd)
ps = subprocess.check_output(cmd).decode('utf-8')
res = re.search(reg_expr, ps, re.MULTILINE)
if res:
pid = res.group(1)

View File

@ -1,4 +1,4 @@
#!/usr/bin/python3
#!/usr/bin/env python3
#
# Copyright 2016 Canonical Ltd
#

View File

@ -69,7 +69,7 @@ quorum {
{% if transport == "udpu" %}
nodelist {
{% for nodeid, ip in ha_nodes.iteritems() %}
{% for nodeid, ip in ha_nodes.items() %}
node {
ring0_addr: {{ ip }}
nodeid: {{ nodeid }}

View File

@ -2,7 +2,7 @@
# This file is managed centrally by release-tools and should not be modified
# within individual charm repos.
[tox]
envlist = pep8,py27
envlist = pep8,py3{5,6}
skipsdist = True
[testenv]
@ -16,11 +16,6 @@ commands = stestr run {posargs}
whitelist_externals = juju
passenv = HOME TERM AMULET_* CS_API_*
[testenv:py27]
basepython = python2.7
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
[testenv:py35]
basepython = python3.5
deps = -r{toxinidir}/requirements.txt

View File

@ -12,5 +12,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import sys
sys.path.append('hooks')
_path = os.path.dirname(os.path.realpath(__file__))
_actions = os.path.abspath(os.path.join(_path, '../actions'))
_hooks = os.path.abspath(os.path.join(_path, '../hooks'))
_charmhelpers = os.path.abspath(os.path.join(_path, '../charmhelpers'))
_unit_tests = os.path.abspath(os.path.join(_path, '../unit_tests'))
def _add_path(path):
if path not in sys.path:
sys.path.insert(1, path)
_add_path(_actions)
_add_path(_hooks)
_add_path(_charmhelpers)
_add_path(_unit_tests)

View File

@ -26,7 +26,7 @@ import pcmk
def write_file(path, content, *args, **kwargs):
with open(path, 'w') as f:
with open(path, 'wt') as f:
f.write(content)
f.flush()
@ -74,7 +74,7 @@ class UtilsTestCaseWriteTmp(unittest.TestCase):
self.assertTrue(utils.emit_corosync_conf())
with open(utils.COROSYNC_CONF) as fd:
with open(utils.COROSYNC_CONF, 'rt') as fd:
content = fd.read()
if enabled:
pattern = 'debug: on\n'
@ -216,7 +216,7 @@ class UtilsTestCase(unittest.TestCase):
@mock.patch.object(subprocess, 'call')
def test_kill_legacy_ocf_daemon_process(self, call_mock,
check_output_mock):
ps_output = '''
ps_output = b'''
PID CMD
6863 sshd: ubuntu@pts/7
11109 /usr/bin/python /usr/bin/ceilometer-agent-central --config

View File

@ -82,25 +82,25 @@ class TestPcmk(unittest.TestCase):
def tearDown(self):
os.remove(self.tmpfile.name)
@mock.patch('commands.getstatusoutput')
@mock.patch('subprocess.getstatusoutput')
def test_crm_res_running_true(self, getstatusoutput):
getstatusoutput.return_value = (0, ("resource res_nova_consoleauth is "
"running on: juju-xxx-machine-6"))
self.assertTrue(pcmk.crm_res_running('res_nova_consoleauth'))
@mock.patch('commands.getstatusoutput')
@mock.patch('subprocess.getstatusoutput')
def test_crm_res_running_stopped(self, getstatusoutput):
getstatusoutput.return_value = (0, ("resource res_nova_consoleauth is "
"NOT running"))
self.assertFalse(pcmk.crm_res_running('res_nova_consoleauth'))
@mock.patch('commands.getstatusoutput')
@mock.patch('subprocess.getstatusoutput')
def test_crm_res_running_undefined(self, getstatusoutput):
getstatusoutput.return_value = (1, "foobar")
self.assertFalse(pcmk.crm_res_running('res_nova_consoleauth'))
@mock.patch('socket.gethostname')
@mock.patch('commands.getstatusoutput')
@mock.patch('subprocess.getstatusoutput')
def test_wait_for_pcmk(self, getstatusoutput, gethostname):
# Pacemaker is down
gethostname.return_value = 'hanode-1'
@ -124,8 +124,8 @@ class TestPcmk(unittest.TestCase):
# trusty
mock_check_output.mock_reset()
mock_check_output.return_value = ("1.2.5 (Build f2f315daf6a5fd7ddea8e5"
"64cd289aa04218427d)\n")
mock_check_output.return_value = (
"1.2.5 (Build f2f315daf6a5fd7ddea8e564cd289aa04218427d)\n")
ret = pcmk.crm_version()
self.assertEqual(StrictVersion('1.2.5'), ret)
mock_check_output.assert_called_with(['crm', '--version'],
@ -170,7 +170,7 @@ class TestPcmk(unittest.TestCase):
'show', 'xml'],
universal_newlines=True)
@mock.patch('subprocess.check_output')
@mock.patch('subprocess.check_call')
def test_set_property(self, mock_check_output):
pcmk.set_property('maintenance-mode', 'false')
mock_check_output.assert_called_with(['crm', 'configure', 'property',
@ -189,7 +189,7 @@ class TestPcmk(unittest.TestCase):
mock_call.assert_any_call(['crm', 'configure', 'load',
'update', self.tmpfile.name])
with open(self.tmpfile.name, 'r') as f:
with open(self.tmpfile.name, 'rt') as f:
self.assertEqual(f.read(),
('primitive res_test IPaddr2 \\\n'
'\tparams ip=1.2.3.4 cidr_netmask=255.255.0.0'))

View File

@ -51,7 +51,7 @@ def get_default_config():
'''
default_config = {}
config = load_config()
for k, v in config.iteritems():
for k, v in config.items():
if 'default' in v:
default_config[k] = v['default']
else:
@ -148,5 +148,5 @@ def patch_open():
mock_open(*args, **kwargs)
yield mock_file
with patch('__builtin__.open', stub_open):
with patch('builtins.open', stub_open):
yield mock_open, mock_file