x
/
kolla-mesos
Code Issues Proposed changes

Fix python 3 unit tests 

- all zookeeper node writes must have string encoded and reads decoded
- list(zip())
- file mocking

Change-Id: I0e858c179cac587ee965aa57c0245050c41dc51a
This commit is contained in:
Angus Salkeld 2016-03-21 10:13:38 +10:00
parent 625e5988ba
commit 11e0f47c57
12 changed files with 86 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
kolla_mesos/commands.py
View File

@ -118,15 +118,15 @@ def get_status(tasks):
task1: {
'register': (register_path, reg_status)
'requirements': {
reqt1_path: reqt_status
reqt2_path: reqt_status
reqt1_path: req_status
reqt2_path: req_status
...
}
}
Where:
reg_status = 'done', 'running', 'waiting'
reqt_status = '', 'done'
req_status = '', 'done'
"""
status = {}
with zk_utils.connection() as zk:
@ -135,20 +135,20 @@ def get_status(tasks):
status[task] = {}
status[task]['requirements'] = {}
for path in info['requires']:
reqt_status = ''
req_status = ''.encode('utf-8')
if zk.exists(path):
reqt_status, _ = zk.get(path)
status[task]['requirements'][path] = reqt_status
req_status, _ = zk.get(path)
status[task]['requirements'][path] = req_status.decode('utf-8')
# get status of registrations
for task, info in tasks.items():
status[task]['register'] = {}
reg_path = info['register']
reg_status = ''
reg_status = ''.encode('utf-8')
if zk.exists(reg_path):
reg_status, _ = zk.get(reg_path)
status[task]['register'] = (reg_path, reg_status)
status[task]['register'] = (reg_path, reg_status.decode('utf-8'))
return status

2
kolla_mesos/common/cli_utils.py
View File

@ -44,4 +44,4 @@ def dict2columns(data, id_col=None):
items = [(key, data[key]) for key in keys]
else:
items = sorted(data.items())
return zip(*items)
return list(zip(*items))

6
kolla_mesos/common/zk_utils.py
View File

@ -33,7 +33,7 @@ def _list_all(path, zk):
values = {}
data, stat = zk.get(path)
if stat.dataLength > 0:
values[path] = data
values[path] = data.decode('utf-8')
try:
children = zk.get_children(path)
except exceptions.NoNodeError:
@ -55,12 +55,12 @@ def list_all(path):
def get_one(path):
with connection() as zk:
data, stat = zk.get(path)
return {path: data}
return {path: data.decode('utf-8')}
def set_one(path, value):
with connection() as zk:
zk.set(path, value)
zk.set(path, value.encode('utf-8'))
@contextlib.contextmanager

6
kolla_mesos/configuration.py
View File

@ -49,7 +49,9 @@ def write_variables_zookeeper(zk, variables, base_node=None, overwrite=True):
var_path)
return
zk.ensure_path(var_path)
zk.set(var_path, "" if var_value is None else var_value)
if var_value is None:
var_value = ''
zk.set(var_path, var_value.encode('utf-8'))
LOG.debug('Updated "%s" node in zookeeper.' % var_path)
@ -86,7 +88,7 @@ def write_common_config_to_zookeeper(config_dir, zk, jinja_vars,
src_file = file_utils.find_file(source_path)
with open(src_file) as fp:
content = fp.read()
zk.set(script_node, content)
zk.set(script_node, content.encode('utf-8'))
def get_variables_from_zookeeper(zk, needed_variables):

20
kolla_mesos/container_scripts/start.py
View File

@ -288,7 +288,7 @@ def write_file(conf, data):
perm = int(conf.get('perm', 0))
with tempfile.NamedTemporaryFile(prefix='kolla-mesos',
delete=False) as tf:
tf.write(data)
tf.write(data.encode('utf-8'))
tf.flush()
tf_name = tf.name
@ -334,10 +334,10 @@ def render_template(zk, templ, variables, var_names):
value = ''
LOG.warning('missing required variable value %s', var)
except kz_exceptions.NoNodeError:
value = ''
value = ''.encode('utf-8')
LOG.error('missing required variable %s', var)
variables[var] = value.encode('utf-8')
variables[var] = value.decode('utf-8')
return jinja_render(templ, variables)
@ -420,20 +420,20 @@ class Command(object):
for check_path in self.check_paths:
self.zk.retry(self.zk.ensure_path, check_path)
current_state, _ = self.zk.get(check_path)
if current_state != state:
if current_state.decode('utf-8') != state:
LOG.info('path: %s, changing state from %s to %s'
% (check_path, current_state, state))
self.zk.set(check_path, state)
self.zk.set(check_path, state.encode('utf-8'))
def get_state(self, path=None):
if not path:
path = self.check_paths[0]
state = None
if self.zk.exists(path):
state, _ = self.zk.get(str(path))
state, _ = self.zk.get(path)
if not state:
state = None
return state
return None
return state.decode('utf-8')
def sleep(self, queue_size, retry=False):
seconds = math.ceil(20 / (1.0 + queue_size))
@ -505,7 +505,7 @@ class Command(object):
continue
raw_content, stat = self.zk.get(os.path.join(SERVICE, 'files',
name))
templ = raw_content.encode('utf-8')
templ = raw_content.decode('utf-8')
var_names = jinja_find_required_variables(templ, name)
if not var_names:
# not a template, doesn't need rendering.
@ -623,7 +623,7 @@ def main():
LOG.info('starting')
with zk_connection(ZK_HOSTS) as zk:
service_conf_raw, stat = zk.get(SERVICE)
service_conf = json.loads(service_conf_raw)
service_conf = json.loads(service_conf_raw.decode('utf-8'))
# don't join a Party if this container is not running a daemon
# process.

10
kolla_mesos/service.py
View File

@ -73,7 +73,7 @@ class File(object):
src_file = file_utils.find_file(src_file)
with open(src_file) as fp:
content = fp.read()
zk.set(dest_node, content)
zk.set(dest_node, content.encode('utf-8'))
class Command(object):
@ -125,7 +125,8 @@ class Runner(object):
raise exception.KollaNotFoundException(
service_name, entity='running service definition')
return Runner(yaml.load(
jinja_utils.jinja_render_str(conf_raw, variables)))
jinja_utils.jinja_render_str(conf_raw.decode('utf-8'),
variables)))
@classmethod
def load_from_file(cls, service_file, variables):
@ -148,7 +149,7 @@ class Runner(object):
dest_node = os.path.join(base_node, self._conf['name'])
zk.ensure_path(dest_node)
try:
zk.set(dest_node, json.dumps(self._conf))
zk.set(dest_node, json.dumps(self._conf).encode('utf-8'))
except Exception as te:
LOG.error('%s=%s -> %s' % (dest_node, self._conf, te))
@ -444,7 +445,8 @@ def _load_variables_from_zk(zk):
except exceptions.NoNodeError:
var_names = []
for var in var_names:
variables[str(var)], _stat = zk.get(os.path.join(path, var))
value, _stat = zk.get(os.path.join(path, var))
variables[var] = value.decode('utf-8')
# Add deployment_id
variables.update({'deployment_id': CONF.kolla.deployment_id})
# override node_config_directory to empty

10
kolla_mesos/tests/common/test_zk_utils.py
View File

@ -28,9 +28,9 @@ class TestConfig(base.BaseTestCase):
def test_list_all(self):
self.client.create('/kolla/t1/status/q/x',
'val-1', makepath=True)
'val-1'.encode('utf-8'), makepath=True)
self.client.create('/kolla/t1/variables/x',
'val-2', makepath=True)
'val-2'.encode('utf-8'), makepath=True)
with mock.patch.object(zk_utils,
'connection') as m_zk_c:
@ -41,7 +41,7 @@ class TestConfig(base.BaseTestCase):
def test_get_one(self):
self.client.create('/kolla/t1/variables/x',
'val', makepath=True)
'val'.encode('utf-8'), makepath=True)
with mock.patch.object(zk_utils,
'connection') as m_zk_c:
@ -51,11 +51,11 @@ class TestConfig(base.BaseTestCase):
def test_set_one(self):
self.client.create('/kolla/t1/variables/x',
'old', makepath=True)
'old'.encode('utf-8'), makepath=True)
with mock.patch.object(zk_utils,
'connection') as m_zk_c:
m_zk_c.return_value.__enter__.return_value = self.client
zk_utils.set_one('/kolla/t1/variables/x', 'new')
val, _st = self.client.get('/kolla/t1/variables/x')
self.assertEqual('new', val)
self.assertEqual('new', val.decode('utf-8'))

35
kolla_mesos/tests/container_scripts/test_start.py
View File

@ -16,6 +16,7 @@ from kazoo.recipe import party
import logging
import mock
import os.path
import six
import sys
from zake import fake_client
@ -86,7 +87,7 @@ class CommandTest(base.BaseTestCase):
self.client)
self.client.create('/kolla/t1/status/q/x',
'done', makepath=True)
'done'.encode('utf-8'), makepath=True)
self.assertFalse(cmd1.requirements_fulfilled())
@ -103,9 +104,9 @@ class CommandTest(base.BaseTestCase):
self.client)
self.client.create('/kolla/t1/status/global/w/x',
'done', makepath=True)
'done'.encode('utf-8'), makepath=True)
self.client.create('/kolla/t1/status/test-hostname/y/l',
'done', makepath=True)
'done'.encode('utf-8'), makepath=True)
self.assertTrue(cmd1.requirements_fulfilled())
@mock.patch('socket.gethostname')
@ -135,7 +136,7 @@ class CommandTest(base.BaseTestCase):
self.client)
self.client.create('/kolla/t1/status/test-hostname/testr/a',
'done', makepath=True)
'done'.encode('utf-8'), makepath=True)
self.assertEqual(None,
cmd1.get_state(
path='/kolla/t1/status/global/testr/a'))
@ -200,8 +201,12 @@ class CommandTest(base.BaseTestCase):
call_order.append('get_state')
return start.CMD_DONE
def lock_side_effect():
call_order.append('lock_enter')
if six.PY3:
def lock_side_effect(lck_self):
call_order.append('lock_enter')
else:
def lock_side_effect():
call_order.append('lock_enter')
m_lock = mock.MagicMock()
with mock.patch.object(self.client, 'Lock', m_lock):
@ -383,7 +388,8 @@ class GenerateConfigTest(base.BaseTestCase):
'command': 'true', 'files': afile}}}
m_gar.return_value = {}, {}
self.client.create('/kolla/deploy_id/testg/testr/files/afile', 'xyz',
self.client.create('/kolla/deploy_id/testg/testr/files/afile',
'xyz'.encode('utf-8'),
makepath=True)
start.run_commands(self.client, conf)
m_wf.assert_called_once_with(afile['afile'], 'xyz')
@ -401,10 +407,11 @@ class GenerateConfigTest(base.BaseTestCase):
conf = {'commands': {'setup': {
'command': 'true', 'files': afile}}}
m_gar.return_value = {}, {}
self.client.create('/kolla/deploy_id/variables/xyz', 'yeah',
self.client.create('/kolla/deploy_id/variables/xyz',
'yeah'.encode('utf-8'),
makepath=True)
self.client.create('/kolla/deploy_id/testg/testr/files/afile',
'{{ xyz }}', makepath=True)
'{{ xyz }}'.encode('utf-8'), makepath=True)
start.run_commands(self.client, conf)
m_wf.assert_called_once_with(afile['afile'], 'yeah')
@ -424,7 +431,7 @@ class GenerateConfigTest(base.BaseTestCase):
m_gar.return_value = {}, {}
m_rt.return_value = ''
self.client.create('/kolla/deploy_id/testg/testr/files/afile',
'{{ xyz }}', makepath=True)
'{{ xyz }}'.encode('utf-8'), makepath=True)
start.run_commands(self.client, conf)
m_wf.assert_called_once_with(afile['afile'], '')
@ -490,7 +497,7 @@ class MainTest(base.BaseTestCase):
acmd = {'command': 'true', 'files': {'afile': afile}}
tconf = {'commands': {'thing': acmd}}
self.client.create('/kolla/deploy_id/testg/testr',
json.dumps(tconf), makepath=True)
json.dumps(tconf).encode('utf-8'), makepath=True)
m_zk_c = mock.MagicMock()
with mock.patch.object(start, 'zk_connection', m_zk_c):
@ -512,7 +519,7 @@ class MainTest(base.BaseTestCase):
acmd = {'command': 'true', 'files': {'afile': afile}}
tconf = {'service': {'daemon': acmd}}
self.client.create('/kolla/deploy_id/testg/testr',
json.dumps(tconf), makepath=True)
json.dumps(tconf).encode('utf-8'), makepath=True)
m_gmc.return_value = tconf
m_zk_c = mock.MagicMock()
@ -704,7 +711,7 @@ class RenderNovaConfTest(base.BaseTestCase):
'nova_api_host': 'nova-api-nova-openstack-did.mfrm.mdom'}
for nam, val in variables.items():
self.client.create('/kolla/did/variables/%s' % nam,
getattr(self, nam, val),
getattr(self, nam, val).encode('utf-8'),
makepath=True)
@mock.patch('time.sleep')
@ -742,7 +749,7 @@ class RenderNovaConfTest(base.BaseTestCase):
template_contents = nc.read()
self.client.create(
'/kolla/did/openstack/nova/nova-compute/files/afile',
template_contents, makepath=True)
template_contents.encode('utf-8'), makepath=True)
cmp_file = os.path.join(mod_dir, 'nova-%s.conf' % self.out)
with open(cmp_file) as cf:

5
kolla_mesos/tests/fakes/config_file.py
View File

@ -29,11 +29,12 @@ class FakeConfigFile(object):
def __enter__(self):
if six.PY3:
func = 'builtins.open'
return_val = io.StringIO(self.text_config)
else:
func = '__builtin__.open'
return_val = io.BytesIO(self.text_config)
self.patcher = mock.patch(func,
return_value=io.BytesIO(self.text_config))
self.patcher = mock.patch(func, return_value=return_val)
self.patcher.start()
def __exit__(self, *args, **kwargs):

17
kolla_mesos/tests/test_commands.py
View File

@ -82,13 +82,13 @@ class CommandsTest(base.BaseTestCase):
}
self.client.create(
'%s/cinder-api/db_sync' % var,
'waiting', makepath=True)
'waiting'.encode('utf-8'), makepath=True)
self.client.create(
'%s/cinder_ansible_tasks/create_database' % var,
'running', makepath=True)
'running'.encode('utf-8'), makepath=True)
self.client.create(
'%s/cinder_ansible_tasks/database_user_create' % var,
'done', makepath=True)
'done'.encode('utf-8'), makepath=True)
status = commands.get_status(test_tasks)
self.assertEqual({'cinder-api/db_sync': exp}, status)
@ -113,10 +113,10 @@ class CommandsTest(base.BaseTestCase):
# create the done states
self.client.create(
'%s/cinder_ansible_tasks/create_database' % var,
'done', makepath=True)
'done'.encode('utf-8'), makepath=True)
self.client.create(
'%s/cinder_ansible_tasks/database_user_create' % var,
'done', makepath=True)
'done'.encode('utf-8'), makepath=True)
status = commands.get_status(test_tasks)
self.assertEqual({'cinder-api/db_sync': exp}, status)
@ -142,11 +142,12 @@ class CommandsTest(base.BaseTestCase):
# create the done state
self.client.create(
'%s/cinder_ansible_tasks/create_database' % var,
'done', makepath=True)
'done'.encode('utf-8'), makepath=True)
self.client.create(
'%s/cinder_ansible_tasks/database_user_create' % var,
'done', makepath=True)
self.client.create('%s/cinder-api/db_sync' % var, 'done',
'done'.encode('utf-8'), makepath=True)
self.client.create('%s/cinder-api/db_sync' % var,
'done'.encode('utf-8'),
makepath=True)
status = commands.get_status(test_tasks)

20
kolla_mesos/tests/test_deployment.py
View File

@ -29,8 +29,7 @@ class TestWriteOpenRC(base.BaseTestCase):
self.addCleanup(self.client.stop)
self.addCleanup(self.client.close)
@mock.patch('kolla_mesos.deployment.open')
def test_write_openrc_ok(self, mock_open):
def test_write_openrc_ok(self):
variables = {'keystone_admin_password': 'foofee',
'kolla_internal_address': 'here.not',
'keystone_admin_port': '4511',
@ -38,15 +37,18 @@ class TestWriteOpenRC(base.BaseTestCase):
'keystone_auth_host': 'not.here'}
configuration.write_variables_zookeeper(self.client, variables)
mock_open.return_value = mock.MagicMock(spec=file)
file_handle = mock_open.return_value.__enter__.return_value
with mock.patch.object(deployment.zk_utils, 'connection') as m_zk_c:
m_zk_c.return_value.__enter__.return_value = self.client
deployment.write_openrc('openrc')
m_open = mock.mock_open()
with mock.patch('kolla_mesos.deployment.open', m_open):
file_handle = m_open.return_value.__enter__.return_value
mock_open.assert_called_once_with('openrc', 'w')
self.assertEqual(1, file_handle.write.call_count)
with mock.patch.object(deployment.zk_utils,
'connection') as m_zk_c:
m_zk_c.return_value.__enter__.return_value = self.client
deployment.write_openrc('openrc')
m_open.assert_called_once_with('openrc', 'w')
self.assertEqual(1, file_handle.write.call_count)
def test_write_openrc_fail(self):
# missing variable "keystone_admin_port"

10
kolla_mesos/tests/test_service.py
View File

@ -72,7 +72,7 @@ class TestAPI(base.BaseTestCase):
def test_kill(self, m_kill):
self.client.create('/kolla/did/openstack/nova/nova-api',
json.dumps({'name': 'openstack/nova/nova-api',
'service': {}}),
'service': {}}).encode('utf-8'),
makepath=True)
with mock.patch.object(service.zk_utils,
@ -86,7 +86,7 @@ class TestAPI(base.BaseTestCase):
def test_get_marathon(self, m_get_state, c_get_state):
self.client.create('/kolla/did/openstack/nova/nova-api',
json.dumps({'name': 'openstack/nova/nova-api',
'service': {}}),
'service': {}}).encode('utf-8'),
makepath=True)
with mock.patch.object(service.zk_utils,
'connection') as m_zk_c:
@ -100,7 +100,7 @@ class TestAPI(base.BaseTestCase):
def test_get_chronos(self, m_get_state, c_get_state):
self.client.create('/kolla/did/openstack/nova/nova_init',
json.dumps({'name': 'openstack/nova/nova_init',
'task': {}}),
'task': {}}).encode('utf-8'),
makepath=True)
with mock.patch.object(service.zk_utils,
'connection') as m_zk_c:
@ -113,7 +113,7 @@ class TestAPI(base.BaseTestCase):
def test_scale_marathon(self, m_scale):
self.client.create('/kolla/did/openstack/nova/nova-api',
json.dumps({'name': 'openstack/nova/nova-api',
'service': {}}),
'service': {}}).encode('utf-8'),
makepath=True)
with mock.patch.object(service.zk_utils,
'connection') as m_zk_c:
@ -127,7 +127,7 @@ class TestAPI(base.BaseTestCase):
def test_update_marathon(self, m_update, m_apply, m_gmf):
self.client.create('/kolla/did/openstack/nova/nova-api',
json.dumps({'name': 'openstack/nova/nova-api',
'service': {}}),
'service': {}}).encode('utf-8'),
makepath=True)
with mock.patch.object(service.zk_utils,
'connection') as m_zk_c: