x
/
kolla-mesos
Code Issues Proposed changes

CLI: implement commands list command 

kolla-mesos commands list <service>

show_tasks.py will be replaced with
this command

partial blueprint per-service-cli

Change-Id: I0fe2f4d3d70e698e2cb3af762572f5f2317292bb
This commit is contained in:
Andrey Pavlov 2016-03-16 17:08:46 +03:00
parent a6ce1448d1
commit 5d88c2ce80
7 changed files with 276 additions and 268 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
kolla_mesos/cli/commands.py Normal file
View File

@ -0,0 +1,70 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from cliff import lister
from oslo_config import cfg
from oslo_log import log
from kolla_mesos import commands
CONF = cfg.CONF
LOG = log.getLogger(__name__)
def format_output(status):
cols = ('Command', 'Status', 'Requirements')
rows = []
for taskname, info in sorted(status.items()):
reg_status = info['register'][1] or 'unknown'
requirements = []
reqts = info['requirements']
for reqt_path, reqt_status in sorted(reqts.items()):
reqt_path = reqt_path.split(
'status/')[1] if 'status/' in reqt_path else reqt_path
if not reqt_status:
reqt_status = 'unknown'
requirements.append('%s:%s' % (reqt_path, reqt_status))
requirements = '\n'.join(requirements)
rows.append((taskname, reg_status, requirements))
return cols, rows
def _clean_path(path):
if 'status/' in path:
path = path.split('status/')[1]
return path
class List(lister.Lister):
"""List all commands and their statuses for this service."""
def get_parser(self, prog_name):
parser = super(List, self).get_parser(prog_name)
parser.add_argument(
'service',
nargs='?',
help='Information for the deployment will be shown if the service '
'is not specified'
)
return parser
def take_action(self, parsed_args):
if parsed_args.service:
status = commands.get_service_status(
parsed_args.service, CONF.service_dir)
else:
status = commands.get_deployment_status(CONF.service_dir)
return format_output(status)

251
kolla_mesos/cmd/show_tasks.py
View File

@ -1,251 +0,0 @@
#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
usage: show_tasks.py [-h] [--id ID] [--list_ids]
Show tasks for a deployment. If there is only one deployment id, an id does
not have to be specified.
optional arguments:
-h, --help show this help message and exit
--id ID show tasks for this deployment id
--list_ids, -l list out the current deployment ids
"""
import argparse
import os
import socket
import sys
import yaml
from kolla_mesos.common import cli_utils
from kolla_mesos.common import file_utils
from kolla_mesos.common import jinja_utils
from kolla_mesos.common import mesos_utils
from kolla_mesos.common import zk_utils
CONFIG_SUFFIX = '_config.yml.j2'
def get_deploy_id(ids):
deploy_id = ''
if ids:
deploy_id = ids[0]
print('Deployment id: %s' % deploy_id)
return deploy_id
def list_ids(ids):
out = '['
comma = ''
for deploy_id in ids:
out += comma + deploy_id
comma = ', '
print(out + ']')
def validate_input(args_ids, ids):
if not ids:
print('Error: No deployment ids exist.')
sys.exit(1)
if args_ids:
if args_ids[0] not in ids:
print("Error: Deployment id %s doesn't exist in " % args_ids +
'the current set of ids: %s' % ids)
sys.exit(1)
elif len(ids) > 1:
print('Error: Multiple deployment ids exist: %s.\n' % ids +
'Use %s --id deployment_id.' % sys.argv[0])
sys.exit(1)
def get_deployment_ids():
ids = []
with zk_utils.connection() as zk:
children = zk.get_children('/kolla')
for child in children:
if child not in ['groups', 'variables', 'common',
'config', 'commands']:
ids.append(child)
return ids
def get_tasks(deploy_id):
"""Get list of tasks
Reads through all the kolla mesos services config files and
parses the requirements and resister options.
Returns a dictionary of all the values registered by tasks
{
taskpath1: {
'requires': [require1, require2...]
'register': register_path
}
}
taskpath examples -
'keystone/keystone/db_sync',
'keystone/keystone_ansible_tasks/create_database',
"""
def get_task_from_cmd(role, cmd, cmd_info):
reg = '/kolla/%s/status/%s/%s' % (deploy_id, role, cmd)
task = {'register': reg, 'requires': []}
for dep in cmd_info.get('dependencies', []):
path = dep['path']
scope = dep.get('scope', 'global')
if scope == 'global':
task['requires'].append(
'/kolla/%s/status/%s' % (deploy_id, path))
elif scope == 'local':
task['requires'].append(
'/kolla/%s/status/%s/%s' % (deploy_id,
socket.gethostname(),
path))
return task
tasks = {}
config_dir = os.path.join(file_utils.find_base_dir(), 'services')
for root, _, files in os.walk(config_dir):
for name in files:
if 'default.' in name:
continue
fpath = os.path.join(root, name)
controller_nodes, compute_nodes, storage_nodes, all_nodes = \
mesos_utils.get_number_of_nodes()
mini_vars = {'cinder_volume_driver': 'lvm',
'deployment_id': deploy_id,
'controller_nodes': str(controller_nodes),
'compute_nodes': str(compute_nodes),
'storage_nodes': str(storage_nodes),
'all_nodes': str(all_nodes)}
cfg = yaml.load(jinja_utils.jinja_render(fpath, mini_vars))
def get_commands():
for cmd in cfg.get('commands', {}):
yield cmd, cfg['commands'][cmd]
if 'service' in cfg:
yield 'daemon', cfg['service']['daemon']
_, _, role = cfg['name'].split('/')
for cmd, cmd_info in get_commands():
task_name = '%s/%s' % (role, cmd)
tasks[task_name] = get_task_from_cmd(role, cmd,
cmd_info)
return tasks
def get_status(tasks):
"""Get status from zookeeper
Returns the status of for each task
{
task1: {
'register': (register_path, reg_status)
'requirements': {
reqt1_path: reqt_status
reqt2_path: reqt_status
...
}
}
Where:
reg_status = 'done', 'running', 'waiting'
reqt_status = '', 'done'
"""
status = {}
with zk_utils.connection() as zk:
# get status of requirements
for task, info in tasks.items():
status[task] = {}
status[task]['requirements'] = {}
for path in info['requires']:
reqt_status = ''
if zk.exists(path):
reqt_status, _ = zk.get(path)
status[task]['requirements'][path] = reqt_status
# get status of registrations
for task, info in tasks.items():
status[task]['register'] = {}
reg_path = info['register']
reg_status = ''
if zk.exists(reg_path):
reg_status, _ = zk.get(reg_path)
status[task]['register'] = (reg_path, reg_status)
return status
def print_status(status):
tasknames = sorted(status)
header = ['Command', 'Status', 'Reqts', 'Reqt Sts']
rows = []
for taskname in tasknames:
reg_status = ''
_, reg_status = status[taskname]['register']
reqts = status[taskname]['requirements']
if reqts:
tname = taskname
reqt_paths = sorted(reqts)
for reqt_path in reqt_paths:
reqt_status = reqts[reqt_path]
reqt_path = clean_path(reqt_path)
rows.append((tname, reg_status,
reqt_path, reqt_status))
tname = ''
reg_status = ''
else:
rows.append((taskname, reg_status,
'', ''))
cli_utils.lister(header, rows, align='l')
def clean_path(path):
if 'status/' in path:
path = path.split('status/')[1]
return path
def main():
parser = argparse.ArgumentParser()
parser.description = (
'Show tasks for a deployment.\n' +
'If there is only one deployment id, an id does not have to be ' +
'specified.')
parser.add_argument('--id', nargs=1,
help='show tasks for this deployment id')
parser.add_argument('--list_ids', '-l', action='store_true',
help='list out the current deployment ids')
args = parser.parse_args()
ids = get_deployment_ids()
if args.list_ids:
list_ids(ids)
sys.exit(0)
validate_input(args.id, ids)
deploy_id = get_deploy_id(args.id or ids)
tasks = get_tasks(deploy_id)
status = get_status(tasks)
print_status(status)
if __name__ == '__main__':
main()

162
kolla_mesos/commands.py Normal file
View File

@ -0,0 +1,162 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import socket
import yaml
from oslo_config import cfg
from oslo_log import log as logging
from kolla_mesos.common import jinja_utils
from kolla_mesos.common import mesos_utils
from kolla_mesos.common import zk_utils
from kolla_mesos import service_definition
LOG = logging.getLogger()
CONF = cfg.CONF
CONF.import_group('kolla', 'kolla_mesos.config.kolla')
def _get_task_from_cmd(role, cmd, cmd_info):
reg = '/kolla/%s/status/%s/%s' % (CONF.kolla.deployment_id, role, cmd)
task = {'register': reg, 'requires': []}
for dep in cmd_info.get('dependencies', []):
path = dep['path']
scope = dep.get('scope', 'global')
if scope == 'global':
task['requires'].append(
'/kolla/%s/status/%s' % (CONF.kolla.deployment_id, path))
elif scope == 'local':
task['requires'].append(
'/kolla/%s/status/%s/%s' % (CONF.kolla.deployment_id,
socket.gethostname(),
path))
return task
def _get_config_tasks(config_path, tasks):
controller_nodes, compute_nodes, storage_nodes, all_nodes = \
mesos_utils.get_number_of_nodes()
mini_vars = {'cinder_volume_driver': 'lvm',
'deployment_id': CONF.kolla.deployment_id,
'controller_nodes': str(controller_nodes),
'compute_nodes': str(compute_nodes),
'storage_nodes': str(storage_nodes),
'all_nodes': str(all_nodes)}
config = yaml.load(jinja_utils.jinja_render(config_path, mini_vars))
def get_commands():
for cmd in config.get('commands', {}):
yield cmd, config['commands'][cmd]
if 'service' in config:
yield 'daemon', config['service']['daemon']
_, _, role = config['name'].split('/')
for cmd, cmd_info in get_commands():
task_name = '%s/%s' % (role, cmd)
tasks[task_name] = _get_task_from_cmd(role, cmd, cmd_info)
return tasks
def get_tasks(config_path):
"""Get list of tasks
Reads through all the kolla mesos services config files located in
config_path and parses the requirements and resister options.
Returns a dictionary of all the values registered by tasks
{
taskpath1: {
'requires': [require1, require2...]
'register': register_path
}
}
taskpath examples -
'keystone/keystone/db_sync',
'keystone/keystone_ansible_tasks/create_database',
"""
tasks = {}
if os.path.isfile(config_path):
_get_config_tasks(config_path, tasks)
else:
for root, _, files in os.walk(config_path):
for name in files:
if 'default.' in name:
continue
fpath = os.path.join(root, name)
_get_config_tasks(fpath, tasks)
return tasks
def get_service_tasks(service_name, service_dir):
if '/' not in service_name:
service_name = service_name.replace('-', '/')
config_path = service_definition.find_service_file(
service_name, service_dir)
return get_tasks(config_path)
def get_status(tasks):
"""Get status from zookeeper
Returns the status for each task
{
task1: {
'register': (register_path, reg_status)
'requirements': {
reqt1_path: reqt_status
reqt2_path: reqt_status
...
}
}
Where:
reg_status = 'done', 'running', 'waiting'
reqt_status = '', 'done'
"""
status = {}
with zk_utils.connection() as zk:
# get status of requirements
for task, info in tasks.items():
status[task] = {}
status[task]['requirements'] = {}
for path in info['requires']:
reqt_status = ''
if zk.exists(path):
reqt_status, _ = zk.get(path)
status[task]['requirements'][path] = reqt_status
# get status of registrations
for task, info in tasks.items():
status[task]['register'] = {}
reg_path = info['register']
reg_status = ''
if zk.exists(reg_path):
reg_status, _ = zk.get(reg_path)
status[task]['register'] = (reg_path, reg_status)
return status
def get_service_status(service_name, service_dir):
tasks = get_service_tasks(service_name, service_dir)
return get_status(tasks)
def get_deployment_status(service_dir):
tasks = get_tasks(service_dir)
return get_status(tasks)

2
kolla_mesos/common/file_utils.py
View File

@ -49,6 +49,8 @@ def find_base_dir():
base_script_path = os.path.basename(script_path)
if base_script_path == 'kolla-mesos':
return script_path
if base_script_path == 'kolla_mesos':
return os.path.join(script_path, '..')
if base_script_path == 'cmd':
return os.path.join(script_path, '..', '..')
if base_script_path == 'subunit':

2
kolla_mesos/service_definition.py
View File

@ -36,7 +36,7 @@ def find_service_file(service_name, service_dir):
raise exception.KollaNotFoundException(service_dir,
entity='service directory')
short_name = service_name.split('/')[-1]
short_name = service_name.split('/')[-1].replace('_ansible_tasks', '-init')
for root, dirs, names in os.walk(service_dir):
for name in names:
if short_name in name:

56
kolla_mesos/tests/test_show_tasks.py → kolla_mesos/tests/test_commands.py
View File

@ -11,28 +11,30 @@
# limitations under the License.
import mock
import os
from oslo_config import cfg
from zake import fake_client
from kolla_mesos.cmd import show_tasks
from kolla_mesos import commands
from kolla_mesos.common import file_utils
from kolla_mesos.tests import base
from kolla_mesos.tests.fakes import mesos as fake_mesos
CONF = cfg.CONF
CONF.import_group('mesos', 'kolla_mesos.config.mesos')
CONF.import_group('kolla', 'kolla_mesos.config.kolla')
class ShowTasksTest(base.BaseTestCase):
class CommandsTest(base.BaseTestCase):
def setUp(self):
super(ShowTasksTest, self).setUp()
super(CommandsTest, self).setUp()
CONF.set_override('host', 'http://127.0.0.1:5050', group='mesos')
CONF.set_override('deployment_id', 'test', group='kolla')
self.client = fake_client.FakeClient()
self.client.start()
self.addCleanup(self.client.stop)
self.addCleanup(self.client.close)
self.dep_id = 'test'
@fake_mesos.FakeMesosStateTaggedSlaves()
def test_get_tasks_sanity(self):
@ -42,14 +44,32 @@ class ShowTasksTest(base.BaseTestCase):
'%s/cinder_ansible_tasks/create_database' % var,
'%s/cinder_ansible_tasks/database_user_create' % var]}
tasks = show_tasks.get_tasks(self.dep_id)
config_path = os.path.join(
file_utils.find_base_dir(), 'services/cinder')
tasks = commands.get_tasks(config_path)
self.assertEqual(exp, tasks['cinder-api/db_sync'])
@fake_mesos.FakeMesosStateTaggedSlaves()
@mock.patch.object(show_tasks.zk_utils, 'connection')
def test_get_service_tasks_sanity(self):
var = '/kolla/test/status'
exp = {'register': '%s/cinder-api/db_sync' % var,
'requires': [
'%s/cinder_ansible_tasks/create_database' % var,
'%s/cinder_ansible_tasks/database_user_create' % var]}
config_dir = os.path.join(
file_utils.find_base_dir(), 'services')
tasks = commands.get_service_tasks(
'openstack/cinder/cinder-api', config_dir)
self.assertEqual(exp, tasks['cinder-api/db_sync'])
@fake_mesos.FakeMesosStateTaggedSlaves()
@mock.patch.object(commands.zk_utils, 'connection')
def test_get_status_waiting(self, m_zk_c):
m_zk_c.return_value.__enter__.return_value = self.client
tasks = show_tasks.get_tasks(self.dep_id)
config_path = os.path.join(
file_utils.find_base_dir(), 'services/cinder')
tasks = commands.get_tasks(config_path)
# just get what we want for the test
test_tasks = {'cinder-api/db_sync':
tasks['cinder-api/db_sync']}
@ -69,14 +89,16 @@ class ShowTasksTest(base.BaseTestCase):
self.client.create(
'%s/cinder_ansible_tasks/database_user_create' % var,
'done', makepath=True)
status = show_tasks.get_status(test_tasks)
status = commands.get_status(test_tasks)
self.assertEqual({'cinder-api/db_sync': exp}, status)
@fake_mesos.FakeMesosStateTaggedSlaves()
@mock.patch.object(show_tasks.zk_utils, 'connection')
@mock.patch.object(commands.zk_utils, 'connection')
def test_get_status_deps_done(self, m_zk_c):
m_zk_c.return_value.__enter__.return_value = self.client
tasks = show_tasks.get_tasks(self.dep_id)
config_path = os.path.join(
file_utils.find_base_dir(), 'services/cinder')
tasks = commands.get_tasks(config_path)
# just get what we want for the test
test_tasks = {'cinder-api/db_sync':
tasks['cinder-api/db_sync']}
@ -96,14 +118,16 @@ class ShowTasksTest(base.BaseTestCase):
'%s/cinder_ansible_tasks/database_user_create' % var,
'done', makepath=True)
status = show_tasks.get_status(test_tasks)
status = commands.get_status(test_tasks)
self.assertEqual({'cinder-api/db_sync': exp}, status)
@fake_mesos.FakeMesosStateTaggedSlaves()
@mock.patch.object(show_tasks.zk_utils, 'connection')
@mock.patch.object(commands.zk_utils, 'connection')
def test_get_status_done(self, m_zk_c):
m_zk_c.return_value.__enter__.return_value = self.client
tasks = show_tasks.get_tasks(self.dep_id)
config_path = os.path.join(
file_utils.find_base_dir(), 'services/cinder')
tasks = commands.get_tasks(config_path)
# just get what we want for the test
test_tasks = {'cinder-api/db_sync':
tasks['cinder-api/db_sync']}
@ -125,5 +149,5 @@ class ShowTasksTest(base.BaseTestCase):
self.client.create('%s/cinder-api/db_sync' % var, 'done',
makepath=True)
status = show_tasks.get_status(test_tasks)
status = commands.get_status(test_tasks)
self.assertEqual({'cinder-api/db_sync': exp}, status)

1
setup.cfg
View File

@ -57,6 +57,7 @@ kolla_mesos.cli =
deployment cleanup = kolla_mesos.cli.deployment:Cleanup
deployment show = kolla_mesos.cli.deployment:Show
deployment list = kolla_mesos.cli.deployment:List
commands list = kolla_mesos.cli.commands:List
oslo.config.opts =
kolla_mesos = kolla_mesos.opts:list_opts