diff --git a/fuelclient/cli/actions/__init__.py b/fuelclient/cli/actions/__init__.py index 80e7128..f9af0c6 100644 --- a/fuelclient/cli/actions/__init__.py +++ b/fuelclient/cli/actions/__init__.py @@ -20,6 +20,7 @@ from fuelclient.cli.actions.deploy import DeployChangesAction from fuelclient.cli.actions.environment import EnvironmentAction from fuelclient.cli.actions.fact import DeploymentAction from fuelclient.cli.actions.fact import ProvisioningAction +from fuelclient.cli.actions.graph import GraphAction from fuelclient.cli.actions.token import TokenAction from fuelclient.cli.actions.health import HealthCheckAction from fuelclient.cli.actions.interrupt import ResetAction @@ -57,7 +58,8 @@ actions_tuple = ( NodeGroupAction, NotificationsAction, NotifyAction, - TokenAction + TokenAction, + GraphAction, ) actions = dict( diff --git a/fuelclient/cli/actions/base.py b/fuelclient/cli/actions/base.py index b874bcf..eb244b7 100644 --- a/fuelclient/cli/actions/base.py +++ b/fuelclient/cli/actions/base.py @@ -17,7 +17,9 @@ from functools import wraps from itertools import imap import os -from fuelclient.cli.error import ArgumentException +import six + +from fuelclient.cli import error from fuelclient.cli.formatting import quote_and_join from fuelclient.cli.serializers import Serializer from fuelclient.client import APIClient @@ -86,7 +88,10 @@ class Action(object): def full_path_directory(self, directory, base_name): full_path = os.path.join(directory, base_name) if not os.path.exists(full_path): - os.mkdir(full_path) + try: + os.mkdir(full_path) + except OSError as e: + raise error.ActionException(six.text_type(e)) return full_path def default_directory(self, directory=None): @@ -105,7 +110,7 @@ def wrap(method, args, f): if method(getattr(params, _arg) for _arg in args): return f(self, params) else: - raise ArgumentException( + raise error.ArgumentException( "{0} required!".format( quote_and_join( "--" + arg for arg in args diff --git a/fuelclient/cli/actions/graph.py b/fuelclient/cli/actions/graph.py new file mode 100644 index 0000000..7850a02 --- /dev/null +++ b/fuelclient/cli/actions/graph.py @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys + +import six + +from fuelclient.cli.actions import base +import fuelclient.cli.arguments as Args +from fuelclient.cli import error +from fuelclient.objects import environment + + +class GraphAction(base.Action): + """Manipulate deployment graph's representation.""" + + action_name = 'graph' + + def __init__(self): + super(GraphAction, self).__init__() + self.args = ( + Args.get_env_arg(), + Args.get_render_arg( + "Render graph from DOT to PNG" + ), + Args.get_download_arg( + "Download graph of specific cluster" + ), + Args.get_dir_arg( + "Select target dir to render graph." + ), + Args.group( + Args.get_skip_tasks(), + Args.get_tasks() + ), + Args.get_graph_endpoint(), + Args.get_graph_startpoint(), + Args.get_parents_arg(), + ) + self.flag_func_map = ( + ('render', self.render), + ('download', self.download), + ) + + @base.check_all("env") + def download(self, params): + """Download deployment graph to stdout + + fuel graph --env 1 --download + fuel graph --env 1 --download --tasks A B C + fuel graph --env 1 --download --skip X Y --end pre_deployment + fuel graph --env 1 --download --skip X Y --start post_deployment + + Sepcify output: + fuel graph --env 1 --download > outpup/dir/file.gv + + Get parents only for task A: + + fuel graph --env 1 --download --parents-for A + """ + env = environment.Environment(params.env) + + parents_for = getattr(params, 'parents-for') + + used_params = "# params:\n" + for param in ('start', 'end', 'skip', 'tasks', 'parents-for'): + used_params += "# - {0}: {1}\n".format(param, + getattr(params, param)) + + if params.tasks: + tasks = params.tasks + else: + tasks = env.get_tasks( + skip=params.skip, end=params.end, start=params.start) + + dotraph = env.get_deployment_tasks_graph(tasks, + parents_for=parents_for) + sys.stdout.write(six.text_type(used_params)) + sys.stdout.write(six.text_type(dotraph)) + + @base.check_all("render") + def render(self, params): + """Render graph in PNG format + + fuel graph --render graph.gv + fuel graph --render graph.gv --dir ./output/dir/ + + Read graph from stdin + some_process | fuel graph --render - + """ + if params.render == '-': + dot_data = sys.stdin.read() + out_filename = 'graph.gv' + elif not os.path.exists(params.render): + raise error.ArgumentException( + "Input file does not exist" + ) + else: + out_filename = os.path.basename(params.render) + with open(params.render, 'r') as f: + dot_data = f.read() + + target_dir = self.full_path_directory( + self.default_directory(params.dir), + '' + ) + target_file = os.path.join( + target_dir, + '{0}.png'.format(out_filename), + ) + + if not os.access(target_file, os.W_OK): + raise error.ActionException( + 'Path {0} is not writable'.format(target_file)) + + render_graph(dot_data, target_file) + print('Graph saved in "{0}"'.format(target_file)) + + +def render_graph(input_data, output_path): + """Renders DOT graph using pydot or pygraphviz depending on their presence. + + If none of the libraries is available and are fully functional it is not + possible to render graphs. + + :param input_data: DOT graph representation + :param output_path: path to the rendered graph + """ + try: + _render_with_pydot(input_data, output_path) + except ImportError: + try: + _render_with_pygraphiz(input_data, output_path) + except ImportError: + raise error.WrongEnvironmentError( + "This action require Graphviz installed toghether with " + "'pydot' or 'pygraphviz' Python library") + + +def _render_with_pydot(input_data, output_path): + """Renders graph using pydot library.""" + import pydot + + graph = pydot.graph_from_dot_data(input_data) + if not graph: + raise error.BadDataException( + "Passed data does not contain graph in DOT format") + try: + graph.write_png(output_path) + except pydot.InvocationException as e: + raise error.WrongEnvironmentError( + "There was an error with rendering graph:\n{0}".format(e)) + + +def _render_with_pygraphiz(input_data, output_path): + """Renders graph using pygraphviz library.""" + import pygraphviz as pgv + + graph = pgv.AGraph(string=input_data) + graph.draw(output_path, prog='dot', format='png') diff --git a/fuelclient/cli/arguments.py b/fuelclient/cli/arguments.py index 85909b3..382388d 100644 --- a/fuelclient/cli/arguments.py +++ b/fuelclient/cli/arguments.py @@ -21,7 +21,7 @@ from fuelclient.cli.error import ArgumentException from fuelclient.client import APIClient substitutions = { - #replace from: to + # replace from: to "env": "environment", "nodes": "node", "net": "network", @@ -302,37 +302,49 @@ def get_net_arg(help_msg): def get_graph_endpoint(): return get_arg( 'end', - flags=('--end',), action="store", default=None, - help="Specify endpoint for the graph traversal.") + help="Specify endpoint for the graph traversal.", + metavar='TASK', + ) def get_graph_startpoint(): return get_arg( 'start', - flags=('--start',), action="store", default=None, - help="Specify start point for the graph traversal.") + help="Specify start point for the graph traversal.", + metavar='TASK', + ) def get_skip_tasks(): return get_arg( 'skip', - flags=('--skip',), nargs='+', default=[], - help="Get list of tasks to be skipped.") + help="Get list of tasks to be skipped.", + metavar='TASK', + ) def get_tasks(): return get_arg( 'tasks', - flags=('--tasks',), nargs='+', default=[], - help="Get list of tasks to be executed.") + help="Get list of tasks to be executed.", + metavar='TASK', + ) + + +def get_parents_arg(): + return get_arg( + 'parents-for', + help="Get parent for given task", + metavar='TASK', + ) def get_nst_arg(help_msg): @@ -412,6 +424,13 @@ def get_release_arg(help_msg, required=False): help=help_msg) +def get_render_arg(help_msg): + return get_str_arg( + "render", + metavar='INPUT', + help=help_msg) + + def get_node_arg(help_msg): default_kwargs = { "action": NodeAction, diff --git a/fuelclient/client.py b/fuelclient/client.py index b857ccf..1cf56e0 100644 --- a/fuelclient/client.py +++ b/fuelclient/client.py @@ -133,9 +133,12 @@ class Client(object): return resp.json() - @exceptions_decorator - def get_request(self, api, ostf=False, params=None): - """Make GET request to specific API + def get_request_raw(self, api, ostf=False, params=None): + """Make a GET request to specific API and return raw response. + + :param api: API endpoint (path) + :param ostf: is this a call to OSTF API + :param params: params passed to GET request """ url = (self.ostf_root if ostf else self.api_root) + api self.print_debug( @@ -145,13 +148,24 @@ class Client(object): headers = {'x-auth-token': self.auth_token} params = params or {} + return requests.get(url, params=params, headers=headers) - resp = requests.get(url, params=params, headers=headers) + @exceptions_decorator + def get_request(self, api, ostf=False, params=None): + """Make GET request to specific API + """ + resp = self.get_request_raw(api, ostf, params) resp.raise_for_status() return resp.json() def post_request_raw(self, api, data, ostf=False): + """Make a POST request to specific API and return raw response. + + :param api: API endpoint (path) + :param data: data send in request, will be serialzied to JSON + :param ostf: is this a call to OSTF API + """ url = (self.ostf_root if ostf else self.api_root) + api data_json = json.dumps(data) self.print_debug( diff --git a/fuelclient/objects/environment.py b/fuelclient/objects/environment.py index dd6c4fe..93e6534 100644 --- a/fuelclient/objects/environment.py +++ b/fuelclient/objects/environment.py @@ -30,6 +30,7 @@ class Environment(BaseObject): class_api_path = "clusters/" instance_api_path = "clusters/{0}/" deployment_tasks_path = 'clusters/{0}/deployment_tasks' + deployment_tasks_graph_path = 'clusters/{0}/deploy_tasks/graph.gv' @classmethod def create(cls, name, release_id, net, net_segment_type=None): @@ -400,3 +401,13 @@ class Environment(BaseObject): def update_deployment_tasks(self, data): url = self.deployment_tasks_path.format(self.id) return self.connection.put_request(url, data) + + def get_deployment_tasks_graph(self, tasks, parents_for=None): + url = self.deployment_tasks_graph_path.format(self.id) + params = { + 'tasks': ','.join(tasks), + 'parents_for': parents_for, + } + resp = self.connection.get_request_raw(url, params=params) + resp.raise_for_status() + return resp.text diff --git a/fuelclient/tests/test_base_action.py b/fuelclient/tests/test_base_action.py new file mode 100644 index 0000000..58a0d7d --- /dev/null +++ b/fuelclient/tests/test_base_action.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from fuelclient.cli.actions import base +from fuelclient.cli import error +from fuelclient.tests import base as base_tests + + +class TestBaseAction(base_tests.UnitTestCase): + + def setUp(self): + super(TestBaseAction, self).setUp() + self.action = base.Action() + + @mock.patch('fuelclient.cli.actions.base.os') + def test_default_directory_with_param(self, m_os): + directory = 'some/dir' + self.action.default_directory(directory) + m_os.path.abspath.assert_called_once_with(directory) + + @mock.patch('fuelclient.cli.actions.base.os') + def test_default_directory_without_param(self, m_os): + self.action.default_directory() + m_os.path.abspath.assert_called_once_with(m_os.curdir) + + @mock.patch('fuelclient.cli.actions.base.os.mkdir') + @mock.patch('fuelclient.cli.actions.base.os.path.exists') + def test_full_path_directory(self, m_exists, m_mkdir): + m_exists.return_value = False + self.assertEqual( + self.action.full_path_directory('/base/path', 'subdir'), + '/base/path/subdir' + ) + m_mkdir.assert_called_once_with('/base/path/subdir') + + @mock.patch('fuelclient.cli.actions.base.os') + def test_full_path_directory_no_access(self, m_os): + exc_msg = 'Bas permissions' + m_os.path.exists.return_value = False + m_os.mkdir.side_effect = OSError(exc_msg) + + with self.assertRaisesRegexp(error.ActionException, exc_msg): + self.action.full_path_directory('/base/path', 'subdir') + + @mock.patch('fuelclient.cli.actions.base.os') + def test_full_path_directory_already_exists(self, m_os): + m_os.path.exists.return_value = True + self.action.full_path_directory('/base/path', 'subdir') + self.assertEqual(m_os.mkdir.call_count, 0) diff --git a/fuelclient/tests/test_graph_action.py b/fuelclient/tests/test_graph_action.py new file mode 100644 index 0000000..a938525 --- /dev/null +++ b/fuelclient/tests/test_graph_action.py @@ -0,0 +1,203 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import io +import os + +import mock +import requests_mock + +from fuelclient.cli.actions import graph +from fuelclient.tests import base + + +GRAPH_API_OUTPUT = "digraph G { A -> B -> C }" +TASKS_API_OUTPUT = [ + {'id': 'primary-controller'}, + {'id': 'sync-time'}, +] + + +class TestGraphAction(base.UnitTestCase): + + def setUp(self): + super(TestGraphAction, self).setUp() + self.requests_mock = requests_mock.mock() + self.requests_mock.start() + self.m_tasks_api = self.requests_mock.get( + '/api/v1/clusters/1/deployment_tasks', + json=TASKS_API_OUTPUT) + self.m_graph_api = self.requests_mock.get( + '/api/v1/clusters/1/deploy_tasks/graph.gv', + text=GRAPH_API_OUTPUT) + + self.m_full_path = mock.patch.object(graph.GraphAction, + 'full_path_directory').start() + self.m_full_path.return_value = '/path' + + def tearDown(self): + super(TestGraphAction, self).tearDown() + self.requests_mock.stop() + self.m_full_path.stop() + + def test_download_all_tasks(self): + with mock.patch('sys.stdout', new=io.StringIO()) as m_stdout: + self.execute_wo_auth( + ['fuel', 'graph', '--download', '--env', '1', '--download'] + ) + + querystring = self.m_graph_api.last_request.qs + for task in TASKS_API_OUTPUT: + self.assertIn(task['id'], querystring['tasks'][0]) + self.assertIn(GRAPH_API_OUTPUT, m_stdout.getvalue()) + + def test_download_selected_tasks(self): + with mock.patch('sys.stdout', new=io.StringIO()) as m_stdout: + self.execute_wo_auth( + ['fuel', 'graph', '--download', '--env', '1', + '--tasks', 'task-a', 'task-b'] + ) + + querystring = self.m_graph_api.last_request.qs + self.assertIn('task-a', querystring['tasks'][0]) + self.assertIn('task-b', querystring['tasks'][0]) + self.assertIn(GRAPH_API_OUTPUT, m_stdout.getvalue()) + + def test_download_with_skip(self): + with mock.patch('sys.stdout', new=io.StringIO()) as m_stdout: + self.execute_wo_auth( + ['fuel', 'graph', '--download', '--env', '1', + '--skip', 'sync-time', 'task-b'] + ) + querystring = self.m_graph_api.last_request.qs + self.assertIn('primary-controller', querystring['tasks'][0]) + self.assertNotIn('sync-time', querystring['tasks'][0]) + self.assertNotIn('task-b', querystring['tasks'][0]) + self.assertIn(GRAPH_API_OUTPUT, m_stdout.getvalue()) + + def test_download_with_end_and_start(self): + with mock.patch('sys.stdout', new=io.StringIO()) as m_stdout: + self.execute_wo_auth( + ['fuel', 'graph', '--download', '--env', '1', + '--start', 'task-a', '--end', 'task-b'] + ) + + tasks_qs = self.m_tasks_api.last_request.qs + self.assertEqual('task-a', tasks_qs['start'][0]) + self.assertEqual('task-b', tasks_qs['end'][0]) + + graph_qs = self.m_graph_api.last_request.qs + for task in TASKS_API_OUTPUT: + self.assertIn(task['id'], graph_qs['tasks'][0]) + self.assertIn(GRAPH_API_OUTPUT, m_stdout.getvalue()) + + def test_download_only_parents(self): + with mock.patch('sys.stdout', new=io.StringIO()) as m_stdout: + self.execute_wo_auth( + ['fuel', 'graph', '--download', '--env', '1', + '--parents-for', 'task-z'] + ) + querystring = self.m_graph_api.last_request.qs + self.assertEqual('task-z', querystring['parents_for'][0]) + self.assertIn(GRAPH_API_OUTPUT, m_stdout.getvalue()) + + def test_params_saved_in_dotfile(self): + with mock.patch('sys.stdout', new=io.StringIO()) as m_stdout: + self.execute_wo_auth( + ['fuel', 'graph', '--download', '--env', '1', + '--parents-for', 'task-z', + '--skip', 'task-a'] + ) + saved_params = ("# params:\n" + "# - start: None\n" + "# - end: None\n" + "# - skip: ['task-a']\n" + "# - tasks: []\n" + "# - parents-for: task-z\n") + self.assertIn(saved_params + GRAPH_API_OUTPUT, m_stdout.getvalue()) + + @mock.patch('fuelclient.cli.actions.graph.open', create=True) + @mock.patch('fuelclient.cli.actions.graph.render_graph') + @mock.patch('fuelclient.cli.actions.graph.os.access') + @mock.patch('fuelclient.cli.actions.graph.os.path.exists') + def test_render(self, m_exists, m_access, m_render, m_open): + graph_data = 'some-dot-data' + m_exists.return_value = True + m_open().__enter__().read.return_value = graph_data + + self.execute_wo_auth( + ['fuel', 'graph', '--render', 'graph.gv'] + ) + + m_open.assert_called_with('graph.gv', 'r') + m_render.assert_called_once_with(graph_data, '/path/graph.gv.png') + + @mock.patch('fuelclient.cli.actions.graph.os.path.exists') + def test_render_no_file(self, m_exists): + m_exists.return_value = False + with self.assertRaises(SystemExit): + self.execute_wo_auth( + ['fuel', 'graph', '--render', 'graph.gv'] + ) + + @mock.patch('fuelclient.cli.actions.graph.open', create=True) + @mock.patch('fuelclient.cli.actions.graph.render_graph') + @mock.patch('fuelclient.cli.actions.graph.os.access') + @mock.patch('fuelclient.cli.actions.graph.os.path.exists') + def test_render_with_output_path(self, m_exists, m_access, m_render, + m_open): + output_dir = '/output/dir' + graph_data = 'some-dot-data' + m_exists.return_value = True + m_open().__enter__().read.return_value = graph_data + self.m_full_path.return_value = output_dir + + self.execute_wo_auth( + ['fuel', 'graph', '--render', 'graph.gv', '--dir', output_dir] + ) + + self.m_full_path.assert_called_once_with(output_dir, '') + m_render.assert_called_once_with(graph_data, + '/output/dir/graph.gv.png') + + @mock.patch('fuelclient.cli.actions.graph.os.access') + @mock.patch('fuelclient.cli.actions.graph.render_graph') + def test_render_from_stdin(self, m_render, m_access): + graph_data = u'graph data' + + with mock.patch('sys.stdin', new=io.StringIO(graph_data)): + self.execute_wo_auth( + ['fuel', 'graph', '--render', '-', ] + ) + + m_render.assert_called_once_with(graph_data, '/path/graph.gv.png') + + @mock.patch('fuelclient.cli.actions.graph.open', create=True) + @mock.patch('fuelclient.cli.actions.graph.os.path.exists') + @mock.patch('fuelclient.cli.actions.graph.os.access') + def test_render_no_access_to_output(self, m_access, m_exists, m_open): + m_exists.return_value = True + m_access.return_value = False + output_dir = '/output/dir' + self.m_full_path.return_value = output_dir + + with self.assertRaises(SystemExit): + self.execute_wo_auth( + ['fuel', 'graph', '--render', 'graph.gv', '--dir', output_dir] + ) + m_access.assert_called_once_with( + os.path.join(output_dir, 'graph.gv.png'), os.W_OK) diff --git a/test-requirements.txt b/test-requirements.txt index 5ccbe5b..f5cd560 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,3 +5,4 @@ nose2==0.4.1 nose-timer==0.2.0 pyprof2calltree==1.3.2 gprof2dot==2014.09.29 +requests-mock>=0.6.0