openstack/deb-python-fuelclient
Code Issues Proposed changes

Add CLI for graph visualization

Browse Source 
* added new action 'graph' to get deployment graph in DOT format:
 - download full graph for cluster 1
   Examples:
     fuel graph --env 1 --download
     fuel graph --env 1 --download > graph.gv

 - it's possible to pass 'start', 'end', 'skip' to manipulate what
   include in graph
   Examples:
     fuel graph --env 1 --download --start netconfig
     fuel graph --env 1 --download --skip hiera

 - get graph with task's parents
   Examples:
     fuel graph --env 1 --download --parents-for post_deployment

 - render graph (optional)
   Examples:
     fuel graph --render path/to/graph.gv

   Rendering requires installed pydot or pygraphivz together with
   Graphviz app.
* params that was used to download graph are saved as comments
  in file with graph in DOT format
* added requests-mock for http requests mocking
* improved readability of tasks related args:
  - removed redundant 'flag' argument causing to display arg two
    times in help
  - added metavar 'TASK' to tell better what we expect

Depends on: Id6fe85efe2549a63737ad50e5e55a70a480c83ab
Implements: blueprint granular-deployment-based-on-tasks

Change-Id: I4d60d8b4c8eca53f459a5459cb3f54af09e04306
This commit is contained in:
Sebastian Kalinowski
2015-02-10 13:38:06 +01:00
committed by Sebastian Kalinowski
parent 8a292dbdfc
commit 0f4ca9c279
9 changed files with 510 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
fuelclient/cli/actions/__init__.py
View File

@@ -20,6 +20,7 @@ from fuelclient.cli.actions.deploy import DeployChangesAction
from fuelclient.cli.actions.environment import EnvironmentAction
from fuelclient.cli.actions.fact import DeploymentAction
from fuelclient.cli.actions.fact import ProvisioningAction
from fuelclient.cli.actions.graph import GraphAction
from fuelclient.cli.actions.token import TokenAction
from fuelclient.cli.actions.health import HealthCheckAction
from fuelclient.cli.actions.interrupt import ResetAction
@@ -57,7 +58,8 @@ actions_tuple = (
NodeGroupAction,
NotificationsAction,
NotifyAction,
TokenAction
TokenAction,
GraphAction,
)
actions = dict(

11
fuelclient/cli/actions/base.py
View File

@@ -17,7 +17,9 @@ from functools import wraps
from itertools import imap
import os
from fuelclient.cli.error import ArgumentException
import six
from fuelclient.cli import error
from fuelclient.cli.formatting import quote_and_join
from fuelclient.cli.serializers import Serializer
from fuelclient.client import APIClient
@@ -86,7 +88,10 @@ class Action(object):
def full_path_directory(self, directory, base_name):
full_path = os.path.join(directory, base_name)
if not os.path.exists(full_path):
os.mkdir(full_path)
try:
os.mkdir(full_path)
except OSError as e:
raise error.ActionException(six.text_type(e))
return full_path
def default_directory(self, directory=None):
@@ -105,7 +110,7 @@ def wrap(method, args, f):
if method(getattr(params, _arg) for _arg in args):
return f(self, params)
else:
raise ArgumentException(
raise error.ArgumentException(
"{0} required!".format(
quote_and_join(
"--" + arg for arg in args

174
fuelclient/cli/actions/graph.py Normal file
View File

@@ -0,0 +1,174 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import sys
import six
from fuelclient.cli.actions import base
import fuelclient.cli.arguments as Args
from fuelclient.cli import error
from fuelclient.objects import environment
class GraphAction(base.Action):
"""Manipulate deployment graph's representation."""
action_name = 'graph'
def __init__(self):
super(GraphAction, self).__init__()
self.args = (
Args.get_env_arg(),
Args.get_render_arg(
"Render graph from DOT to PNG"
),
Args.get_download_arg(
"Download graph of specific cluster"
),
Args.get_dir_arg(
"Select target dir to render graph."
),
Args.group(
Args.get_skip_tasks(),
Args.get_tasks()
),
Args.get_graph_endpoint(),
Args.get_graph_startpoint(),
Args.get_parents_arg(),
)
self.flag_func_map = (
('render', self.render),
('download', self.download),
)
@base.check_all("env")
def download(self, params):
"""Download deployment graph to stdout
fuel graph --env 1 --download
fuel graph --env 1 --download --tasks A B C
fuel graph --env 1 --download --skip X Y --end pre_deployment
fuel graph --env 1 --download --skip X Y --start post_deployment
Sepcify output:
fuel graph --env 1 --download > outpup/dir/file.gv
Get parents only for task A:
fuel graph --env 1 --download --parents-for A
"""
env = environment.Environment(params.env)
parents_for = getattr(params, 'parents-for')
used_params = "# params:\n"
for param in ('start', 'end', 'skip', 'tasks', 'parents-for'):
used_params += "# - {0}: {1}\n".format(param,
getattr(params, param))
if params.tasks:
tasks = params.tasks
else:
tasks = env.get_tasks(
skip=params.skip, end=params.end, start=params.start)
dotraph = env.get_deployment_tasks_graph(tasks,
parents_for=parents_for)
sys.stdout.write(six.text_type(used_params))
sys.stdout.write(six.text_type(dotraph))
@base.check_all("render")
def render(self, params):
"""Render graph in PNG format
fuel graph --render graph.gv
fuel graph --render graph.gv --dir ./output/dir/
Read graph from stdin
some_process | fuel graph --render -
"""
if params.render == '-':
dot_data = sys.stdin.read()
out_filename = 'graph.gv'
elif not os.path.exists(params.render):
raise error.ArgumentException(
"Input file does not exist"
)
else:
out_filename = os.path.basename(params.render)
with open(params.render, 'r') as f:
dot_data = f.read()
target_dir = self.full_path_directory(
self.default_directory(params.dir),
''
)
target_file = os.path.join(
target_dir,
'{0}.png'.format(out_filename),
)
if not os.access(target_file, os.W_OK):
raise error.ActionException(
'Path {0} is not writable'.format(target_file))
render_graph(dot_data, target_file)
print('Graph saved in "{0}"'.format(target_file))
def render_graph(input_data, output_path):
"""Renders DOT graph using pydot or pygraphviz depending on their presence.
If none of the libraries is available and are fully functional it is not
possible to render graphs.
:param input_data: DOT graph representation
:param output_path: path to the rendered graph
"""
try:
_render_with_pydot(input_data, output_path)
except ImportError:
try:
_render_with_pygraphiz(input_data, output_path)
except ImportError:
raise error.WrongEnvironmentError(
"This action require Graphviz installed toghether with "
"'pydot' or 'pygraphviz' Python library")
def _render_with_pydot(input_data, output_path):
"""Renders graph using pydot library."""
import pydot
graph = pydot.graph_from_dot_data(input_data)
if not graph:
raise error.BadDataException(
"Passed data does not contain graph in DOT format")
try:
graph.write_png(output_path)
except pydot.InvocationException as e:
raise error.WrongEnvironmentError(
"There was an error with rendering graph:\n{0}".format(e))
def _render_with_pygraphiz(input_data, output_path):
"""Renders graph using pygraphviz library."""
import pygraphviz as pgv
graph = pgv.AGraph(string=input_data)
graph.draw(output_path, prog='dot', format='png')

37
fuelclient/cli/arguments.py
View File

@@ -21,7 +21,7 @@ from fuelclient.cli.error import ArgumentException
from fuelclient.client import APIClient
substitutions = {
#replace from: to
# replace from: to
"env": "environment",
"nodes": "node",
"net": "network",
@@ -302,37 +302,49 @@ def get_net_arg(help_msg):
def get_graph_endpoint():
return get_arg(
'end',
flags=('--end',),
action="store",
default=None,
help="Specify endpoint for the graph traversal.")
help="Specify endpoint for the graph traversal.",
metavar='TASK',
)
def get_graph_startpoint():
return get_arg(
'start',
flags=('--start',),
action="store",
default=None,
help="Specify start point for the graph traversal.")
help="Specify start point for the graph traversal.",
metavar='TASK',
)
def get_skip_tasks():
return get_arg(
'skip',
flags=('--skip',),
nargs='+',
default=[],
help="Get list of tasks to be skipped.")
help="Get list of tasks to be skipped.",
metavar='TASK',
)
def get_tasks():
return get_arg(
'tasks',
flags=('--tasks',),
nargs='+',
default=[],
help="Get list of tasks to be executed.")
help="Get list of tasks to be executed.",
metavar='TASK',
)
def get_parents_arg():
return get_arg(
'parents-for',
help="Get parent for given task",
metavar='TASK',
)
def get_nst_arg(help_msg):
@@ -412,6 +424,13 @@ def get_release_arg(help_msg, required=False):
help=help_msg)
def get_render_arg(help_msg):
return get_str_arg(
"render",
metavar='INPUT',
help=help_msg)
def get_node_arg(help_msg):
default_kwargs = {
"action": NodeAction,

22
fuelclient/client.py
View File

@@ -133,9 +133,12 @@ class Client(object):
return resp.json()
@exceptions_decorator
def get_request(self, api, ostf=False, params=None):
"""Make GET request to specific API
def get_request_raw(self, api, ostf=False, params=None):
"""Make a GET request to specific API and return raw response.
:param api: API endpoint (path)
:param ostf: is this a call to OSTF API
:param params: params passed to GET request
"""
url = (self.ostf_root if ostf else self.api_root) + api
self.print_debug(
@@ -145,13 +148,24 @@ class Client(object):
headers = {'x-auth-token': self.auth_token}
params = params or {}
return requests.get(url, params=params, headers=headers)
resp = requests.get(url, params=params, headers=headers)
@exceptions_decorator
def get_request(self, api, ostf=False, params=None):
"""Make GET request to specific API
"""
resp = self.get_request_raw(api, ostf, params)
resp.raise_for_status()
return resp.json()
def post_request_raw(self, api, data, ostf=False):
"""Make a POST request to specific API and return raw response.
:param api: API endpoint (path)
:param data: data send in request, will be serialzied to JSON
:param ostf: is this a call to OSTF API
"""
url = (self.ostf_root if ostf else self.api_root) + api
data_json = json.dumps(data)
self.print_debug(

11
fuelclient/objects/environment.py
View File

@@ -30,6 +30,7 @@ class Environment(BaseObject):
class_api_path = "clusters/"
instance_api_path = "clusters/{0}/"
deployment_tasks_path = 'clusters/{0}/deployment_tasks'
deployment_tasks_graph_path = 'clusters/{0}/deploy_tasks/graph.gv'
@classmethod
def create(cls, name, release_id, net, net_segment_type=None):
@@ -400,3 +401,13 @@ class Environment(BaseObject):
def update_deployment_tasks(self, data):
url = self.deployment_tasks_path.format(self.id)
return self.connection.put_request(url, data)
def get_deployment_tasks_graph(self, tasks, parents_for=None):
url = self.deployment_tasks_graph_path.format(self.id)
params = {
'tasks': ','.join(tasks),
'parents_for': parents_for,
}
resp = self.connection.get_request_raw(url, params=params)
resp.raise_for_status()
return resp.text

64
fuelclient/tests/test_base_action.py Normal file
View File

@@ -0,0 +1,64 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from fuelclient.cli.actions import base
from fuelclient.cli import error
from fuelclient.tests import base as base_tests
class TestBaseAction(base_tests.UnitTestCase):
def setUp(self):
super(TestBaseAction, self).setUp()
self.action = base.Action()
@mock.patch('fuelclient.cli.actions.base.os')
def test_default_directory_with_param(self, m_os):
directory = 'some/dir'
self.action.default_directory(directory)
m_os.path.abspath.assert_called_once_with(directory)
@mock.patch('fuelclient.cli.actions.base.os')
def test_default_directory_without_param(self, m_os):
self.action.default_directory()
m_os.path.abspath.assert_called_once_with(m_os.curdir)
@mock.patch('fuelclient.cli.actions.base.os.mkdir')
@mock.patch('fuelclient.cli.actions.base.os.path.exists')
def test_full_path_directory(self, m_exists, m_mkdir):
m_exists.return_value = False
self.assertEqual(
self.action.full_path_directory('/base/path', 'subdir'),
'/base/path/subdir'
)
m_mkdir.assert_called_once_with('/base/path/subdir')
@mock.patch('fuelclient.cli.actions.base.os')
def test_full_path_directory_no_access(self, m_os):
exc_msg = 'Bas permissions'
m_os.path.exists.return_value = False
m_os.mkdir.side_effect = OSError(exc_msg)
with self.assertRaisesRegexp(error.ActionException, exc_msg):
self.action.full_path_directory('/base/path', 'subdir')
@mock.patch('fuelclient.cli.actions.base.os')
def test_full_path_directory_already_exists(self, m_os):
m_os.path.exists.return_value = True
self.action.full_path_directory('/base/path', 'subdir')
self.assertEqual(m_os.mkdir.call_count, 0)

203
fuelclient/tests/test_graph_action.py Normal file
View File

@@ -0,0 +1,203 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import io
import os
import mock
import requests_mock
from fuelclient.cli.actions import graph
from fuelclient.tests import base
GRAPH_API_OUTPUT = "digraph G { A -> B -> C }"
TASKS_API_OUTPUT = [
{'id': 'primary-controller'},
{'id': 'sync-time'},
]
class TestGraphAction(base.UnitTestCase):
def setUp(self):
super(TestGraphAction, self).setUp()
self.requests_mock = requests_mock.mock()
self.requests_mock.start()
self.m_tasks_api = self.requests_mock.get(
'/api/v1/clusters/1/deployment_tasks',
json=TASKS_API_OUTPUT)
self.m_graph_api = self.requests_mock.get(
'/api/v1/clusters/1/deploy_tasks/graph.gv',
text=GRAPH_API_OUTPUT)
self.m_full_path = mock.patch.object(graph.GraphAction,
'full_path_directory').start()
self.m_full_path.return_value = '/path'
def tearDown(self):
super(TestGraphAction, self).tearDown()
self.requests_mock.stop()
self.m_full_path.stop()
def test_download_all_tasks(self):
with mock.patch('sys.stdout', new=io.StringIO()) as m_stdout:
self.execute_wo_auth(
['fuel', 'graph', '--download', '--env', '1', '--download']
)
querystring = self.m_graph_api.last_request.qs
for task in TASKS_API_OUTPUT:
self.assertIn(task['id'], querystring['tasks'][0])
self.assertIn(GRAPH_API_OUTPUT, m_stdout.getvalue())
def test_download_selected_tasks(self):
with mock.patch('sys.stdout', new=io.StringIO()) as m_stdout:
self.execute_wo_auth(
['fuel', 'graph', '--download', '--env', '1',
'--tasks', 'task-a', 'task-b']
)
querystring = self.m_graph_api.last_request.qs
self.assertIn('task-a', querystring['tasks'][0])
self.assertIn('task-b', querystring['tasks'][0])
self.assertIn(GRAPH_API_OUTPUT, m_stdout.getvalue())
def test_download_with_skip(self):
with mock.patch('sys.stdout', new=io.StringIO()) as m_stdout:
self.execute_wo_auth(
['fuel', 'graph', '--download', '--env', '1',
'--skip', 'sync-time', 'task-b']
)
querystring = self.m_graph_api.last_request.qs
self.assertIn('primary-controller', querystring['tasks'][0])
self.assertNotIn('sync-time', querystring['tasks'][0])
self.assertNotIn('task-b', querystring['tasks'][0])
self.assertIn(GRAPH_API_OUTPUT, m_stdout.getvalue())
def test_download_with_end_and_start(self):
with mock.patch('sys.stdout', new=io.StringIO()) as m_stdout:
self.execute_wo_auth(
['fuel', 'graph', '--download', '--env', '1',
'--start', 'task-a', '--end', 'task-b']
)
tasks_qs = self.m_tasks_api.last_request.qs
self.assertEqual('task-a', tasks_qs['start'][0])
self.assertEqual('task-b', tasks_qs['end'][0])
graph_qs = self.m_graph_api.last_request.qs
for task in TASKS_API_OUTPUT:
self.assertIn(task['id'], graph_qs['tasks'][0])
self.assertIn(GRAPH_API_OUTPUT, m_stdout.getvalue())
def test_download_only_parents(self):
with mock.patch('sys.stdout', new=io.StringIO()) as m_stdout:
self.execute_wo_auth(
['fuel', 'graph', '--download', '--env', '1',
'--parents-for', 'task-z']
)
querystring = self.m_graph_api.last_request.qs
self.assertEqual('task-z', querystring['parents_for'][0])
self.assertIn(GRAPH_API_OUTPUT, m_stdout.getvalue())
def test_params_saved_in_dotfile(self):
with mock.patch('sys.stdout', new=io.StringIO()) as m_stdout:
self.execute_wo_auth(
['fuel', 'graph', '--download', '--env', '1',
'--parents-for', 'task-z',
'--skip', 'task-a']
)
saved_params = ("# params:\n"
"# - start: None\n"
"# - end: None\n"
"# - skip: ['task-a']\n"
"# - tasks: []\n"
"# - parents-for: task-z\n")
self.assertIn(saved_params + GRAPH_API_OUTPUT, m_stdout.getvalue())
@mock.patch('fuelclient.cli.actions.graph.open', create=True)
@mock.patch('fuelclient.cli.actions.graph.render_graph')
@mock.patch('fuelclient.cli.actions.graph.os.access')
@mock.patch('fuelclient.cli.actions.graph.os.path.exists')
def test_render(self, m_exists, m_access, m_render, m_open):
graph_data = 'some-dot-data'
m_exists.return_value = True
m_open().__enter__().read.return_value = graph_data
self.execute_wo_auth(
['fuel', 'graph', '--render', 'graph.gv']
)
m_open.assert_called_with('graph.gv', 'r')
m_render.assert_called_once_with(graph_data, '/path/graph.gv.png')
@mock.patch('fuelclient.cli.actions.graph.os.path.exists')
def test_render_no_file(self, m_exists):
m_exists.return_value = False
with self.assertRaises(SystemExit):
self.execute_wo_auth(
['fuel', 'graph', '--render', 'graph.gv']
)
@mock.patch('fuelclient.cli.actions.graph.open', create=True)
@mock.patch('fuelclient.cli.actions.graph.render_graph')
@mock.patch('fuelclient.cli.actions.graph.os.access')
@mock.patch('fuelclient.cli.actions.graph.os.path.exists')
def test_render_with_output_path(self, m_exists, m_access, m_render,
m_open):
output_dir = '/output/dir'
graph_data = 'some-dot-data'
m_exists.return_value = True
m_open().__enter__().read.return_value = graph_data
self.m_full_path.return_value = output_dir
self.execute_wo_auth(
['fuel', 'graph', '--render', 'graph.gv', '--dir', output_dir]
)
self.m_full_path.assert_called_once_with(output_dir, '')
m_render.assert_called_once_with(graph_data,
'/output/dir/graph.gv.png')
@mock.patch('fuelclient.cli.actions.graph.os.access')
@mock.patch('fuelclient.cli.actions.graph.render_graph')
def test_render_from_stdin(self, m_render, m_access):
graph_data = u'graph data'
with mock.patch('sys.stdin', new=io.StringIO(graph_data)):
self.execute_wo_auth(
['fuel', 'graph', '--render', '-', ]
)
m_render.assert_called_once_with(graph_data, '/path/graph.gv.png')
@mock.patch('fuelclient.cli.actions.graph.open', create=True)
@mock.patch('fuelclient.cli.actions.graph.os.path.exists')
@mock.patch('fuelclient.cli.actions.graph.os.access')
def test_render_no_access_to_output(self, m_access, m_exists, m_open):
m_exists.return_value = True
m_access.return_value = False
output_dir = '/output/dir'
self.m_full_path.return_value = output_dir
with self.assertRaises(SystemExit):
self.execute_wo_auth(
['fuel', 'graph', '--render', 'graph.gv', '--dir', output_dir]
)
m_access.assert_called_once_with(
os.path.join(output_dir, 'graph.gv.png'), os.W_OK)

1
test-requirements.txt
View File

@@ -5,3 +5,4 @@ nose2==0.4.1
nose-timer==0.2.0
pyprof2calltree==1.3.2
gprof2dot==2014.09.29
requests-mock>=0.6.0