Improve task_graph
Related-Blueprint: fuel-library-modularization Change-Id: I13f784229a5cd0c20a5496e8764a868750ed5c1d
This commit is contained in:
Dmitry Ilyin
parent
36f30ae7f1
commit
d29e56a100
@ -1,58 +0,0 @@
|
||||
usage: task_graph.py [-h] [--workbook] [--clear_workbook] [--dot] [--png]
|
||||
[--png_file PNG_FILE] [--debug] [--group GROUP]
|
||||
[--topology]
|
||||
FILE [FILE ...]
|
||||
|
||||
positional arguments:
|
||||
FILE The list of files of directories where tasks can be
|
||||
found
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
--workbook, -w Output the raw workbook
|
||||
--clear_workbook, -c Output the clear workbook
|
||||
--dot, -D Output the graph in dot format
|
||||
--png, -p Write the graph in png format (default)
|
||||
--open, -o Open the image after creation
|
||||
--png_file PNG_FILE, -f PNG_FILE
|
||||
Write graph image to this file
|
||||
--debug, -d Print debug messages
|
||||
--group GROUP, -g GROUP
|
||||
Group or stage to build the graph for
|
||||
--topology, -t Show the tasks topology (possible execution order)
|
||||
|
||||
This tools can be used to create a task graph image. Just pouint it
|
||||
at the folder where tasks.yaml files can be found.
|
||||
|
||||
> utils/task_graph/task_graph.py deployment/puppet/osnailyfacter/modular
|
||||
|
||||
It will create task_graph.png file in the current directroy.
|
||||
|
||||
You can also use -w and -c options to inspect the workbook yamls
|
||||
files and output graph as graphviz file with -o option.
|
||||
|
||||
You can filter graph by roles and stages with -g option like this
|
||||
> utils/task_graph/task_graph.py -g post_deployment deployment/puppet/osnailyfacter/modular
|
||||
|
||||
And you can use -t option to view the possible order of task execution
|
||||
with the current graph.
|
||||
|
||||
> utils/task_graph/task_graph.py -t deployment/puppet/osnailyfacter/modular
|
||||
|
||||
1 hiera primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo
|
||||
2 globals primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo
|
||||
3 logging primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo
|
||||
4 netconfig primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo
|
||||
5 firewall primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo
|
||||
6 hosts primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo
|
||||
7 top-role-compute compute
|
||||
8 top-role-primary-mongo primary-mongo
|
||||
9 top-role-mongo mongo
|
||||
10 cluster primary-controller, controller
|
||||
11 virtual_ips primary-controller, controller
|
||||
12 zabbix primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo
|
||||
13 top-role-ceph-osd ceph-osd
|
||||
14 top-role-cinder cinder
|
||||
15 tools primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo
|
||||
16 top-role-controller primary-controller, controller
|
||||
|
||||
@ -14,13 +14,60 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# usage: task_graph.py [-h] [--workbook] [--clear_workbook] [--dot] [--png]
|
||||
# [--png_file PNG_FILE] [--open] [--debug]
|
||||
# [--filter FILTER] [--topology]
|
||||
# PLACE [PLACE ...]
|
||||
#
|
||||
# positional arguments:
|
||||
# PLACE The list of files of directories wheretasks can be
|
||||
# found
|
||||
#
|
||||
# optional arguments:
|
||||
# -h, --help show this help message and exit
|
||||
# --workbook, -w Output the raw workbook
|
||||
# --clear_workbook, -c Output the clear workbook
|
||||
# --dot, -D Output the graph in dot format
|
||||
# --png, -p Write the graph in png format (default)
|
||||
# --png_file PNG_FILE, -P PNG_FILE
|
||||
# Write graph image to this file
|
||||
# --open, -o Open the image after creation
|
||||
# --debug, -d Print debug messages
|
||||
# --filter FILTER, -f FILTER
|
||||
# Filter tasks by this group or role
|
||||
# --topology, -t Show the tasks topology(possible execution order)
|
||||
#
|
||||
# This tools can be used to create a task graph image. Just point it
|
||||
# at the folder where tasks.yaml files can be found.
|
||||
#
|
||||
# > ./task_graph.py deployment/puppet/osnailyfacter/modular
|
||||
#
|
||||
# It will create task_graph.png file in the current directroy.
|
||||
#
|
||||
# You can also use -w and -c options to inspect the workbook yamls
|
||||
# files and output graph as graphviz file with -o option.
|
||||
#
|
||||
# You can filter graph by roles and groups with -f option like this
|
||||
# > ./task_graph.py -f controller deployment/puppet/osnailyfacter/modular
|
||||
#
|
||||
# And you can use -t option to view the possible order of task execution
|
||||
# with the current graph.
|
||||
#
|
||||
# > ./task_graph.py -t deployment/puppet/osnailyfacter/modular
|
||||
|
||||
import fnmatch
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import networkx
|
||||
import yaml
|
||||
|
||||
try:
|
||||
import pygraphviz
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
import networkx
|
||||
|
||||
|
||||
class IO(object):
|
||||
@classmethod
|
||||
@ -32,12 +79,11 @@ class IO(object):
|
||||
sys.stdout.write(msg)
|
||||
|
||||
@classmethod
|
||||
def output(cls, line, fill=None):
|
||||
def output(cls, line, fill=None, newline=True):
|
||||
line = str(line)
|
||||
if fill:
|
||||
line = line[0:fill].ljust(fill)
|
||||
else:
|
||||
if not line.endswith("\n"):
|
||||
if newline and not line.endswith("\n"):
|
||||
line += "\n"
|
||||
sys.stdout.write(line)
|
||||
|
||||
@ -73,7 +119,7 @@ class IO(object):
|
||||
action="store_true",
|
||||
default=True,
|
||||
help='Write the graph in png format (default)')
|
||||
parser.add_argument("--png_file", "-f",
|
||||
parser.add_argument("--png_file", "-P",
|
||||
type=str,
|
||||
default='task_graph.png',
|
||||
help='Write graph image to this file')
|
||||
@ -85,12 +131,9 @@ class IO(object):
|
||||
action="store_true",
|
||||
default=False,
|
||||
help='Print debug messages')
|
||||
parser.add_argument("--group", "-g",
|
||||
help="Group to build the graph for",
|
||||
parser.add_argument("--filter", "-f",
|
||||
help="Filter tasks by this group or role",
|
||||
default=None)
|
||||
parser.add_argument("--stage", "-s",
|
||||
help="Stage to build the graph for",
|
||||
default='deployment')
|
||||
parser.add_argument("--topology", "-t",
|
||||
action="store_true",
|
||||
help="Show the tasks topology"
|
||||
@ -125,17 +168,17 @@ class IO(object):
|
||||
for task_file in cls.task_files(place):
|
||||
task_graph.load_yaml_file(task_file)
|
||||
|
||||
task_graph.process_data(stage=cls.args.stage, group=cls.args.group)
|
||||
if cls.args.workbook:
|
||||
IO.output(yaml.dump(task_graph.workbook))
|
||||
return
|
||||
|
||||
task_graph.process_data(filter=cls.args.filter)
|
||||
task_graph.build_graph()
|
||||
|
||||
if cls.args.topology:
|
||||
task_graph.show_topology()
|
||||
return
|
||||
|
||||
if cls.args.workbook:
|
||||
IO.output(yaml.dump(task_graph.workbook))
|
||||
return
|
||||
|
||||
if cls.args.clear_workbook:
|
||||
IO.output(yaml.dump(task_graph.data))
|
||||
return
|
||||
@ -152,12 +195,20 @@ class TaskGraph(object):
|
||||
self.workbook = []
|
||||
self.graph = networkx.DiGraph()
|
||||
self._max_task_id_length = None
|
||||
self._max_task_stage_length = None
|
||||
|
||||
self.options = {
|
||||
'debug': False,
|
||||
'prog': 'dot',
|
||||
'default_node': {
|
||||
'fillcolor': 'yellow',
|
||||
},
|
||||
'stage_node': {
|
||||
'fillcolor': 'blue',
|
||||
'shape': 'rectangle',
|
||||
},
|
||||
'group_node': {
|
||||
'fillcolor': 'green',
|
||||
'shape': 'rectangle',
|
||||
},
|
||||
'default_edge': {
|
||||
},
|
||||
@ -171,16 +222,11 @@ class TaskGraph(object):
|
||||
'global_node': {
|
||||
'style': 'filled',
|
||||
'shape': 'ellipse',
|
||||
'fillcolor': 'yellow',
|
||||
},
|
||||
'global_edge': {
|
||||
'style': 'solid',
|
||||
'arrowhead': 'vee',
|
||||
},
|
||||
'non_task_types': [
|
||||
'role',
|
||||
'stage',
|
||||
]
|
||||
}
|
||||
|
||||
def clear(self):
|
||||
@ -189,46 +235,67 @@ class TaskGraph(object):
|
||||
self.workbook = []
|
||||
|
||||
def node_options(self, id):
|
||||
if self.data.get(id, {}).get('type', None) == 'stage':
|
||||
return self.options['stage_node']
|
||||
if self.data.get(id, {}).get('type', None) == 'group':
|
||||
return self.options['group_node']
|
||||
return self.options['default_node']
|
||||
|
||||
def edge_options(self, id_from, id_to):
|
||||
return self.options['default_edge']
|
||||
|
||||
def add_graph_node(self, id, options=None):
|
||||
if not options:
|
||||
options = self.node_options(id)
|
||||
if id not in self.data:
|
||||
return
|
||||
if not options:
|
||||
options = self.node_options(id)
|
||||
self.graph.add_node(id, options)
|
||||
|
||||
def add_graph_edge(self, id_from, id_to, options=None):
|
||||
if not options:
|
||||
options = self.edge_options(id_from, id_to)
|
||||
if id_from not in self.data:
|
||||
return
|
||||
if id_to not in self.data:
|
||||
return
|
||||
if not options:
|
||||
options = self.edge_options(id_from, id_to)
|
||||
self.graph.add_edge(id_from, id_to, options)
|
||||
|
||||
def process_data(self, stage=None, group=None):
|
||||
@staticmethod
|
||||
def filter_by_group(node, filter=None):
|
||||
# if group is not specified accept only the group tasks
|
||||
# and show only them on the graph/list
|
||||
# if there is a group, filter out group tasks
|
||||
# and show only normal tasks in this group
|
||||
type = node.get('type', None)
|
||||
if not filter:
|
||||
return type == 'group'
|
||||
else:
|
||||
if type == 'group':
|
||||
return False
|
||||
# always accept 'stage' tasks
|
||||
if type == 'stage':
|
||||
return True
|
||||
# accept task only if it has matching group or role
|
||||
# or its group or role is set to 'any'
|
||||
if 'groups' in node:
|
||||
if ('*' in node['groups']) or (filter in node['groups']):
|
||||
return True
|
||||
if 'role' in node:
|
||||
if ('*' in node['role']) or (filter in node['role']):
|
||||
return True
|
||||
return False
|
||||
|
||||
def process_data(self, filter=None):
|
||||
for node in self.workbook:
|
||||
if not type(node) is dict:
|
||||
continue
|
||||
if not node.get('type', None) and node.get('id', None):
|
||||
continue
|
||||
if node.get('type', None) in self.options['non_task_types']:
|
||||
continue
|
||||
if not 'stage' in node:
|
||||
node['stage'] = 'deployment'
|
||||
if not 'requires' in node:
|
||||
node['requires'] = []
|
||||
if not 'required_for' in node:
|
||||
node['required_for'] = []
|
||||
if not 'groups' in node:
|
||||
node['groups'] = []
|
||||
if stage and not node['stage'] == stage:
|
||||
continue
|
||||
if group and not group in node['groups']:
|
||||
if not self.filter_by_group(node, filter=filter):
|
||||
continue
|
||||
self.data[node['id']] = node
|
||||
|
||||
@ -247,30 +314,26 @@ class TaskGraph(object):
|
||||
self._max_task_id_length = len(max(self.data.keys(), key=len))
|
||||
return self._max_task_id_length
|
||||
|
||||
@property
|
||||
def max_task_stage_length(self):
|
||||
if self._max_task_stage_length:
|
||||
return self._max_task_stage_length
|
||||
self._max_task_stage_length = max(map(lambda n: len(n['stage']),
|
||||
self.data.values()))
|
||||
return self._max_task_stage_length
|
||||
|
||||
def make_dot_graph(self):
|
||||
return networkx.to_agraph(self.graph)
|
||||
|
||||
def show_topology(self):
|
||||
number = 1
|
||||
for node in networkx.topological_sort(self.graph):
|
||||
groups = self.data.get(node, {}).get('groups', [])
|
||||
groups = ', '.join(groups)
|
||||
stage = self.data.get(node, {}).get('stage', '')
|
||||
IO.output(number, 5)
|
||||
IO.output(node, self.max_task_id_length + 1)
|
||||
IO.output(stage, self.max_task_stage_length + 1)
|
||||
IO.output(groups)
|
||||
type = self.data.get(node, {}).get('type', None)
|
||||
if type == 'stage':
|
||||
node = '<' + node + '>'
|
||||
IO.output(number, fill=5, newline=False)
|
||||
IO.output(node, fill=self.max_task_id_length + 1)
|
||||
number += 1
|
||||
|
||||
def create_image(self, img_file):
|
||||
try:
|
||||
pygraphviz
|
||||
except NameError:
|
||||
IO.output('You need "pygraphviz" to draw the graph. '
|
||||
'But you can use -t to view the topology.')
|
||||
sys.exit(1)
|
||||
graph = self.make_dot_graph()
|
||||
for attr_name in self.options['global_graph']:
|
||||
graph.graph_attr[attr_name] = \
|
||||
|
||||
Loading…
Reference in New Issue
Block a user