Improve task_graph

Related-Blueprint: fuel-library-modularization
Change-Id: I13f784229a5cd0c20a5496e8764a868750ed5c1d

utils/task_graph/readme.txt

@@ -1,58 +0,0 @@
-usage: task_graph.py [-h] [--workbook] [--clear_workbook] [--dot] [--png]
-                     [--png_file PNG_FILE] [--debug] [--group GROUP]
-                     [--topology]
-                     FILE [FILE ...]
-
-positional arguments:
-  FILE                  The list of files of directories where tasks can be
-                        found
-
-optional arguments:
-  -h, --help            show this help message and exit
-  --workbook, -w        Output the raw workbook
-  --clear_workbook, -c  Output the clear workbook
-  --dot, -D             Output the graph in dot format
-  --png, -p             Write the graph in png format (default)
-  --open, -o            Open the image after creation
-  --png_file PNG_FILE, -f PNG_FILE
-                        Write graph image to this file
-  --debug, -d           Print debug messages
-  --group GROUP, -g GROUP
-                        Group or stage to build the graph for
-  --topology, -t        Show the tasks topology (possible execution order)
-
-This tools can be used to create a task graph image. Just pouint it
-at the folder where tasks.yaml files can be found.
-
-> utils/task_graph/task_graph.py deployment/puppet/osnailyfacter/modular
-
-It will create task_graph.png file in the current directroy.
-
-You can also use -w and -c options to inspect the workbook yamls
-files and output graph as graphviz file with -o option.
-
-You can filter graph by roles and stages with -g option like this
-> utils/task_graph/task_graph.py -g post_deployment deployment/puppet/osnailyfacter/modular
-
-And you can use -t option to view the possible order of task execution
-with the current graph.
-
-> utils/task_graph/task_graph.py -t deployment/puppet/osnailyfacter/modular
-
-1   hiera                  primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo
-2   globals                primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo
-3   logging                primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo
-4   netconfig              primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo
-5   firewall               primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo
-6   hosts                  primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo
-7   top-role-compute       compute
-8   top-role-primary-mongo primary-mongo
-9   top-role-mongo         mongo
-10  cluster                primary-controller, controller
-11  virtual_ips            primary-controller, controller
-12  zabbix                 primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo
-13  top-role-ceph-osd      ceph-osd
-14  top-role-cinder        cinder
-15  tools                  primary-controller, controller, cinder, compute, ceph-osd, zabbix-server, primary-mongo, mongo
-16  top-role-controller    primary-controller, controller
-

utils/task_graph/task_graph.py

@@ -14,13 +14,60 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+# usage: task_graph.py [-h] [--workbook] [--clear_workbook] [--dot] [--png]
+#                      [--png_file PNG_FILE] [--open] [--debug]
+#                      [--filter FILTER] [--topology]
+#                      PLACE [PLACE ...]
+#
+# positional arguments:
+#   PLACE                 The list of files of directories wheretasks can be
+#                         found
+#
+# optional arguments:
+#   -h, --help            show this help message and exit
+#   --workbook, -w        Output the raw workbook
+#   --clear_workbook, -c  Output the clear workbook
+#   --dot, -D             Output the graph in dot format
+#   --png, -p             Write the graph in png format (default)
+#   --png_file PNG_FILE, -P PNG_FILE
+#                         Write graph image to this file
+#   --open, -o            Open the image after creation
+#   --debug, -d           Print debug messages
+#   --filter FILTER, -f FILTER
+#                         Filter tasks by this group or role
+#   --topology, -t        Show the tasks topology(possible execution order)
+#
+# This tools can be used to create a task graph image. Just point it
+# at the folder where tasks.yaml files can be found.
+#
+# > ./task_graph.py deployment/puppet/osnailyfacter/modular
+#
+# It will create task_graph.png file in the current directroy.
+#
+# You can also use -w and -c options to inspect the workbook yamls
+# files and output graph as graphviz file with -o option.
+#
+# You can filter graph by roles and groups with -f option like this
+# > ./task_graph.py -f controller deployment/puppet/osnailyfacter/modular
+#
+# And you can use -t option to view the possible order of task execution
+# with the current graph.
+#
+# > ./task_graph.py -t deployment/puppet/osnailyfacter/modular
+
 import fnmatch
 import os
 import sys
 import argparse
-import networkx
 import yaml
 
+try:
+    import pygraphviz
+except ImportError:
+    pass
+
+import networkx
+
 
 class IO(object):
     @classmethod
@@ -32,12 +79,11 @@ class IO(object):
         sys.stdout.write(msg)
 
     @classmethod
-    def output(cls, line, fill=None):
+    def output(cls, line, fill=None, newline=True):
         line = str(line)
         if fill:
             line = line[0:fill].ljust(fill)
-        else:
+        if newline and not line.endswith("\n"):
-            if not line.endswith("\n"):
             line += "\n"
         sys.stdout.write(line)
 
@@ -73,7 +119,7 @@ class IO(object):
                             action="store_true",
                             default=True,
                             help='Write the graph in png format (default)')
-        parser.add_argument("--png_file", "-f",
+        parser.add_argument("--png_file", "-P",
                             type=str,
                             default='task_graph.png',
                             help='Write graph image to this file')
@@ -85,12 +131,9 @@ class IO(object):
                             action="store_true",
                             default=False,
                             help='Print debug messages')
-        parser.add_argument("--group", "-g",
+        parser.add_argument("--filter", "-f",
-                            help="Group to build the graph for",
+                            help="Filter tasks by this group or role",
                             default=None)
-        parser.add_argument("--stage", "-s",
-                            help="Stage to build the graph for",
-                            default='deployment')
         parser.add_argument("--topology", "-t",
                             action="store_true",
                             help="Show the tasks topology"
@@ -125,17 +168,17 @@ class IO(object):
             for task_file in cls.task_files(place):
                 task_graph.load_yaml_file(task_file)
 
-        task_graph.process_data(stage=cls.args.stage, group=cls.args.group)
+        if cls.args.workbook:
+            IO.output(yaml.dump(task_graph.workbook))
+            return
+
+        task_graph.process_data(filter=cls.args.filter)
         task_graph.build_graph()
 
         if cls.args.topology:
             task_graph.show_topology()
             return
 
-        if cls.args.workbook:
-            IO.output(yaml.dump(task_graph.workbook))
-            return
-
         if cls.args.clear_workbook:
             IO.output(yaml.dump(task_graph.data))
             return
@@ -152,12 +195,20 @@ class TaskGraph(object):
         self.workbook = []
         self.graph = networkx.DiGraph()
         self._max_task_id_length = None
-        self._max_task_stage_length = None
 
         self.options = {
             'debug': False,
             'prog': 'dot',
             'default_node': {
+                'fillcolor': 'yellow',
+            },
+            'stage_node': {
+                'fillcolor': 'blue',
+                'shape': 'rectangle',
+            },
+            'group_node': {
+                'fillcolor': 'green',
+                'shape': 'rectangle',
             },
             'default_edge': {
             },
@@ -171,16 +222,11 @@ class TaskGraph(object):
             'global_node': {
                 'style': 'filled',
                 'shape': 'ellipse',
-                'fillcolor': 'yellow',
             },
             'global_edge': {
                 'style': 'solid',
                 'arrowhead': 'vee',
             },
-            'non_task_types': [
-                'role',
-                'stage',
-            ]
         }
 
     def clear(self):
@@ -189,46 +235,67 @@ class TaskGraph(object):
         self.workbook = []
 
     def node_options(self, id):
+        if self.data.get(id, {}).get('type', None) == 'stage':
+            return self.options['stage_node']
+        if self.data.get(id, {}).get('type', None) == 'group':
+            return self.options['group_node']
         return self.options['default_node']
 
     def edge_options(self, id_from, id_to):
         return self.options['default_edge']
 
     def add_graph_node(self, id, options=None):
-        if not options:
-            options = self.node_options(id)
         if id not in self.data:
             return
+        if not options:
+            options = self.node_options(id)
         self.graph.add_node(id, options)
 
     def add_graph_edge(self, id_from, id_to, options=None):
-        if not options:
-            options = self.edge_options(id_from, id_to)
         if id_from not in self.data:
             return
         if id_to not in self.data:
             return
+        if not options:
+            options = self.edge_options(id_from, id_to)
         self.graph.add_edge(id_from, id_to, options)
 
-    def process_data(self, stage=None, group=None):
+    @staticmethod
+    def filter_by_group(node, filter=None):
+        # if group is not specified accept only the group tasks
+        # and show only them on the graph/list
+        # if there is a group, filter out group tasks
+        # and show only normal tasks in this group
+        type = node.get('type', None)
+        if not filter:
+            return type == 'group'
+        else:
+            if type == 'group':
+                return False
+        # always accept 'stage' tasks
+        if type == 'stage':
+            return True
+        # accept task only if it has matching group or role
+        # or its group or role is set to 'any'
+        if 'groups' in node:
+            if ('*' in node['groups']) or (filter in node['groups']):
+                return True
+        if 'role' in node:
+            if ('*' in node['role']) or (filter in node['role']):
+                return True
+        return False
+
+    def process_data(self, filter=None):
         for node in self.workbook:
             if not type(node) is dict:
                 continue
             if not node.get('type', None) and node.get('id', None):
                 continue
-            if node.get('type', None) in self.options['non_task_types']:
-                continue
-            if not 'stage' in node:
-                node['stage'] = 'deployment'
             if not 'requires' in node:
                 node['requires'] = []
             if not 'required_for' in node:
                 node['required_for'] = []
-            if not 'groups' in node:
+            if not self.filter_by_group(node, filter=filter):
-                node['groups'] = []
-            if stage and not node['stage'] == stage:
-                continue
-            if group and not group in node['groups']:
                 continue
             self.data[node['id']] = node
 
@@ -247,30 +314,26 @@ class TaskGraph(object):
         self._max_task_id_length = len(max(self.data.keys(), key=len))
         return self._max_task_id_length
 
-    @property
-    def max_task_stage_length(self):
-        if self._max_task_stage_length:
-            return self._max_task_stage_length
-        self._max_task_stage_length = max(map(lambda n: len(n['stage']),
-                                              self.data.values()))
-        return self._max_task_stage_length
-
     def make_dot_graph(self):
         return networkx.to_agraph(self.graph)
 
     def show_topology(self):
         number = 1
         for node in networkx.topological_sort(self.graph):
-            groups = self.data.get(node, {}).get('groups', [])
+            type = self.data.get(node, {}).get('type', None)
-            groups = ', '.join(groups)
+            if type == 'stage':
-            stage = self.data.get(node, {}).get('stage', '')
+                node = '<' + node + '>'
-            IO.output(number, 5)
+            IO.output(number, fill=5, newline=False)
-            IO.output(node, self.max_task_id_length + 1)
+            IO.output(node, fill=self.max_task_id_length + 1)
-            IO.output(stage, self.max_task_stage_length + 1)
-            IO.output(groups)
             number += 1
 
     def create_image(self, img_file):
+        try:
+            pygraphviz
+        except NameError:
+            IO.output('You need "pygraphviz" to draw the graph. '
+                      'But you can use -t to view the topology.')
+            sys.exit(1)
         graph = self.make_dot_graph()
         for attr_name in self.options['global_graph']:
             graph.graph_attr[attr_name] = \