Adds a process to generate key flow documentation

Octavia extensively uses TaskFlow flows for orchestration.
To make it easier for developers to understand these flows,
this patch adds a mechanism for generating graphviz representations
of the key Octavia TaskFlow flows.
It also updates our tox docs task to generate this documentation.

This patch depends on a fix to the TaskFlow export_to_dot method.

Added into conf.py as first step of configuration

Closes-Bug: #1561063
Change-Id: I914e1c062b400148565def37ccf618b3d2ea2573
Depends-On: I99f87af0b2bed959fcb43ef611b3186e23bd9549

.gitignore

@@ -7,6 +7,7 @@ cover/
 covhtml/
 dist/
 doc/build
+doc/source/devref/flow_diagrams/
 .idea/*
 *.DS_Store
 *.pyc

doc-requirements.txt

@@ -10,3 +10,8 @@ sphinxcontrib-blockdiag
 sphinxcontrib-nwdiag
 sphinxcontrib-seqdiag
 graphviz
+
+# This needs to be installed after above modules
+pydotplus
+pyparsing
+networkx

doc/source/conf.py

@@ -14,6 +14,11 @@
 import sys
 import subprocess
 import os
+from tools import create_flow_docs
+
+# Generate our flow diagrams
+create_flow_docs.generate(
+    'tools/flow-list.txt', 'doc/source/devref/flow_diagrams')
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the

doc/source/devref/flows.rst

@@ -0,0 +1,21 @@
+========================
+Octavia Controller Flows
+========================
+
+Octavia uses OpenStack TaskFlow to orchestrate the actions the Octavia
+controller needs to take while managing load balancers.
+
+This document is meant as a reference for the key flows used in the
+Octavia controller.
+
+.. toctree::
+    :maxdepth: 1
+
+    flow_diagrams/AmphoraFlows.rst
+    flow_diagrams/HealthMonitorFlows.rst
+    flow_diagrams/L7PolicyFlows.rst
+    flow_diagrams/L7RuleFlows.rst
+    flow_diagrams/ListenerFlows.rst
+    flow_diagrams/LoadBalancerFlows.rst
+    flow_diagrams/MemberFlows.rst
+    flow_diagrams/PoolFlows.rst

doc/source/index.rst

@@ -28,6 +28,7 @@ For developers
 
    main/CONSTITUTION.rst
    main/HACKING.rst
+   devref/flows.rst
 
 ====
 APIs

octavia/common/constants.py

@@ -316,3 +316,12 @@ AMPHORA_NAMESPACE = 'amphora-haproxy'
 # List of HTTP headers which are supported for insertion
 SUPPORTED_HTTP_HEADERS = ['X-Forwarded-For',
                           'X-Forwarded-Port']
+
+FLOW_DOC_TITLES = {'AmphoraFlows': 'Amphora Flows',
+                   'LoadBalancerFlows': 'Load Balancer Flows',
+                   'ListenerFlows': 'Listener Flows',
+                   'PoolFlows': 'Pool Flows',
+                   'MemberFlows': 'Member Flows',
+                   'HealthMonitorFlows': 'Health Monitor Flows',
+                   'L7PolicyFlows': 'Layer 7 Policy Flows',
+                   'L7RuleFlows': 'Layer 7 Rule Flows'}

tools/create_flow_docs.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python
+# Copyright 2016 Hewlett Packard Enterprise Development Company LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+import argparse
+import importlib
+import os
+
+import graphviz
+from taskflow import engines
+
+from octavia.common import constants
+from octavia.tests.common import data_model_helpers as dmh
+
+
+def main():
+    arg_parser = argparse.ArgumentParser(
+        description='Generate graphviz representations of the '
+                    'Octavia TaskFlow flows.')
+    arg_parser.add_argument('-f', '--flow-list', required=True,
+                            help='Path to flow list file')
+    arg_parser.add_argument('-o', '--output-directory', required=True,
+                            help='Path to flow list file')
+    args = arg_parser.parse_args()
+    generate(args.flow_list, args.output_directory)
+
+
+def generate(flow_list, output_directory):
+    # Create the diagrams
+    base_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                             os.path.pardir)
+    diagram_list = []
+    with open(os.path.join(base_path, flow_list), 'r') as flowlist:
+        for row in flowlist:
+            if row.startswith('#'):
+                continue
+            current_tuple = tuple(row.strip().split(' '))
+            current_class = getattr(importlib.import_module(current_tuple[0]),
+                                    current_tuple[1])
+            current_instance = current_class()
+            get_flow_method = getattr(current_instance, current_tuple[2])
+            if (current_tuple[1] == 'LoadBalancerFlows' and
+                    current_tuple[2] == 'get_create_load_balancer_flow'):
+                current_engine = engines.load(
+                    get_flow_method(
+                        constants.TOPOLOGY_ACTIVE_STANDBY))
+            elif (current_tuple[1] == 'LoadBalancerFlows' and
+                  current_tuple[2] == 'get_create_load_balancer_graph_flows'):
+                # This is lame until we refactor the create load balancer
+                # flow into one flow now that
+                # https://bugs.launchpad.net/taskflow/+bug/1479466
+                # is fixed.
+                allocate_amp_flow, lb_create_graph_flow = get_flow_method(
+                    constants.TOPOLOGY_ACTIVE_STANDBY, 'prefixname')
+                current_engine = engines.load(lb_create_graph_flow)
+            elif (current_tuple[1] == 'LoadBalancerFlows' and
+                  current_tuple[2] == 'get_delete_load_balancer_flow'):
+                lb = dmh.generate_load_balancer()
+                delete_flow, store = get_flow_method(lb)
+                current_engine = engines.load(delete_flow)
+            elif (current_tuple[1] == 'LoadBalancerFlows' and
+                  current_tuple[2] == 'get_cascade_delete_load_balancer_flow'):
+                lb = dmh.generate_load_balancer()
+                delete_flow, store = get_flow_method(lb)
+                current_engine = engines.load(delete_flow)
+            else:
+                current_engine = engines.load(get_flow_method())
+            current_engine.compile()
+            # We need to render svg and not dot here so we can scale
+            # the image in the restructured text page
+            src = graphviz.Source(current_engine.compilation.
+                                  execution_graph.export_to_dot())
+            src.format = 'svg'
+            src.render(filename=current_tuple[1] + '-' + current_tuple[2],
+                       directory=os.path.join(base_path, output_directory),
+                       cleanup=True)
+            diagram_list.append((current_tuple[1], current_tuple[2]))
+
+    # Create the class docs
+    diagram_list = sorted(diagram_list, key=getDiagKey)
+    class_tracker = None
+    current_doc_file = None
+    for doc_tuple in diagram_list:
+        # If we are still working on the same class, append
+        if doc_tuple[0] == class_tracker:
+            current_doc_file.write('\n')
+            current_doc_file.write(doc_tuple[1] + '\n')
+            current_doc_file.write('-' * len(doc_tuple[1]) + '\n')
+            current_doc_file.write('\n')
+            current_doc_file.write('.. image:: ' + doc_tuple[0] +
+                                   '-' + doc_tuple[1] + '.svg\n')
+            current_doc_file.write('    :width: 660px\n')
+            current_doc_file.write('    :target: ../../_images/' +
+                                   doc_tuple[0] +
+                                   '-' + doc_tuple[1] + '.svg\n')
+
+        # First or new class, create the file
+        else:
+            if current_doc_file is not None:
+                current_doc_file.close()
+            current_doc_file = open(os.path.join(
+                base_path, output_directory, doc_tuple[0] + '.rst'), 'w+')
+            class_tracker = doc_tuple[0]
+
+            file_title = constants.FLOW_DOC_TITLES.get(doc_tuple[0],
+                                                       'Unknown Flows')
+
+            current_doc_file.write('=' * len(file_title) + '\n')
+            current_doc_file.write(file_title + '\n')
+            current_doc_file.write('=' * len(file_title) + '\n')
+            current_doc_file.write('\n')
+            current_doc_file.write('.. contents::\n')
+            current_doc_file.write('   :depth: 2\n')
+            current_doc_file.write('   :backlinks: top\n')
+            current_doc_file.write('\n')
+            current_doc_file.write('Click on any flow to view full size.\n')
+            current_doc_file.write('\n')
+            current_doc_file.write(doc_tuple[1] + '\n')
+            current_doc_file.write('-' * len(doc_tuple[1]) + '\n')
+            current_doc_file.write('\n')
+            current_doc_file.write('.. image:: ' + doc_tuple[0] +
+                                   '-' + doc_tuple[1] + '.svg\n')
+            current_doc_file.write('    :width: 660px\n')
+            current_doc_file.write('    :target: ../../_images/' +
+                                   doc_tuple[0] +
+                                   '-' + doc_tuple[1] + '.svg\n')
+
+    current_doc_file.close()
+
+
+def getDiagKey(item):
+    return item[0] + '-' + item[1]
+
+if __name__ == "__main__":
+    main()

tools/flow-list.txt

@@ -0,0 +1,31 @@
+# List of TaskFlow flows that should be documented
+# Some flows are used by other flows, so just list the primary flows here
+# Format:
+# module class flow
+octavia.controller.worker.flows.amphora_flows AmphoraFlows get_create_amphora_flow
+octavia.controller.worker.flows.amphora_flows AmphoraFlows get_failover_flow
+octavia.controller.worker.flows.amphora_flows AmphoraFlows cert_rotate_amphora_flow
+octavia.controller.worker.flows.load_balancer_flows LoadBalancerFlows get_create_load_balancer_flow
+octavia.controller.worker.flows.load_balancer_flows LoadBalancerFlows get_create_load_balancer_graph_flows
+octavia.controller.worker.flows.load_balancer_flows LoadBalancerFlows get_delete_load_balancer_flow
+octavia.controller.worker.flows.load_balancer_flows LoadBalancerFlows get_cascade_delete_load_balancer_flow
+octavia.controller.worker.flows.load_balancer_flows LoadBalancerFlows get_update_load_balancer_flow
+octavia.controller.worker.flows.listener_flows ListenerFlows get_create_listener_flow
+octavia.controller.worker.flows.listener_flows ListenerFlows get_create_all_listeners_flow
+octavia.controller.worker.flows.listener_flows ListenerFlows get_delete_listener_flow
+octavia.controller.worker.flows.listener_flows ListenerFlows get_update_listener_flow
+octavia.controller.worker.flows.pool_flows PoolFlows get_create_pool_flow
+octavia.controller.worker.flows.pool_flows PoolFlows get_delete_pool_flow
+octavia.controller.worker.flows.pool_flows PoolFlows get_update_pool_flow
+octavia.controller.worker.flows.member_flows MemberFlows get_create_member_flow
+octavia.controller.worker.flows.member_flows MemberFlows get_delete_member_flow
+octavia.controller.worker.flows.member_flows MemberFlows get_update_member_flow
+octavia.controller.worker.flows.health_monitor_flows HealthMonitorFlows get_create_health_monitor_flow
+octavia.controller.worker.flows.health_monitor_flows HealthMonitorFlows get_delete_health_monitor_flow
+octavia.controller.worker.flows.health_monitor_flows HealthMonitorFlows get_update_health_monitor_flow
+octavia.controller.worker.flows.l7policy_flows L7PolicyFlows get_create_l7policy_flow
+octavia.controller.worker.flows.l7policy_flows L7PolicyFlows get_delete_l7policy_flow
+octavia.controller.worker.flows.l7policy_flows L7PolicyFlows get_update_l7policy_flow
+octavia.controller.worker.flows.l7rule_flows L7RuleFlows get_create_l7rule_flow
+octavia.controller.worker.flows.l7rule_flows L7RuleFlows get_delete_l7rule_flow
+octavia.controller.worker.flows.l7rule_flows L7RuleFlows get_update_l7rule_flow

tox.ini

@@ -43,7 +43,8 @@ commands = flake8
            CONSTITUTION.rst HACKING.rst README.rst
 
 [testenv:docs]
-commands = python setup.py build_sphinx
+commands =
+    python setup.py build_sphinx
 
 [testenv:venv]
 # TODO(ihrachys): remove once infra supports constraints for this target