From bc2e9beb3a8a41b773c1a5f16baabf4cf8ed4e18 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Thu, 24 Mar 2016 18:03:27 +0000 Subject: [PATCH] Adds a process to generate key flow documentation Octavia extensively uses TaskFlow flows for orchestration. To make it easier for developers to understand these flows, this patch adds a mechanism for generating graphviz representations of the key Octavia TaskFlow flows. It also updates our tox docs task to generate this documentation. This patch depends on a fix to the TaskFlow export_to_dot method. Added into conf.py as first step of configuration Closes-Bug: #1561063 Change-Id: I914e1c062b400148565def37ccf618b3d2ea2573 Depends-On: I99f87af0b2bed959fcb43ef611b3186e23bd9549 --- .gitignore | 1 + doc-requirements.txt | 5 ++ doc/source/conf.py | 5 ++ doc/source/devref/flows.rst | 21 ++++++ doc/source/index.rst | 1 + octavia/common/constants.py | 9 +++ tools/__init__.py | 0 tools/create_flow_docs.py | 147 ++++++++++++++++++++++++++++++++++++ tools/flow-list.txt | 31 ++++++++ tox.ini | 3 +- 10 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 doc/source/devref/flows.rst create mode 100644 tools/__init__.py create mode 100755 tools/create_flow_docs.py create mode 100644 tools/flow-list.txt diff --git a/.gitignore b/.gitignore index bbe0afbad8..1fb9d8a8a3 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ cover/ covhtml/ dist/ doc/build +doc/source/devref/flow_diagrams/ .idea/* *.DS_Store *.pyc diff --git a/doc-requirements.txt b/doc-requirements.txt index 822e660cab..d4ace281a6 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -10,3 +10,8 @@ sphinxcontrib-blockdiag sphinxcontrib-nwdiag sphinxcontrib-seqdiag graphviz + +# This needs to be installed after above modules +pydotplus +pyparsing +networkx diff --git a/doc/source/conf.py b/doc/source/conf.py index 670d44b193..bf535e54f7 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -14,6 +14,11 @@ import sys import subprocess import os +from tools import create_flow_docs + +# Generate our flow diagrams +create_flow_docs.generate( + 'tools/flow-list.txt', 'doc/source/devref/flow_diagrams') # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the diff --git a/doc/source/devref/flows.rst b/doc/source/devref/flows.rst new file mode 100644 index 0000000000..04295f3dc3 --- /dev/null +++ b/doc/source/devref/flows.rst @@ -0,0 +1,21 @@ +======================== +Octavia Controller Flows +======================== + +Octavia uses OpenStack TaskFlow to orchestrate the actions the Octavia +controller needs to take while managing load balancers. + +This document is meant as a reference for the key flows used in the +Octavia controller. + +.. toctree:: + :maxdepth: 1 + + flow_diagrams/AmphoraFlows.rst + flow_diagrams/HealthMonitorFlows.rst + flow_diagrams/L7PolicyFlows.rst + flow_diagrams/L7RuleFlows.rst + flow_diagrams/ListenerFlows.rst + flow_diagrams/LoadBalancerFlows.rst + flow_diagrams/MemberFlows.rst + flow_diagrams/PoolFlows.rst diff --git a/doc/source/index.rst b/doc/source/index.rst index 43b6267239..d283fdb0f2 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -28,6 +28,7 @@ For developers main/CONSTITUTION.rst main/HACKING.rst + devref/flows.rst ==== APIs diff --git a/octavia/common/constants.py b/octavia/common/constants.py index f1e4c58448..de08e0faa8 100644 --- a/octavia/common/constants.py +++ b/octavia/common/constants.py @@ -316,3 +316,12 @@ AMPHORA_NAMESPACE = 'amphora-haproxy' # List of HTTP headers which are supported for insertion SUPPORTED_HTTP_HEADERS = ['X-Forwarded-For', 'X-Forwarded-Port'] + +FLOW_DOC_TITLES = {'AmphoraFlows': 'Amphora Flows', + 'LoadBalancerFlows': 'Load Balancer Flows', + 'ListenerFlows': 'Listener Flows', + 'PoolFlows': 'Pool Flows', + 'MemberFlows': 'Member Flows', + 'HealthMonitorFlows': 'Health Monitor Flows', + 'L7PolicyFlows': 'Layer 7 Policy Flows', + 'L7RuleFlows': 'Layer 7 Rule Flows'} diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/create_flow_docs.py b/tools/create_flow_docs.py new file mode 100755 index 0000000000..8297e26488 --- /dev/null +++ b/tools/create_flow_docs.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python +# Copyright 2016 Hewlett Packard Enterprise Development Company LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +import argparse +import importlib +import os + +import graphviz +from taskflow import engines + +from octavia.common import constants +from octavia.tests.common import data_model_helpers as dmh + + +def main(): + arg_parser = argparse.ArgumentParser( + description='Generate graphviz representations of the ' + 'Octavia TaskFlow flows.') + arg_parser.add_argument('-f', '--flow-list', required=True, + help='Path to flow list file') + arg_parser.add_argument('-o', '--output-directory', required=True, + help='Path to flow list file') + args = arg_parser.parse_args() + generate(args.flow_list, args.output_directory) + + +def generate(flow_list, output_directory): + # Create the diagrams + base_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), + os.path.pardir) + diagram_list = [] + with open(os.path.join(base_path, flow_list), 'r') as flowlist: + for row in flowlist: + if row.startswith('#'): + continue + current_tuple = tuple(row.strip().split(' ')) + current_class = getattr(importlib.import_module(current_tuple[0]), + current_tuple[1]) + current_instance = current_class() + get_flow_method = getattr(current_instance, current_tuple[2]) + if (current_tuple[1] == 'LoadBalancerFlows' and + current_tuple[2] == 'get_create_load_balancer_flow'): + current_engine = engines.load( + get_flow_method( + constants.TOPOLOGY_ACTIVE_STANDBY)) + elif (current_tuple[1] == 'LoadBalancerFlows' and + current_tuple[2] == 'get_create_load_balancer_graph_flows'): + # This is lame until we refactor the create load balancer + # flow into one flow now that + # https://bugs.launchpad.net/taskflow/+bug/1479466 + # is fixed. + allocate_amp_flow, lb_create_graph_flow = get_flow_method( + constants.TOPOLOGY_ACTIVE_STANDBY, 'prefixname') + current_engine = engines.load(lb_create_graph_flow) + elif (current_tuple[1] == 'LoadBalancerFlows' and + current_tuple[2] == 'get_delete_load_balancer_flow'): + lb = dmh.generate_load_balancer() + delete_flow, store = get_flow_method(lb) + current_engine = engines.load(delete_flow) + elif (current_tuple[1] == 'LoadBalancerFlows' and + current_tuple[2] == 'get_cascade_delete_load_balancer_flow'): + lb = dmh.generate_load_balancer() + delete_flow, store = get_flow_method(lb) + current_engine = engines.load(delete_flow) + else: + current_engine = engines.load(get_flow_method()) + current_engine.compile() + # We need to render svg and not dot here so we can scale + # the image in the restructured text page + src = graphviz.Source(current_engine.compilation. + execution_graph.export_to_dot()) + src.format = 'svg' + src.render(filename=current_tuple[1] + '-' + current_tuple[2], + directory=os.path.join(base_path, output_directory), + cleanup=True) + diagram_list.append((current_tuple[1], current_tuple[2])) + + # Create the class docs + diagram_list = sorted(diagram_list, key=getDiagKey) + class_tracker = None + current_doc_file = None + for doc_tuple in diagram_list: + # If we are still working on the same class, append + if doc_tuple[0] == class_tracker: + current_doc_file.write('\n') + current_doc_file.write(doc_tuple[1] + '\n') + current_doc_file.write('-' * len(doc_tuple[1]) + '\n') + current_doc_file.write('\n') + current_doc_file.write('.. image:: ' + doc_tuple[0] + + '-' + doc_tuple[1] + '.svg\n') + current_doc_file.write(' :width: 660px\n') + current_doc_file.write(' :target: ../../_images/' + + doc_tuple[0] + + '-' + doc_tuple[1] + '.svg\n') + + # First or new class, create the file + else: + if current_doc_file is not None: + current_doc_file.close() + current_doc_file = open(os.path.join( + base_path, output_directory, doc_tuple[0] + '.rst'), 'w+') + class_tracker = doc_tuple[0] + + file_title = constants.FLOW_DOC_TITLES.get(doc_tuple[0], + 'Unknown Flows') + + current_doc_file.write('=' * len(file_title) + '\n') + current_doc_file.write(file_title + '\n') + current_doc_file.write('=' * len(file_title) + '\n') + current_doc_file.write('\n') + current_doc_file.write('.. contents::\n') + current_doc_file.write(' :depth: 2\n') + current_doc_file.write(' :backlinks: top\n') + current_doc_file.write('\n') + current_doc_file.write('Click on any flow to view full size.\n') + current_doc_file.write('\n') + current_doc_file.write(doc_tuple[1] + '\n') + current_doc_file.write('-' * len(doc_tuple[1]) + '\n') + current_doc_file.write('\n') + current_doc_file.write('.. image:: ' + doc_tuple[0] + + '-' + doc_tuple[1] + '.svg\n') + current_doc_file.write(' :width: 660px\n') + current_doc_file.write(' :target: ../../_images/' + + doc_tuple[0] + + '-' + doc_tuple[1] + '.svg\n') + + current_doc_file.close() + + +def getDiagKey(item): + return item[0] + '-' + item[1] + +if __name__ == "__main__": + main() diff --git a/tools/flow-list.txt b/tools/flow-list.txt new file mode 100644 index 0000000000..78331283a1 --- /dev/null +++ b/tools/flow-list.txt @@ -0,0 +1,31 @@ +# List of TaskFlow flows that should be documented +# Some flows are used by other flows, so just list the primary flows here +# Format: +# module class flow +octavia.controller.worker.flows.amphora_flows AmphoraFlows get_create_amphora_flow +octavia.controller.worker.flows.amphora_flows AmphoraFlows get_failover_flow +octavia.controller.worker.flows.amphora_flows AmphoraFlows cert_rotate_amphora_flow +octavia.controller.worker.flows.load_balancer_flows LoadBalancerFlows get_create_load_balancer_flow +octavia.controller.worker.flows.load_balancer_flows LoadBalancerFlows get_create_load_balancer_graph_flows +octavia.controller.worker.flows.load_balancer_flows LoadBalancerFlows get_delete_load_balancer_flow +octavia.controller.worker.flows.load_balancer_flows LoadBalancerFlows get_cascade_delete_load_balancer_flow +octavia.controller.worker.flows.load_balancer_flows LoadBalancerFlows get_update_load_balancer_flow +octavia.controller.worker.flows.listener_flows ListenerFlows get_create_listener_flow +octavia.controller.worker.flows.listener_flows ListenerFlows get_create_all_listeners_flow +octavia.controller.worker.flows.listener_flows ListenerFlows get_delete_listener_flow +octavia.controller.worker.flows.listener_flows ListenerFlows get_update_listener_flow +octavia.controller.worker.flows.pool_flows PoolFlows get_create_pool_flow +octavia.controller.worker.flows.pool_flows PoolFlows get_delete_pool_flow +octavia.controller.worker.flows.pool_flows PoolFlows get_update_pool_flow +octavia.controller.worker.flows.member_flows MemberFlows get_create_member_flow +octavia.controller.worker.flows.member_flows MemberFlows get_delete_member_flow +octavia.controller.worker.flows.member_flows MemberFlows get_update_member_flow +octavia.controller.worker.flows.health_monitor_flows HealthMonitorFlows get_create_health_monitor_flow +octavia.controller.worker.flows.health_monitor_flows HealthMonitorFlows get_delete_health_monitor_flow +octavia.controller.worker.flows.health_monitor_flows HealthMonitorFlows get_update_health_monitor_flow +octavia.controller.worker.flows.l7policy_flows L7PolicyFlows get_create_l7policy_flow +octavia.controller.worker.flows.l7policy_flows L7PolicyFlows get_delete_l7policy_flow +octavia.controller.worker.flows.l7policy_flows L7PolicyFlows get_update_l7policy_flow +octavia.controller.worker.flows.l7rule_flows L7RuleFlows get_create_l7rule_flow +octavia.controller.worker.flows.l7rule_flows L7RuleFlows get_delete_l7rule_flow +octavia.controller.worker.flows.l7rule_flows L7RuleFlows get_update_l7rule_flow diff --git a/tox.ini b/tox.ini index 6d562eadff..b949bfdadd 100644 --- a/tox.ini +++ b/tox.ini @@ -43,7 +43,8 @@ commands = flake8 CONSTITUTION.rst HACKING.rst README.rst [testenv:docs] -commands = python setup.py build_sphinx +commands = + python setup.py build_sphinx [testenv:venv] # TODO(ihrachys): remove once infra supports constraints for this target