Make developer debugging easier

This patch makes developer debugging of Octavia easier.  It adds
a configuration option that disables the controller worker taskflow
flows from reverting and cleaning up resources.
It also changes the amphora agent to keep a copy of a haproxy
configuration that failed validation.

Change-Id: Iaca070a0ab9589fb25513eb5fad7d1e99974d572

etc/octavia.conf

@@ -225,6 +225,11 @@
 [task_flow]
 # engine = serial
 # max_workers = 5
+#
+# This setting prevents the controller worker from reverting taskflow flows.
+# This will leave resources in an inconsistent state and should only be used
+# for debugging purposes.
+# disable_revert = False
 
 [oslo_messaging]
 # Queue Consumer Thread Pool Size

octavia/amphorae/backends/agent/api_server/listener.py

@@ -136,7 +136,8 @@ class Listener(object):
             subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT)
         except subprocess.CalledProcessError as e:
             LOG.error("Failed to verify haproxy file: %s", e)
-            os.remove(name)  # delete file
+            # Save the last config that failed validation for debugging
+            os.rename(name, ''.join([name, '-failed']))
             return webob.Response(
                 json=dict(message="Invalid request", details=e.output),
                 status=400)

octavia/common/base_taskflow.py

@@ -37,6 +37,7 @@ class BaseTaskFlowEngine(object):
             flow,
             engine_conf=CONF.task_flow.engine,
             executor=self.executor,
+            never_resolve=CONF.task_flow.disable_revert,
             **kwargs)
         eng.compile()
         eng.prepare()

octavia/common/config.py

@@ -336,7 +336,12 @@ task_flow_opts = [
                help=_('TaskFlow engine to use')),
     cfg.IntOpt('max_workers',
                default=5,
-               help=_('The maximum number of workers'))
+               help=_('The maximum number of workers')),
+    cfg.BoolOpt('disable_revert', default=False,
+                help=_('If True, disables the controller worker taskflow '
+                       'flows from reverting.  This will leave resources in '
+                       'an inconsistent state and should only be used for '
+                       'debugging purposes.'))
 ]
 
 core_cli_opts = []

octavia/tests/functional/amphorae/backend/agent/api_server/test_server.py

@@ -86,9 +86,8 @@ class TestServerTestCase(base.TestCase):
     @mock.patch('os.makedirs')
     @mock.patch('os.rename')
     @mock.patch('subprocess.check_output')
-    @mock.patch('os.remove')
     def _test_haproxy(self, init_system, distro, mock_init_system,
-                      mock_remove, mock_subprocess, mock_rename,
+                      mock_subprocess, mock_rename,
                       mock_makedirs, mock_exists):
 
         self.assertIn(distro, [consts.UBUNTU, consts.CENTOS])
@@ -122,7 +121,7 @@ class TestServerTestCase(base.TestCase):
                     peer=(octavia_utils.
                           base64_sha1_string('amp_123').rstrip('='))).split(),
                 stderr=-2)
-            mock_rename.assert_called_once_with(
+            mock_rename.assert_called_with(
                 '/var/lib/octavia/123/haproxy.cfg.new',
                 '/var/lib/octavia/123/haproxy.cfg')
 
@@ -222,7 +221,9 @@ class TestServerTestCase(base.TestCase):
                     peer=(octavia_utils.
                           base64_sha1_string('amp_123').rstrip('='))).split(),
                 stderr=-2)
-            mock_remove.assert_called_once_with(file_name)
+            mock_rename.assert_called_with(
+                '/var/lib/octavia/123/haproxy.cfg.new',
+                '/var/lib/octavia/123/haproxy.cfg.new-failed')
 
         # unhappy path with bogus init system
         mock_init_system.return_value = 'bogus'

octavia/tests/unit/common/test_base_taskflow.py

@@ -35,6 +35,7 @@ class TestBaseTaskFlowEngine(base.TestCase):
         conf = oslo_fixture.Config(cfg.CONF)
         conf.config(group="task_flow", max_workers=MAX_WORKERS)
         conf.config(group="task_flow", engine='TESTENGINE')
+        conf.config(group="task_flow", disable_revert=True)
         super(TestBaseTaskFlowEngine, self).setUp()
 
     @mock.patch('concurrent.futures.ThreadPoolExecutor',
@@ -59,7 +60,8 @@ class TestBaseTaskFlowEngine(base.TestCase):
         tf_engines.load.assert_called_once_with(
             'TEST',
             engine_conf='TESTENGINE',
-            executor='TESTEXECUTOR')
+            executor='TESTEXECUTOR',
+            never_resolve=True)
 
         _engine_mock.compile.assert_called_once_with()
         _engine_mock.prepare.assert_called_once_with()