From f91c9c3df58fd896d8770550c04ffa559e442b67 Mon Sep 17 00:00:00 2001 From: Adam Harwell Date: Thu, 2 Feb 2017 14:05:15 -0800 Subject: [PATCH] Improve the failover handling for spares-pool amps We don't need to replace spares-pool amphorae, they will be replaced automatically by the housekeeping agent. This fixes the issue with creating more spares than are specified in config (or at least is one less way for it to happen). Change-Id: I243a86b81293fdfd40f418c3a41c73416f743da3 --- .../controller/worker/controller_worker.py | 3 +- .../controller/worker/flows/amphora_flows.py | 70 +++++++++---------- .../worker/flows/test_amphora_flows.py | 26 +++++-- 3 files changed, 57 insertions(+), 42 deletions(-) diff --git a/octavia/controller/worker/controller_worker.py b/octavia/controller/worker/controller_worker.py index 62b8767263..82316dd322 100644 --- a/octavia/controller/worker/controller_worker.py +++ b/octavia/controller/worker/controller_worker.py @@ -619,7 +619,8 @@ class ControllerWorker(base_taskflow.BaseTaskFlowEngine): id=amphora_id) failover_amphora_tf = self._taskflow_load( - self._amphora_flows.get_failover_flow(role=amp.role), + self._amphora_flows.get_failover_flow(role=amp.role, + status=amp.status), store={constants.FAILED_AMPHORA: amp, constants.LOADBALANCER_ID: amp.load_balancer_id}) with tf_logging.DynamicLoggingListener( diff --git a/octavia/controller/worker/flows/amphora_flows.py b/octavia/controller/worker/flows/amphora_flows.py index dcef071780..e4112346f7 100644 --- a/octavia/controller/worker/flows/amphora_flows.py +++ b/octavia/controller/worker/flows/amphora_flows.py @@ -274,7 +274,8 @@ class AmphoraFlows(object): requires=constants.AMPHORA)) return delete_amphora_flow - def get_failover_flow(self, role=constants.ROLE_STANDALONE): + def get_failover_flow(self, role=constants.ROLE_STANDALONE, + status=constants.AMPHORA_READY): """Creates a flow to failover a stale amphora :returns: The flow for amphora failover @@ -310,6 +311,10 @@ class AmphoraFlows(object): rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, requires=constants.AMPHORA)) + # If this is an unallocated amp (spares pool), we're done + if status != constants.AMPHORA_ALLOCATED: + return failover_amphora_flow + # Save failed amphora details for later failover_amphora_flow.add( database_tasks.GetAmphoraDetails( @@ -323,43 +328,37 @@ class AmphoraFlows(object): prefix=constants.FAILOVER_AMPHORA_FLOW) failover_amphora_flow.add(get_amp_subflow) - if role: - # Update the new amphora with the failed amphora details - failover_amphora_flow.add(database_tasks.UpdateAmpFailoverDetails( - requires=(constants.AMPHORA, constants.AMP_DATA))) + # Update the new amphora with the failed amphora details + failover_amphora_flow.add(database_tasks.UpdateAmpFailoverDetails( + requires=(constants.AMPHORA, constants.AMP_DATA))) - failover_amphora_flow.add(database_tasks.ReloadLoadBalancer( - requires=constants.LOADBALANCER_ID, - provides=constants.LOADBALANCER)) + failover_amphora_flow.add(database_tasks.ReloadLoadBalancer( + requires=constants.LOADBALANCER_ID, + provides=constants.LOADBALANCER)) - failover_amphora_flow.add(network_tasks.GetAmphoraeNetworkConfigs( - requires=constants.LOADBALANCER, - provides=constants.AMPHORAE_NETWORK_CONFIG)) - failover_amphora_flow.add( - database_tasks.GetListenersFromLoadbalancer( - requires=constants.LOADBALANCER, - provides=constants.LISTENERS)) + failover_amphora_flow.add(network_tasks.GetAmphoraeNetworkConfigs( + requires=constants.LOADBALANCER, + provides=constants.AMPHORAE_NETWORK_CONFIG)) + failover_amphora_flow.add(database_tasks.GetListenersFromLoadbalancer( + requires=constants.LOADBALANCER, provides=constants.LISTENERS)) - failover_amphora_flow.add(amphora_driver_tasks.ListenersUpdate( - requires=(constants.LOADBALANCER, constants.LISTENERS))) + failover_amphora_flow.add(amphora_driver_tasks.ListenersUpdate( + requires=(constants.LOADBALANCER, constants.LISTENERS))) - # Plug the VIP ports into the new amphora - failover_amphora_flow.add(network_tasks.PlugVIPPort( - requires=(constants.AMPHORA, - constants.AMPHORAE_NETWORK_CONFIG))) - failover_amphora_flow.add(amphora_driver_tasks.AmphoraPostVIPPlug( - requires=(constants.AMPHORA, constants.LOADBALANCER, - constants.AMPHORAE_NETWORK_CONFIG))) + # Plug the VIP ports into the new amphora + failover_amphora_flow.add(network_tasks.PlugVIPPort( + requires=(constants.AMPHORA, constants.AMPHORAE_NETWORK_CONFIG))) + failover_amphora_flow.add(amphora_driver_tasks.AmphoraPostVIPPlug( + requires=(constants.AMPHORA, constants.LOADBALANCER, + constants.AMPHORAE_NETWORK_CONFIG))) - # Plug the member networks into the new amphora - failover_amphora_flow.add(network_tasks.CalculateDelta( - requires=constants.LOADBALANCER, - provides=constants.DELTAS)) - failover_amphora_flow.add(network_tasks.HandleNetworkDeltas( - requires=constants.DELTAS, provides=constants.ADDED_PORTS)) - failover_amphora_flow.add( - amphora_driver_tasks.AmphoraePostNetworkPlug( - requires=(constants.LOADBALANCER, constants.ADDED_PORTS))) + # Plug the member networks into the new amphora + failover_amphora_flow.add(network_tasks.CalculateDelta( + requires=constants.LOADBALANCER, provides=constants.DELTAS)) + failover_amphora_flow.add(network_tasks.HandleNetworkDeltas( + requires=constants.DELTAS, provides=constants.ADDED_PORTS)) + failover_amphora_flow.add(amphora_driver_tasks.AmphoraePostNetworkPlug( + requires=(constants.LOADBALANCER, constants.ADDED_PORTS))) # Handle the amphora role and VRRP if necessary if role == constants.ROLE_MASTER: @@ -380,9 +379,8 @@ class AmphoraFlows(object): name=constants.MARK_AMP_STANDALONE_INDB, requires=constants.AMPHORA)) - if role: - failover_amphora_flow.add(amphora_driver_tasks.ListenersStart( - requires=(constants.LOADBALANCER, constants.LISTENERS))) + failover_amphora_flow.add(amphora_driver_tasks.ListenersStart( + requires=(constants.LOADBALANCER, constants.LISTENERS))) return failover_amphora_flow diff --git a/octavia/tests/unit/controller/worker/flows/test_amphora_flows.py b/octavia/tests/unit/controller/worker/flows/test_amphora_flows.py index 4a1568e031..1a90e7d63a 100644 --- a/octavia/tests/unit/controller/worker/flows/test_amphora_flows.py +++ b/octavia/tests/unit/controller/worker/flows/test_amphora_flows.py @@ -234,9 +234,10 @@ class TestAmphoraFlows(base.TestCase): result = self.AmpFlow._create_new_amp_for_lb_decider(history) self.assertFalse(result) - def test_get_failover_flow(self, mock_get_net_driver): + def test_get_failover_flow_allocated(self, mock_get_net_driver): - amp_flow = self.AmpFlow.get_failover_flow() + amp_flow = self.AmpFlow.get_failover_flow( + status=constants.AMPHORA_ALLOCATED) self.assertIsInstance(amp_flow, flow.Flow) @@ -255,7 +256,8 @@ class TestAmphoraFlows(base.TestCase): self.assertEqual(2, len(amp_flow.requires)) self.assertEqual(11, len(amp_flow.provides)) - amp_flow = self.AmpFlow.get_failover_flow(role=constants.ROLE_MASTER) + amp_flow = self.AmpFlow.get_failover_flow( + role=constants.ROLE_MASTER, status=constants.AMPHORA_ALLOCATED) self.assertIsInstance(amp_flow, flow.Flow) @@ -274,7 +276,8 @@ class TestAmphoraFlows(base.TestCase): self.assertEqual(2, len(amp_flow.requires)) self.assertEqual(11, len(amp_flow.provides)) - amp_flow = self.AmpFlow.get_failover_flow(role=constants.ROLE_BACKUP) + amp_flow = self.AmpFlow.get_failover_flow( + role=constants.ROLE_BACKUP, status=constants.AMPHORA_ALLOCATED) self.assertIsInstance(amp_flow, flow.Flow) @@ -293,7 +296,8 @@ class TestAmphoraFlows(base.TestCase): self.assertEqual(2, len(amp_flow.requires)) self.assertEqual(11, len(amp_flow.provides)) - amp_flow = self.AmpFlow.get_failover_flow(role='BOGUSROLE') + amp_flow = self.AmpFlow.get_failover_flow( + role='BOGUSROLE', status=constants.AMPHORA_ALLOCATED) self.assertIsInstance(amp_flow, flow.Flow) @@ -312,6 +316,18 @@ class TestAmphoraFlows(base.TestCase): self.assertEqual(2, len(amp_flow.requires)) self.assertEqual(11, len(amp_flow.provides)) + def test_get_failover_flow_spare(self, mock_get_net_driver): + + amp_flow = self.AmpFlow.get_failover_flow( + status=constants.AMPHORA_READY) + + self.assertIsInstance(amp_flow, flow.Flow) + + self.assertIn(constants.FAILED_AMPHORA, amp_flow.requires) + + self.assertEqual(1, len(amp_flow.requires)) + self.assertEqual(0, len(amp_flow.provides)) + def test_cert_rotate_amphora_flow(self, mock_get_net_driver): self.AmpFlow = amphora_flows.AmphoraFlows()