From d5fe092a3e0444a45ad11a6db8323fe4edade170 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Fri, 26 Jun 2020 17:09:08 -0700 Subject: [PATCH] Update amphora v2 for the failover refactor This patch will update the amphora v2 code for the failover refactor[1]. [1] https://review.opendev.org/705317 Change-Id: I43803d0b750e8ca4722ababe296f2725148da405 --- .gitignore | 1 + doc/source/conf.py | 2 + doc/source/contributor/devref/flows.rst | 14 + .../api/drivers/amphora_driver/v2/driver.py | 7 +- octavia/common/constants.py | 1 + .../worker/v1/tasks/amphora_driver_tasks.py | 13 +- .../controller/worker/v2/controller_worker.py | 424 ++++--- .../worker/v2/flows/amphora_flows.py | 790 +++++++------ .../controller/worker/v2/flows/flow_utils.py | 47 +- .../worker/v2/flows/listener_flows.py | 14 +- .../worker/v2/flows/load_balancer_flows.py | 503 ++++++-- .../controller/worker/v2/flows/pool_flows.py | 9 +- .../worker/v2/tasks/amphora_driver_tasks.py | 262 +++-- .../worker/v2/tasks/compute_tasks.py | 92 +- .../worker/v2/tasks/database_tasks.py | 124 +- .../worker/v2/tasks/network_tasks.py | 254 ++++- .../controller/worker/v2/tasks/retry_tasks.py | 73 ++ .../worker/v1/flows/test_amphora_flows.py | 22 +- .../v1/tasks/test_amphora_driver_tasks.py | 21 +- .../worker/v1/test_controller_worker.py | 48 - .../worker/v2/flows/test_amphora_flows.py | 169 +-- .../worker/v2/flows/test_listener_flows.py | 8 +- .../v2/flows/test_load_balancer_flows.py | 274 ++++- .../worker/v2/flows/test_pool_flows.py | 3 +- .../v2/tasks/test_amphora_driver_tasks.py | 290 +++-- .../worker/v2/tasks/test_compute_tasks.py | 67 +- .../worker/v2/tasks/test_database_tasks.py | 37 +- .../worker/v2/tasks/test_network_tasks.py | 436 ++++++- .../worker/v2/tasks/test_retry_tasks.py | 47 + .../worker/v2/test_controller_worker.py | 1013 +++++++++++++---- tools/create_flow_docs.py | 31 +- tools/flow-list-v2.txt | 32 + 32 files changed, 3742 insertions(+), 1386 deletions(-) create mode 100644 octavia/controller/worker/v2/tasks/retry_tasks.py create mode 100644 octavia/tests/unit/controller/worker/v2/tasks/test_retry_tasks.py create mode 100644 tools/flow-list-v2.txt diff --git a/.gitignore b/.gitignore index 051803b17f..826aad1947 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ doc/build doc/source/configuration/_static/octavia.policy.yaml.sample doc/source/contributor/devref/erd.svg doc/source/contributor/devref/flow_diagrams/ +doc/source/contributor/devref/flow_diagrams_v2/ doc/source/contributor/modules api-ref/build .idea/* diff --git a/doc/source/conf.py b/doc/source/conf.py index ed2a88e918..142678945a 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -29,6 +29,8 @@ from tools import create_flow_docs # Generate our flow diagrams create_flow_docs.generate( 'tools/flow-list.txt', 'doc/source/contributor/devref/flow_diagrams') +create_flow_docs.generate( + 'tools/flow-list-v2.txt', 'doc/source/contributor/devref/flow_diagrams_v2') # Generate entity relationship diagram desc = sadisplay.describe( diff --git a/doc/source/contributor/devref/flows.rst b/doc/source/contributor/devref/flows.rst index 04295f3dc3..93cd0a449b 100644 --- a/doc/source/contributor/devref/flows.rst +++ b/doc/source/contributor/devref/flows.rst @@ -19,3 +19,17 @@ Octavia controller. flow_diagrams/LoadBalancerFlows.rst flow_diagrams/MemberFlows.rst flow_diagrams/PoolFlows.rst + +The following are flow diagrams for the **amphora V2** driver. + +.. toctree:: + :maxdepth: 1 + + flow_diagrams_v2/AmphoraFlows.rst + flow_diagrams_v2/HealthMonitorFlows.rst + flow_diagrams_v2/L7PolicyFlows.rst + flow_diagrams_v2/L7RuleFlows.rst + flow_diagrams_v2/ListenerFlows.rst + flow_diagrams_v2/LoadBalancerFlows.rst + flow_diagrams_v2/MemberFlows.rst + flow_diagrams_v2/PoolFlows.rst diff --git a/octavia/api/drivers/amphora_driver/v2/driver.py b/octavia/api/drivers/amphora_driver/v2/driver.py index 5337d06c5a..dd69d735c7 100644 --- a/octavia/api/drivers/amphora_driver/v2/driver.py +++ b/octavia/api/drivers/amphora_driver/v2/driver.py @@ -87,8 +87,11 @@ class AmphoraProviderDriver(driver_base.ProviderDriver): try: vip = network_driver.allocate_vip(lb_obj) except network_base.AllocateVIPException as e: - raise exceptions.DriverError(user_fault_string=e.orig_msg, - operator_fault_string=e.orig_msg) + message = str(e) + if getattr(e, 'orig_msg', None) is not None: + message = e.orig_msg + raise exceptions.DriverError(user_fault_string=message, + operator_fault_string=message) LOG.info('Amphora provider created VIP port %s for load balancer %s.', vip.port_id, loadbalancer_id) diff --git a/octavia/common/constants.py b/octavia/common/constants.py index bdd05f6444..e20b07ef7d 100644 --- a/octavia/common/constants.py +++ b/octavia/common/constants.py @@ -300,6 +300,7 @@ ALLOWED_ADDRESS_PAIRS = 'allowed_address_pairs' AMP_DATA = 'amp_data' AMP_VRRP_INT = 'amp_vrrp_int' AMPHORA = 'amphora' +AMPHORA_DICT = 'amphora_dict' AMPHORA_ID = 'amphora_id' AMPHORA_INDEX = 'amphora_index' AMPHORA_NETWORK_CONFIG = 'amphora_network_config' diff --git a/octavia/controller/worker/v1/tasks/amphora_driver_tasks.py b/octavia/controller/worker/v1/tasks/amphora_driver_tasks.py index 6a2a0239d4..6832a8534a 100644 --- a/octavia/controller/worker/v1/tasks/amphora_driver_tasks.py +++ b/octavia/controller/worker/v1/tasks/amphora_driver_tasks.py @@ -122,7 +122,7 @@ class ListenersStart(BaseAmphoraTask): class AmphoraIndexListenersReload(BaseAmphoraTask): """Task to reload all listeners on an amphora.""" - def execute(self, loadbalancer, amphorae, amphora_index, + def execute(self, loadbalancer, amphora_index, amphorae, timeout_dict=None): """Execute listener reload routines for listeners on an amphora.""" if loadbalancer.listeners: @@ -304,7 +304,7 @@ class AmphoraUpdateVRRPInterface(BaseAmphoraTask): class AmphoraIndexUpdateVRRPInterface(BaseAmphoraTask): """Task to get and update the VRRP interface device name from amphora.""" - def execute(self, amphorae, amphora_index, timeout_dict=None): + def execute(self, amphora_index, amphorae, timeout_dict=None): amphora_id = amphorae[amphora_index].id try: interface = self.amphora_driver.get_interface_from_ip( @@ -376,15 +376,6 @@ class AmphoraIndexVRRPUpdate(BaseAmphoraTask): LOG.debug("Uploaded VRRP configuration of amphora %s.", amphora_id) -class AmphoraVRRPStop(BaseAmphoraTask): - """Task to stop keepalived of all amphorae of a LB.""" - - def execute(self, loadbalancer): - self.amphora_driver.stop_vrrp_service(loadbalancer) - LOG.debug("Stopped VRRP of loadbalancer %s amphorae", - loadbalancer.id) - - class AmphoraVRRPStart(BaseAmphoraTask): """Task to start keepalived on an amphora. diff --git a/octavia/controller/worker/v2/controller_worker.py b/octavia/controller/worker/v2/controller_worker.py index c8a23eb1a8..ea85c14807 100644 --- a/octavia/controller/worker/v2/controller_worker.py +++ b/octavia/controller/worker/v2/controller_worker.py @@ -13,6 +13,7 @@ # under the License. # +from octavia_lib.common import constants as lib_consts from oslo_config import cfg from oslo_log import log as logging from oslo_utils import excutils @@ -21,10 +22,12 @@ from stevedore import driver as stevedore_driver from taskflow.listeners import logging as tf_logging import tenacity -from octavia.amphorae.driver_exceptions import exceptions +from octavia.amphorae.driver_exceptions import exceptions as driver_exc from octavia.api.drivers import utils as provider_utils from octavia.common import base_taskflow from octavia.common import constants +from octavia.common import exceptions +from octavia.common import utils from octavia.controller.worker.v2.flows import flow_utils from octavia.controller.worker.v2 import taskflow_jobboard_driver as tsk_driver from octavia.db import api as db_apis @@ -33,17 +36,12 @@ from octavia.db import repositories as repo CONF = cfg.CONF LOG = logging.getLogger(__name__) -RETRY_ATTEMPTS = 15 -RETRY_INITIAL_DELAY = 1 -RETRY_BACKOFF = 1 -RETRY_MAX = 5 - # We do not need to log retry exception information. Warning "Could not connect # to instance" will be logged as usual. def retryMaskFilter(record): if record.exc_info is not None and isinstance( - record.exc_info[1], exceptions.AmpConnectionRetry): + record.exc_info[1], driver_exc.AmpConnectionRetry): return False return True @@ -87,8 +85,11 @@ class ControllerWorker(object): tenacity.retry_if_result(_is_provisioning_status_pending_update) | tenacity.retry_if_exception_type()), wait=tenacity.wait_incrementing( - RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX), - stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS)) + CONF.haproxy_amphora.api_db_commit_retry_initial_delay, + CONF.haproxy_amphora.api_db_commit_retry_backoff, + CONF.haproxy_amphora.api_db_commit_retry_max), + stop=tenacity.stop_after_attempt( + CONF.haproxy_amphora.api_db_commit_retry_attempts)) def _get_db_obj_until_pending_update(self, repo, id): return repo.get(db_apis.get_session(), id=id) @@ -117,6 +118,7 @@ class ControllerWorker(object): store = {constants.BUILD_TYPE_PRIORITY: constants.LB_CREATE_SPARES_POOL_PRIORITY, constants.FLAVOR: None, + constants.SERVER_GROUP_ID: None, constants.AVAILABILITY_ZONE: None} if availability_zone: store[constants.AVAILABILITY_ZONE] = ( @@ -145,8 +147,11 @@ class ControllerWorker(object): @tenacity.retry( retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound), wait=tenacity.wait_incrementing( - RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX), - stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS)) + CONF.haproxy_amphora.api_db_commit_retry_initial_delay, + CONF.haproxy_amphora.api_db_commit_retry_backoff, + CONF.haproxy_amphora.api_db_commit_retry_max), + stop=tenacity.stop_after_attempt( + CONF.haproxy_amphora.api_db_commit_retry_attempts)) def create_health_monitor(self, health_monitor): """Creates a health monitor. @@ -251,8 +256,11 @@ class ControllerWorker(object): @tenacity.retry( retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound), wait=tenacity.wait_incrementing( - RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX), - stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS)) + CONF.haproxy_amphora.api_db_commit_retry_initial_delay, + CONF.haproxy_amphora.api_db_commit_retry_backoff, + CONF.haproxy_amphora.api_db_commit_retry_max), + stop=tenacity.stop_after_attempt( + CONF.haproxy_amphora.api_db_commit_retry_attempts)) def create_listener(self, listener): """Creates a listener. @@ -292,14 +300,10 @@ class ControllerWorker(object): :returns: None :raises ListenerNotFound: The referenced listener was not found """ - # TODO(johnsom) Remove once the provider data model includes - # the project ID - lb = self._lb_repo.get(db_apis.get_session(), - id=listener[constants.LOADBALANCER_ID]) store = {constants.LISTENER: listener, constants.LOADBALANCER_ID: listener[constants.LOADBALANCER_ID], - constants.PROJECT_ID: lb.project_id} + constants.PROJECT_ID: listener[constants.PROJECT_ID]} self.run_flow( flow_utils.get_delete_listener_flow, store=store) @@ -325,8 +329,11 @@ class ControllerWorker(object): @tenacity.retry( retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound), wait=tenacity.wait_incrementing( - RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX), - stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS)) + CONF.haproxy_amphora.api_db_commit_retry_initial_delay, + CONF.haproxy_amphora.api_db_commit_retry_backoff, + CONF.haproxy_amphora.api_db_commit_retry_max), + stop=tenacity.stop_after_attempt( + CONF.haproxy_amphora.api_db_commit_retry_attempts)) def create_load_balancer(self, loadbalancer, flavor=None, availability_zone=None): """Creates a load balancer by allocating Amphorae. @@ -347,16 +354,18 @@ class ControllerWorker(object): loadbalancer[constants.LOADBALANCER_ID]) raise db_exceptions.NoResultFound - # TODO(johnsom) convert this to octavia_lib constant flavor - # once octavia is transitioned to use octavia_lib - store = {constants.LOADBALANCER_ID: - loadbalancer[constants.LOADBALANCER_ID], + store = {lib_consts.LOADBALANCER_ID: + loadbalancer[lib_consts.LOADBALANCER_ID], constants.BUILD_TYPE_PRIORITY: constants.LB_CREATE_NORMAL_PRIORITY, - constants.FLAVOR: flavor, - constants.AVAILABILITY_ZONE: availability_zone} + lib_consts.FLAVOR: flavor, + lib_consts.AVAILABILITY_ZONE: availability_zone} topology = lb.topology + if (not CONF.nova.enable_anti_affinity or + topology == constants.TOPOLOGY_SINGLE): + store[constants.SERVER_GROUP_ID] = None + listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( lb.listeners) @@ -377,17 +386,19 @@ class ControllerWorker(object): :returns: None :raises LBNotFound: The referenced load balancer was not found """ - db_lb = self._lb_repo.get(db_apis.get_session(), - id=load_balancer[constants.LOADBALANCER_ID]) + loadbalancer_id = load_balancer[constants.LOADBALANCER_ID] + db_lb = self._lb_repo.get(db_apis.get_session(), id=loadbalancer_id) store = {constants.LOADBALANCER: load_balancer, + constants.LOADBALANCER_ID: loadbalancer_id, constants.SERVER_GROUP_ID: db_lb.server_group_id, constants.PROJECT_ID: db_lb.project_id} if cascade: - store.update(flow_utils.get_delete_pools_store(db_lb)) - store.update(flow_utils.get_delete_listeners_store(db_lb)) + listeners = flow_utils.get_listeners_on_lb(db_lb) + pools = flow_utils.get_pools_on_lb(db_lb) + self.run_flow( flow_utils.get_cascade_delete_load_balancer_flow, - load_balancer, store=store) + load_balancer, listeners, pools, store=store) else: self.run_flow( flow_utils.get_delete_load_balancer_flow, @@ -548,7 +559,6 @@ class ControllerWorker(object): listeners_dicts = ( provider_utils.db_listeners_to_provider_dicts_list_of_dicts( pool.listeners)) - store = { constants.MEMBER: member, constants.LISTENERS: listeners_dicts, @@ -570,8 +580,11 @@ class ControllerWorker(object): @tenacity.retry( retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound), wait=tenacity.wait_incrementing( - RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX), - stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS)) + CONF.haproxy_amphora.api_db_commit_retry_initial_delay, + CONF.haproxy_amphora.api_db_commit_retry_backoff, + CONF.haproxy_amphora.api_db_commit_retry_max), + stop=tenacity.stop_after_attempt( + CONF.haproxy_amphora.api_db_commit_retry_attempts)) def create_pool(self, pool): """Creates a node pool. @@ -816,153 +829,250 @@ class ControllerWorker(object): flow_utils.get_update_l7rule_flow, store=store) - def _perform_amphora_failover(self, amp, priority): - """Internal method to perform failover operations for an amphora. - - :param amp: The amphora to failover - :param priority: The create priority - :returns: None - """ - stored_params = {constants.FAILED_AMPHORA: amp.to_dict(), - constants.LOADBALANCER_ID: amp.load_balancer_id, - constants.BUILD_TYPE_PRIORITY: priority, } - - if amp.role in (constants.ROLE_MASTER, constants.ROLE_BACKUP): - amp_role = 'master_or_backup' - elif amp.role == constants.ROLE_STANDALONE: - amp_role = 'standalone' - elif amp.role is None: - amp_role = 'spare' - else: - amp_role = 'undefined' - - LOG.info("Perform failover for an amphora: %s", - {"id": amp.id, - "load_balancer_id": amp.load_balancer_id, - "lb_network_ip": amp.lb_network_ip, - "compute_id": amp.compute_id, - "role": amp_role}) - - if amp.status == constants.DELETED: - LOG.warning('Amphora %s is marked DELETED in the database but ' - 'was submitted for failover. Deleting it from the ' - 'amphora health table to exclude it from health ' - 'checks and skipping the failover.', amp.id) - self._amphora_health_repo.delete(db_apis.get_session(), - amphora_id=amp.id) - return - - if (CONF.house_keeping.spare_amphora_pool_size == 0) and ( - CONF.nova.enable_anti_affinity is False): - LOG.warning("Failing over amphora with no spares pool may " - "cause delays in failover times while a new " - "amphora instance boots.") - - # if we run with anti-affinity we need to set the server group - # as well - lb = self._amphora_repo.get_lb_for_amphora( - db_apis.get_session(), amp.id) - provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( - lb).to_dict() if lb else lb - if CONF.nova.enable_anti_affinity and lb: - stored_params[constants.SERVER_GROUP_ID] = lb.server_group_id - if lb is not None and lb.flavor_id: - stored_params[constants.FLAVOR] = ( - self._flavor_repo.get_flavor_metadata_dict( - db_apis.get_session(), lb.flavor_id)) - else: - stored_params[constants.FLAVOR] = {} - if lb and lb.availability_zone: - stored_params[constants.AVAILABILITY_ZONE] = ( - self._az_repo.get_availability_zone_metadata_dict( - db_apis.get_session(), lb.availability_zone)) - else: - stored_params[constants.AVAILABILITY_ZONE] = {} - - self.run_flow( - flow_utils.get_failover_flow, - role=amp.role, load_balancer=provider_lb, - store=stored_params, wait=True) - - LOG.info("Successfully completed the failover for an amphora: %s", - {"id": amp.id, - "load_balancer_id": amp.load_balancer_id, - "lb_network_ip": amp.lb_network_ip, - "compute_id": amp.compute_id, - "role": amp_role}) - def failover_amphora(self, amphora_id): """Perform failover operations for an amphora. + Note: This expects the load balancer to already be in + provisioning_status=PENDING_UPDATE state. + :param amphora_id: ID for amphora to failover :returns: None - :raises AmphoraNotFound: The referenced amphora was not found + :raises octavia.common.exceptions.NotFound: The referenced amphora was + not found """ + amphora = None try: - amp = self._amphora_repo.get(db_apis.get_session(), - id=amphora_id) - if not amp: - LOG.warning("Could not fetch Amphora %s from DB, ignoring " - "failover request.", amphora_id) + amphora = self._amphora_repo.get(db_apis.get_session(), + id=amphora_id) + if amphora is None: + LOG.error('Amphora failover for amphora %s failed because ' + 'there is no record of this amphora in the ' + 'database. Check that the [house_keeping] ' + 'amphora_expiry_age configuration setting is not ' + 'too short. Skipping failover.', amphora_id) + raise exceptions.NotFound(resource=constants.AMPHORA, + id=amphora_id) + + if amphora.status == constants.DELETED: + LOG.warning('Amphora %s is marked DELETED in the database but ' + 'was submitted for failover. Deleting it from the ' + 'amphora health table to exclude it from health ' + 'checks and skipping the failover.', amphora.id) + self._amphora_health_repo.delete(db_apis.get_session(), + amphora_id=amphora.id) return - self._perform_amphora_failover( - amp, constants.LB_CREATE_FAILOVER_PRIORITY) - if amp.load_balancer_id: - LOG.info("Mark ACTIVE in DB for load balancer id: %s", - amp.load_balancer_id) - self._lb_repo.update( - db_apis.get_session(), amp.load_balancer_id, - provisioning_status=constants.ACTIVE) + + loadbalancer = None + if amphora.load_balancer_id: + loadbalancer = self._lb_repo.get(db_apis.get_session(), + id=amphora.load_balancer_id) + lb_amp_count = None + if loadbalancer: + if loadbalancer.topology == constants.TOPOLOGY_ACTIVE_STANDBY: + lb_amp_count = 2 + elif loadbalancer.topology == constants.TOPOLOGY_SINGLE: + lb_amp_count = 1 + + az_metadata = {} + flavor_dict = {} + lb_id = None + vip_dict = {} + server_group_id = None + if loadbalancer: + lb_id = loadbalancer.id + # Even if the LB doesn't have a flavor, create one and + # pass through the topology. + if loadbalancer.flavor_id: + flavor_dict = self._flavor_repo.get_flavor_metadata_dict( + db_apis.get_session(), loadbalancer.flavor_id) + flavor_dict[constants.LOADBALANCER_TOPOLOGY] = ( + loadbalancer.topology) + else: + flavor_dict = {constants.LOADBALANCER_TOPOLOGY: + loadbalancer.topology} + if loadbalancer.availability_zone: + az_metadata = ( + self._az_repo.get_availability_zone_metadata_dict( + db_apis.get_session(), + loadbalancer.availability_zone)) + vip_dict = loadbalancer.vip.to_dict() + server_group_id = loadbalancer.server_group_id + provider_lb_dict = (provider_utils. + db_loadbalancer_to_provider_loadbalancer)( + loadbalancer).to_dict() if loadbalancer else loadbalancer + + stored_params = {constants.AVAILABILITY_ZONE: az_metadata, + constants.BUILD_TYPE_PRIORITY: + constants.LB_CREATE_FAILOVER_PRIORITY, + constants.FLAVOR: flavor_dict, + constants.LOADBALANCER: provider_lb_dict, + constants.SERVER_GROUP_ID: server_group_id, + constants.LOADBALANCER_ID: lb_id, + constants.VIP: vip_dict} + + self.services_controller.run_poster( + flow_utils.get_failover_amphora_flow, + amphora.to_dict(), lb_amp_count, + store=stored_params, wait=True) + + LOG.info("Successfully completed the failover for an amphora: %s", + {"id": amphora_id, + "load_balancer_id": lb_id, + "lb_network_ip": amphora.lb_network_ip, + "compute_id": amphora.compute_id, + "role": amphora.role}) + except Exception as e: - try: - self._lb_repo.update( - db_apis.get_session(), amp.load_balancer_id, - provisioning_status=constants.ERROR) - except Exception: - LOG.error("Unable to revert LB status to ERROR.") - with excutils.save_and_reraise_exception(): - LOG.error("Amphora %(id)s failover exception: %(exc)s", - {'id': amphora_id, 'exc': e}) + with excutils.save_and_reraise_exception(reraise=False): + LOG.exception("Amphora %s failover exception: %s", + amphora_id, str(e)) + self._amphora_repo.update(db_apis.get_session(), + amphora_id, status=constants.ERROR) + if amphora and amphora.load_balancer_id: + self._lb_repo.update( + db_apis.get_session(), amphora.load_balancer_id, + provisioning_status=constants.ERROR) + + @staticmethod + def _get_amphorae_for_failover(load_balancer): + """Returns an ordered list of amphora to failover. + + :param load_balancer: The load balancer being failed over. + :returns: An ordered list of amphora to failover, + first amp to failover is last in the list + :raises octavia.common.exceptions.InvalidTopology: LB has an unknown + topology. + """ + if load_balancer.topology == constants.TOPOLOGY_SINGLE: + # In SINGLE topology, amp failover order does not matter + return [a.to_dict() for a in load_balancer.amphorae + if a.status != constants.DELETED] + + if load_balancer.topology == constants.TOPOLOGY_ACTIVE_STANDBY: + # In Active/Standby we should preference the standby amp + # for failover first in case the Active is still able to pass + # traffic. + # Note: The active amp can switch at any time and in less than a + # second, so this is "best effort". + amphora_driver = utils.get_amphora_driver() + timeout_dict = { + constants.CONN_MAX_RETRIES: + CONF.haproxy_amphora.failover_connection_max_retries, + constants.CONN_RETRY_INTERVAL: + CONF.haproxy_amphora.failover_connection_retry_interval} + amps = [] + selected_amp = None + for amp in load_balancer.amphorae: + if amp.status == constants.DELETED: + continue + if selected_amp is None: + try: + if amphora_driver.get_interface_from_ip( + amp, load_balancer.vip.ip_address, + timeout_dict): + # This is a potential ACTIVE, add it to the list + amps.append(amp.to_dict()) + else: + # This one doesn't have the VIP IP, so start + # failovers here. + selected_amp = amp + LOG.debug("Selected amphora %s as the initial " + "failover amphora.", amp.id) + except Exception: + # This amphora is broken, so start failovers here. + selected_amp = amp + else: + # We have already found a STANDBY, so add the rest to the + # list without querying them. + amps.append(amp.to_dict()) + # Put the selected amphora at the end of the list so it is + # first to failover. + if selected_amp: + amps.append(selected_amp.to_dict()) + return amps + + LOG.error('Unknown load balancer topology found: %s, aborting ' + 'failover.', load_balancer.topology) + raise exceptions.InvalidTopology(topology=load_balancer.topology) def failover_loadbalancer(self, load_balancer_id): """Perform failover operations for a load balancer. + Note: This expects the load balancer to already be in + provisioning_status=PENDING_UPDATE state. + :param load_balancer_id: ID for load balancer to failover :returns: None - :raises LBNotFound: The referenced load balancer was not found + :raises octavia.commom.exceptions.NotFound: The load balancer was not + found. """ - - # Note: This expects that the load balancer is already in - # provisioning_status=PENDING_UPDATE state try: lb = self._lb_repo.get(db_apis.get_session(), id=load_balancer_id) + if lb is None: + raise exceptions.NotFound(resource=constants.LOADBALANCER, + id=load_balancer_id) - # Exclude amphora already deleted - amps = [a for a in lb.amphorae if a.status != constants.DELETED] - for amp in amps: - # failover amphora in backup role - # Note: this amp may not currently be the backup - # TODO(johnsom) Change this to query the amp state - # once the amp API supports it. - if amp.role == constants.ROLE_BACKUP: - self._perform_amphora_failover( - amp, constants.LB_CREATE_ADMIN_FAILOVER_PRIORITY) + # Get the ordered list of amphorae to failover for this LB. + amps = self._get_amphorae_for_failover(lb) - for amp in amps: - # failover everyhting else - if amp.role != constants.ROLE_BACKUP: - self._perform_amphora_failover( - amp, constants.LB_CREATE_ADMIN_FAILOVER_PRIORITY) + if lb.topology == constants.TOPOLOGY_SINGLE: + if len(amps) != 1: + LOG.warning('%d amphorae found on load balancer %s where ' + 'one should exist. Repairing.', len(amps), + load_balancer_id) + elif lb.topology == constants.TOPOLOGY_ACTIVE_STANDBY: - self._lb_repo.update( - db_apis.get_session(), load_balancer_id, - provisioning_status=constants.ACTIVE) + if len(amps) != 2: + LOG.warning('%d amphorae found on load balancer %s where ' + 'two should exist. Repairing.', len(amps), + load_balancer_id) + else: + LOG.error('Unknown load balancer topology found: %s, aborting ' + 'failover!', lb.topology) + raise exceptions.InvalidTopology(topology=lb.topology) + + # We must provide a topology in the flavor definition + # here for the amphora to be created with the correct + # configuration. + if lb.flavor_id: + flavor = self._flavor_repo.get_flavor_metadata_dict( + db_apis.get_session(), lb.flavor_id) + flavor[constants.LOADBALANCER_TOPOLOGY] = lb.topology + else: + flavor = {constants.LOADBALANCER_TOPOLOGY: lb.topology} + + provider_lb_dict = ( + provider_utils.db_loadbalancer_to_provider_loadbalancer( + lb).to_dict() if lb else lb) + + provider_lb_dict[constants.FLAVOR] = flavor + + stored_params = {constants.LOADBALANCER: provider_lb_dict, + constants.BUILD_TYPE_PRIORITY: + constants.LB_CREATE_FAILOVER_PRIORITY, + constants.SERVER_GROUP_ID: lb.server_group_id, + constants.LOADBALANCER_ID: lb.id, + constants.FLAVOR: flavor} + + if lb.availability_zone: + stored_params[constants.AVAILABILITY_ZONE] = ( + self._az_repo.get_availability_zone_metadata_dict( + db_apis.get_session(), lb.availability_zone)) + else: + stored_params[constants.AVAILABILITY_ZONE] = {} + + self.services_controller.run_poster( + flow_utils.get_failover_LB_flow, amps, provider_lb_dict, + store=stored_params, wait=True) + + LOG.info('Failover of load balancer %s completed successfully.', + lb.id) except Exception as e: - with excutils.save_and_reraise_exception(): - LOG.error("LB %(lbid)s failover exception: %(exc)s", - {'lbid': load_balancer_id, 'exc': e}) + with excutils.save_and_reraise_exception(reraise=False): + LOG.exception("LB %(lbid)s failover exception: %(exc)s", + {'lbid': load_balancer_id, 'exc': e}) self._lb_repo.update( db_apis.get_session(), load_balancer_id, provisioning_status=constants.ERROR) diff --git a/octavia/controller/worker/v2/flows/amphora_flows.py b/octavia/controller/worker/v2/flows/amphora_flows.py index 926bbeacd8..1ae186fbc1 100644 --- a/octavia/controller/worker/v2/flows/amphora_flows.py +++ b/octavia/controller/worker/v2/flows/amphora_flows.py @@ -1,4 +1,5 @@ # Copyright 2015 Hewlett-Packard Development Company, L.P. +# Copyright 2020 Red Hat, Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -14,28 +15,27 @@ # from oslo_config import cfg +from oslo_log import log as logging from taskflow.patterns import graph_flow from taskflow.patterns import linear_flow from taskflow.patterns import unordered_flow from octavia.common import constants +from octavia.common import utils from octavia.controller.worker.v2.tasks import amphora_driver_tasks from octavia.controller.worker.v2.tasks import cert_task from octavia.controller.worker.v2.tasks import compute_tasks from octavia.controller.worker.v2.tasks import database_tasks from octavia.controller.worker.v2.tasks import lifecycle_tasks from octavia.controller.worker.v2.tasks import network_tasks -from octavia.db import api as db_apis -from octavia.db import repositories as repo +from octavia.controller.worker.v2.tasks import retry_tasks CONF = cfg.CONF +LOG = logging.getLogger(__name__) class AmphoraFlows(object): - def __init__(self): - self.lb_repo = repo.LoadBalancerRepository() - def get_create_amphora_flow(self): """Creates a flow to create an amphora. @@ -46,24 +46,16 @@ class AmphoraFlows(object): provides=constants.AMPHORA_ID)) create_amphora_flow.add(lifecycle_tasks.AmphoraIDToErrorOnRevertTask( requires=constants.AMPHORA_ID)) - if (CONF.controller_worker.amphora_driver == - 'amphora_haproxy_rest_driver'): - create_amphora_flow.add(cert_task.GenerateServerPEMTask( - provides=constants.SERVER_PEM)) - - create_amphora_flow.add( - database_tasks.UpdateAmphoraDBCertExpiration( - requires=(constants.AMPHORA_ID, constants.SERVER_PEM))) - - create_amphora_flow.add(compute_tasks.CertComputeCreate( - requires=(constants.AMPHORA_ID, constants.SERVER_PEM, - constants.BUILD_TYPE_PRIORITY, constants.FLAVOR), - provides=constants.COMPUTE_ID)) - else: - create_amphora_flow.add(compute_tasks.ComputeCreate( - requires=(constants.AMPHORA_ID, constants.BUILD_TYPE_PRIORITY, - constants.FLAVOR), - provides=constants.COMPUTE_ID)) + create_amphora_flow.add(cert_task.GenerateServerPEMTask( + provides=constants.SERVER_PEM)) + create_amphora_flow.add( + database_tasks.UpdateAmphoraDBCertExpiration( + requires=(constants.AMPHORA_ID, constants.SERVER_PEM))) + create_amphora_flow.add(compute_tasks.CertComputeCreate( + requires=(constants.AMPHORA_ID, constants.SERVER_PEM, + constants.SERVER_GROUP_ID, + constants.BUILD_TYPE_PRIORITY, constants.FLAVOR), + provides=constants.COMPUTE_ID)) create_amphora_flow.add(database_tasks.MarkAmphoraBootingInDB( requires=(constants.AMPHORA_ID, constants.COMPUTE_ID))) create_amphora_flow.add(compute_tasks.ComputeActiveWait( @@ -97,11 +89,6 @@ class AmphoraFlows(object): post_map_amp_to_lb = linear_flow.Flow( sf_name) - post_map_amp_to_lb.add(database_tasks.ReloadAmphora( - name=sf_name + '-' + constants.RELOAD_AMPHORA, - requires=constants.AMPHORA, - provides=constants.AMPHORA)) - post_map_amp_to_lb.add(amphora_driver_tasks.AmphoraConfigUpdate( name=sf_name + '-' + constants.AMPHORA_CONFIG_UPDATE_TASK, requires=(constants.AMPHORA, constants.FLAVOR))) @@ -121,7 +108,7 @@ class AmphoraFlows(object): return post_map_amp_to_lb - def _get_create_amp_for_lb_subflow(self, prefix, role): + def _get_create_amp_for_lb_subflow(self, prefix, role, is_spare=False): """Create a new amphora for lb.""" sf_name = prefix + '-' + constants.CREATE_AMP_FOR_LB_SUBFLOW @@ -131,67 +118,22 @@ class AmphoraFlows(object): requires=constants.LOADBALANCER_ID, provides=constants.AMPHORA_ID)) - require_server_group_id_condition = ( - role in (constants.ROLE_BACKUP, constants.ROLE_MASTER) and - CONF.nova.enable_anti_affinity) + create_amp_for_lb_subflow.add(cert_task.GenerateServerPEMTask( + name=sf_name + '-' + constants.GENERATE_SERVER_PEM, + provides=constants.SERVER_PEM)) - if (CONF.controller_worker.amphora_driver == - 'amphora_haproxy_rest_driver'): - create_amp_for_lb_subflow.add(cert_task.GenerateServerPEMTask( - name=sf_name + '-' + constants.GENERATE_SERVER_PEM, - provides=constants.SERVER_PEM)) - - create_amp_for_lb_subflow.add( - database_tasks.UpdateAmphoraDBCertExpiration( - name=sf_name + '-' + constants.UPDATE_CERT_EXPIRATION, - requires=(constants.AMPHORA_ID, constants.SERVER_PEM))) - - if require_server_group_id_condition: - create_amp_for_lb_subflow.add(compute_tasks.CertComputeCreate( - name=sf_name + '-' + constants.CERT_COMPUTE_CREATE, - requires=( - constants.AMPHORA_ID, - constants.SERVER_PEM, - constants.BUILD_TYPE_PRIORITY, - constants.SERVER_GROUP_ID, - constants.FLAVOR, - constants.AVAILABILITY_ZONE, - ), - provides=constants.COMPUTE_ID)) - else: - create_amp_for_lb_subflow.add(compute_tasks.CertComputeCreate( - name=sf_name + '-' + constants.CERT_COMPUTE_CREATE, - requires=( - constants.AMPHORA_ID, - constants.SERVER_PEM, - constants.BUILD_TYPE_PRIORITY, - constants.FLAVOR, - constants.AVAILABILITY_ZONE, - ), - provides=constants.COMPUTE_ID)) - else: - if require_server_group_id_condition: - create_amp_for_lb_subflow.add(compute_tasks.ComputeCreate( - name=sf_name + '-' + constants.COMPUTE_CREATE, - requires=( - constants.AMPHORA_ID, - constants.BUILD_TYPE_PRIORITY, - constants.SERVER_GROUP_ID, - constants.FLAVOR, - constants.AVAILABILITY_ZONE, - ), - provides=constants.COMPUTE_ID)) - else: - create_amp_for_lb_subflow.add(compute_tasks.ComputeCreate( - name=sf_name + '-' + constants.COMPUTE_CREATE, - requires=( - constants.AMPHORA_ID, - constants.BUILD_TYPE_PRIORITY, - constants.FLAVOR, - constants.AVAILABILITY_ZONE, - ), - provides=constants.COMPUTE_ID)) + create_amp_for_lb_subflow.add( + database_tasks.UpdateAmphoraDBCertExpiration( + name=sf_name + '-' + constants.UPDATE_CERT_EXPIRATION, + requires=(constants.AMPHORA_ID, constants.SERVER_PEM))) + create_amp_for_lb_subflow.add(compute_tasks.CertComputeCreate( + name=sf_name + '-' + constants.CERT_COMPUTE_CREATE, + requires=(constants.AMPHORA_ID, constants.SERVER_PEM, + constants.BUILD_TYPE_PRIORITY, + constants.SERVER_GROUP_ID, + constants.FLAVOR, constants.AVAILABILITY_ZONE), + provides=constants.COMPUTE_ID)) create_amp_for_lb_subflow.add(database_tasks.UpdateAmphoraComputeId( name=sf_name + '-' + constants.UPDATE_AMPHORA_COMPUTEID, requires=(constants.AMPHORA_ID, constants.COMPUTE_ID))) @@ -207,6 +149,33 @@ class AmphoraFlows(object): name=sf_name + '-' + constants.UPDATE_AMPHORA_INFO, requires=(constants.AMPHORA_ID, constants.COMPUTE_OBJ), provides=constants.AMPHORA)) + create_amp_for_lb_subflow.add(self._retry_flow(sf_name)) + create_amp_for_lb_subflow.add(amphora_driver_tasks.AmphoraFinalize( + name=sf_name + '-' + constants.AMPHORA_FINALIZE, + requires=constants.AMPHORA)) + if is_spare: + create_amp_for_lb_subflow.add( + database_tasks.MarkAmphoraReadyInDB( + name=sf_name + '-' + constants.MARK_AMPHORA_READY_INDB, + requires=constants.AMPHORA)) + else: + create_amp_for_lb_subflow.add( + database_tasks.MarkAmphoraAllocatedInDB( + name=sf_name + '-' + constants.MARK_AMPHORA_ALLOCATED_INDB, + requires=(constants.AMPHORA, constants.LOADBALANCER_ID))) + if role == constants.ROLE_MASTER: + create_amp_for_lb_subflow.add(database_tasks.MarkAmphoraMasterInDB( + name=sf_name + '-' + constants.MARK_AMP_MASTER_INDB, + requires=constants.AMPHORA)) + elif role == constants.ROLE_BACKUP: + create_amp_for_lb_subflow.add(database_tasks.MarkAmphoraBackupInDB( + name=sf_name + '-' + constants.MARK_AMP_BACKUP_INDB, + requires=constants.AMPHORA)) + elif role == constants.ROLE_STANDALONE: + create_amp_for_lb_subflow.add( + database_tasks.MarkAmphoraStandAloneInDB( + name=sf_name + '-' + constants.MARK_AMP_STANDALONE_INDB, + requires=constants.AMPHORA)) return create_amp_for_lb_subflow @@ -268,7 +237,7 @@ class AmphoraFlows(object): return create_amp_for_lb_subflow def get_amphora_for_lb_subflow( - self, prefix, role=constants.ROLE_STANDALONE): + self, prefix, role=constants.ROLE_STANDALONE, is_spare=False): """Tries to allocate a spare amphora to a loadbalancer if none exists, create a new amphora. @@ -276,6 +245,14 @@ class AmphoraFlows(object): sf_name = prefix + '-' + constants.GET_AMPHORA_FOR_LB_SUBFLOW + # Don't replace a spare with another spare, just build a fresh one. + if is_spare: + get_spare_amp_flow = linear_flow.Flow(sf_name) + + get_spare_amp_flow.add(self._get_create_amp_for_lb_subflow( + prefix, role, is_spare=is_spare)) + return get_spare_amp_flow + # We need a graph flow here for a conditional flow amp_for_lb_flow = graph_flow.Flow(sf_name) @@ -318,289 +295,145 @@ class AmphoraFlows(object): decider=self._create_new_amp_for_lb_decider, decider_depth='flow') - # Plug the network - # todo(xgerman): Rework failover flow - if prefix != constants.FAILOVER_AMPHORA_FLOW: - sf_name = prefix + '-' + constants.AMP_PLUG_NET_SUBFLOW - amp_for_lb_net_flow = linear_flow.Flow(sf_name) - amp_for_lb_net_flow.add(amp_for_lb_flow) - amp_for_lb_net_flow.add(*self._get_amp_net_subflow(sf_name)) - return amp_for_lb_net_flow - return amp_for_lb_flow - def _get_amp_net_subflow(self, sf_name): - flows = [] - flows.append(network_tasks.PlugVIPAmpphora( - name=sf_name + '-' + constants.PLUG_VIP_AMPHORA, - requires=(constants.LOADBALANCER, constants.AMPHORA, - constants.SUBNET), - provides=constants.AMP_DATA)) + def get_delete_amphora_flow( + self, amphora, + retry_attempts=CONF.controller_worker.amphora_delete_retries, + retry_interval=( + CONF.controller_worker.amphora_delete_retry_interval)): + """Creates a subflow to delete an amphora and it's port. - flows.append(network_tasks.ApplyQosAmphora( - name=sf_name + '-' + constants.APPLY_QOS_AMP, - requires=(constants.LOADBALANCER, constants.AMP_DATA, - constants.UPDATE_DICT))) - flows.append(database_tasks.UpdateAmphoraVIPData( - name=sf_name + '-' + constants.UPDATE_AMPHORA_VIP_DATA, - requires=constants.AMP_DATA)) - flows.append(database_tasks.ReloadAmphora( - name=sf_name + '-' + constants.RELOAD_AMP_AFTER_PLUG_VIP, - requires=constants.AMPHORA, - provides=constants.AMPHORA)) - flows.append(database_tasks.ReloadLoadBalancer( - name=sf_name + '-' + constants.RELOAD_LB_AFTER_PLUG_VIP, - requires=constants.LOADBALANCER_ID, - provides=constants.LOADBALANCER)) - flows.append(network_tasks.GetAmphoraNetworkConfigs( - name=sf_name + '-' + constants.GET_AMP_NETWORK_CONFIG, - requires=(constants.LOADBALANCER, constants.AMPHORA), - provides=constants.AMPHORA_NETWORK_CONFIG)) - flows.append(amphora_driver_tasks.AmphoraPostVIPPlug( - name=sf_name + '-' + constants.AMP_POST_VIP_PLUG, - rebind={constants.AMPHORAE_NETWORK_CONFIG: - constants.AMPHORA_NETWORK_CONFIG}, - requires=(constants.LOADBALANCER, - constants.AMPHORAE_NETWORK_CONFIG))) - return flows + This flow is idempotent and safe to retry. - def get_delete_amphora_flow(self): - """Creates a flow to delete an amphora. - - This should be configurable in the config file - :returns: The flow for deleting the amphora - :raises AmphoraNotFound: The referenced Amphora was not found + :param amphora: An amphora dict object. + :param retry_attempts: The number of times the flow is retried. + :param retry_interval: The time to wait, in seconds, between retries. + :returns: The subflow for deleting the amphora. + :raises AmphoraNotFound: The referenced Amphora was not found. """ - - delete_amphora_flow = linear_flow.Flow(constants.DELETE_AMPHORA_FLOW) + amphora_id = amphora[constants.ID] + delete_amphora_flow = linear_flow.Flow( + name=constants.DELETE_AMPHORA_FLOW + '-' + amphora_id, + retry=retry_tasks.SleepingRetryTimesController( + name='retry-' + constants.DELETE_AMPHORA_FLOW + '-' + + amphora_id, + attempts=retry_attempts, interval=retry_interval)) delete_amphora_flow.add(lifecycle_tasks.AmphoraToErrorOnRevertTask( - requires=constants.AMPHORA)) - delete_amphora_flow.add(database_tasks. - MarkAmphoraPendingDeleteInDB( - requires=constants.AMPHORA)) - delete_amphora_flow.add(database_tasks. - MarkAmphoraHealthBusy( - requires=constants.AMPHORA)) + name=constants.AMPHORA_TO_ERROR_ON_REVERT + '-' + amphora_id, + inject={constants.AMPHORA: amphora})) + delete_amphora_flow.add( + database_tasks.MarkAmphoraPendingDeleteInDB( + name=constants.MARK_AMPHORA_PENDING_DELETE + '-' + amphora_id, + inject={constants.AMPHORA: amphora})) + delete_amphora_flow.add(database_tasks.MarkAmphoraHealthBusy( + name=constants.MARK_AMPHORA_HEALTH_BUSY + '-' + amphora_id, + inject={constants.AMPHORA: amphora})) delete_amphora_flow.add(compute_tasks.ComputeDelete( - requires=constants.AMPHORA)) - delete_amphora_flow.add(database_tasks. - DisableAmphoraHealthMonitoring( - requires=constants.AMPHORA)) - delete_amphora_flow.add(database_tasks. - MarkAmphoraDeletedInDB( - requires=constants.AMPHORA)) + name=constants.DELETE_AMPHORA + '-' + amphora_id, + inject={constants.AMPHORA: amphora, + constants.PASSIVE_FAILURE: True})) + delete_amphora_flow.add(database_tasks.DisableAmphoraHealthMonitoring( + name=constants.DISABLE_AMP_HEALTH_MONITORING + '-' + amphora_id, + inject={constants.AMPHORA: amphora})) + delete_amphora_flow.add(database_tasks.MarkAmphoraDeletedInDB( + name=constants.MARK_AMPHORA_DELETED + '-' + amphora_id, + inject={constants.AMPHORA: amphora})) + if amphora.get(constants.VRRP_PORT_ID): + delete_amphora_flow.add(network_tasks.DeletePort( + name=(constants.DELETE_PORT + '-' + str(amphora_id) + '-' + + str(amphora[constants.VRRP_PORT_ID])), + inject={constants.PORT_ID: amphora[constants.VRRP_PORT_ID], + constants.PASSIVE_FAILURE: True})) + # TODO(johnsom) What about cleaning up any member ports? + # maybe we should get the list of attached ports prior to delete + # and call delete on them here. Fix this as part of + # https://storyboard.openstack.org/#!/story/2007077 + return delete_amphora_flow - def get_failover_flow(self, role=constants.ROLE_STANDALONE, - load_balancer=None): - """Creates a flow to failover a stale amphora - - :returns: The flow for amphora failover - """ - - failover_amphora_flow = linear_flow.Flow( - constants.FAILOVER_AMPHORA_FLOW) - - failover_amphora_flow.add(lifecycle_tasks.AmphoraToErrorOnRevertTask( - rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, - requires=constants.AMPHORA)) - - failover_amphora_flow.add(network_tasks.FailoverPreparationForAmphora( - rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, - requires=constants.AMPHORA)) - - # Note: It seems intuitive to boot an amphora prior to deleting - # the old amphora, however this is a complicated issue. - # If the target host (due to anit-affinity) is resource - # constrained, this will fail where a post-delete will - # succeed. Since this is async with the API it would result - # in the LB ending in ERROR though the amps are still alive. - # Consider in the future making this a complicated - # try-on-failure-retry flow, or move upgrade failovers to be - # synchronous with the API. For now spares pool and act/stdby - # will mitigate most of this delay. - - # Delete the old amphora - failover_amphora_flow.add( - database_tasks.MarkAmphoraPendingDeleteInDB( - rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, - requires=constants.AMPHORA)) - failover_amphora_flow.add( - database_tasks.MarkAmphoraHealthBusy( - rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, - requires=constants.AMPHORA)) - failover_amphora_flow.add(compute_tasks.ComputeDelete( - rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, - requires=constants.AMPHORA)) - failover_amphora_flow.add(network_tasks.WaitForPortDetach( - rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, - requires=constants.AMPHORA)) - failover_amphora_flow.add(database_tasks.MarkAmphoraDeletedInDB( - rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, - requires=constants.AMPHORA)) - - # If this is an unallocated amp (spares pool), we're done - if not load_balancer: - failover_amphora_flow.add( - database_tasks.DisableAmphoraHealthMonitoring( - rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, - requires=constants.AMPHORA)) - return failover_amphora_flow - - # Save failed amphora details for later - failover_amphora_flow.add( - database_tasks.GetAmphoraDetails( - rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, - requires=constants.AMPHORA, - provides=constants.AMP_DATA)) - - # Get a new amphora - # Note: Role doesn't matter here. We will update it later. - get_amp_subflow = self.get_amphora_for_lb_subflow( - prefix=constants.FAILOVER_AMPHORA_FLOW) - failover_amphora_flow.add(get_amp_subflow) - - # Update the new amphora with the failed amphora details - failover_amphora_flow.add(database_tasks.UpdateAmpFailoverDetails( - requires=(constants.AMPHORA, constants.AMP_DATA))) - - # Update the data stored in the flow from the database - failover_amphora_flow.add(database_tasks.ReloadLoadBalancer( - requires=constants.LOADBALANCER_ID, - provides=constants.LOADBALANCER)) - - failover_amphora_flow.add(database_tasks.ReloadAmphora( - requires=constants.AMPHORA, - provides=constants.AMPHORA)) - - # Prepare to reconnect the network interface(s) - failover_amphora_flow.add(network_tasks.GetAmphoraeNetworkConfigs( - requires=constants.LOADBALANCER, - provides=constants.AMPHORAE_NETWORK_CONFIG)) - failover_amphora_flow.add(database_tasks.GetListenersFromLoadbalancer( - requires=constants.LOADBALANCER, provides=constants.LISTENERS)) - failover_amphora_flow.add(database_tasks.GetAmphoraeFromLoadbalancer( - requires=constants.LOADBALANCER, provides=constants.AMPHORAE)) - - # Plug the VIP ports into the new amphora - # The reason for moving these steps here is the udp listeners want to - # do some kernel configuration before Listener update for forbidding - # failure during rebuild amphora. - failover_amphora_flow.add(network_tasks.PlugVIPPort( - requires=(constants.AMPHORA, constants.AMPHORAE_NETWORK_CONFIG))) - failover_amphora_flow.add(amphora_driver_tasks.AmphoraPostVIPPlug( - requires=(constants.AMPHORA, constants.LOADBALANCER, - constants.AMPHORAE_NETWORK_CONFIG))) - - # Listeners update needs to be run on all amphora to update - # their peer configurations. So parallelize this with an - # unordered subflow. - update_amps_subflow = unordered_flow.Flow( - constants.UPDATE_AMPS_SUBFLOW) - - timeout_dict = { - constants.CONN_MAX_RETRIES: - CONF.haproxy_amphora.active_connection_max_retries, - constants.CONN_RETRY_INTERVAL: - CONF.haproxy_amphora.active_connection_rety_interval} - - # Setup parallel flows for each amp. We don't know the new amp - # details at flow creation time, so setup a subflow for each - # amp on the LB, they let the task index into a list of amps - # to find the amphora it should work on. - amp_index = 0 - db_lb = self.lb_repo.get(db_apis.get_session(), - id=load_balancer[constants.LOADBALANCER_ID]) - for amp in db_lb.amphorae: - if amp.status == constants.DELETED: - continue - update_amps_subflow.add( - amphora_driver_tasks.AmpListenersUpdate( - name=constants.AMP_LISTENER_UPDATE + '-' + str(amp_index), - requires=(constants.LOADBALANCER, constants.AMPHORAE), - inject={constants.AMPHORA_INDEX: amp_index, - constants.TIMEOUT_DICT: timeout_dict})) - amp_index += 1 - - failover_amphora_flow.add(update_amps_subflow) - - # Plug the member networks into the new amphora - failover_amphora_flow.add(network_tasks.CalculateAmphoraDelta( - requires=(constants.LOADBALANCER, constants.AMPHORA, - constants.AVAILABILITY_ZONE), - provides=constants.DELTA)) - - failover_amphora_flow.add(network_tasks.HandleNetworkDelta( - requires=(constants.AMPHORA, constants.DELTA), - provides=constants.ADDED_PORTS)) - - failover_amphora_flow.add(amphora_driver_tasks.AmphoraePostNetworkPlug( - requires=(constants.LOADBALANCER, constants.ADDED_PORTS))) - - failover_amphora_flow.add(database_tasks.ReloadLoadBalancer( - name='octavia-failover-LB-reload-2', - requires=constants.LOADBALANCER_ID, - provides=constants.LOADBALANCER)) - - # Handle the amphora role and VRRP if necessary - if role == constants.ROLE_MASTER: - failover_amphora_flow.add(database_tasks.MarkAmphoraMasterInDB( - name=constants.MARK_AMP_MASTER_INDB, - requires=constants.AMPHORA)) - vrrp_subflow = self.get_vrrp_subflow(role) - failover_amphora_flow.add(vrrp_subflow) - elif role == constants.ROLE_BACKUP: - failover_amphora_flow.add(database_tasks.MarkAmphoraBackupInDB( - name=constants.MARK_AMP_BACKUP_INDB, - requires=constants.AMPHORA)) - vrrp_subflow = self.get_vrrp_subflow(role) - failover_amphora_flow.add(vrrp_subflow) - elif role == constants.ROLE_STANDALONE: - failover_amphora_flow.add( - database_tasks.MarkAmphoraStandAloneInDB( - name=constants.MARK_AMP_STANDALONE_INDB, - requires=constants.AMPHORA)) - - failover_amphora_flow.add(amphora_driver_tasks.ListenersStart( - requires=(constants.LOADBALANCER, constants.AMPHORA))) - failover_amphora_flow.add( - database_tasks.DisableAmphoraHealthMonitoring( - rebind={constants.AMPHORA: constants.FAILED_AMPHORA}, - requires=constants.AMPHORA)) - - return failover_amphora_flow - - def get_vrrp_subflow(self, prefix): + def get_vrrp_subflow(self, prefix, timeout_dict=None, + create_vrrp_group=True): sf_name = prefix + '-' + constants.GET_VRRP_SUBFLOW vrrp_subflow = linear_flow.Flow(sf_name) + + # Optimization for failover flow. No reason to call this + # when configuring the secondary amphora. + if create_vrrp_group: + vrrp_subflow.add(database_tasks.CreateVRRPGroupForLB( + name=sf_name + '-' + constants.CREATE_VRRP_GROUP_FOR_LB, + requires=constants.LOADBALANCER_ID)) + vrrp_subflow.add(network_tasks.GetAmphoraeNetworkConfigs( name=sf_name + '-' + constants.GET_AMP_NETWORK_CONFIG, - requires=constants.LOADBALANCER, + requires=constants.LOADBALANCER_ID, provides=constants.AMPHORAE_NETWORK_CONFIG)) - vrrp_subflow.add(amphora_driver_tasks.AmphoraUpdateVRRPInterface( - name=sf_name + '-' + constants.AMP_UPDATE_VRRP_INTF, - requires=constants.LOADBALANCER, - provides=constants.LOADBALANCER)) - vrrp_subflow.add(database_tasks.CreateVRRPGroupForLB( - name=sf_name + '-' + constants.CREATE_VRRP_GROUP_FOR_LB, - requires=constants.LOADBALANCER, - provides=constants.LOADBALANCER)) - vrrp_subflow.add(amphora_driver_tasks.AmphoraVRRPUpdate( - name=sf_name + '-' + constants.AMP_VRRP_UPDATE, - requires=(constants.LOADBALANCER, - constants.AMPHORAE_NETWORK_CONFIG))) - vrrp_subflow.add(amphora_driver_tasks.AmphoraVRRPStart( - name=sf_name + '-' + constants.AMP_VRRP_START, - requires=constants.LOADBALANCER)) + + # VRRP update needs to be run on all amphora to update + # their peer configurations. So parallelize this with an + # unordered subflow. + update_amps_subflow = unordered_flow.Flow('VRRP-update-subflow') + + # We have three tasks to run in order, per amphora + amp_0_subflow = linear_flow.Flow('VRRP-amp-0-update-subflow') + + amp_0_subflow.add(amphora_driver_tasks.AmphoraIndexUpdateVRRPInterface( + name=sf_name + '-0-' + constants.AMP_UPDATE_VRRP_INTF, + requires=constants.AMPHORAE, + inject={constants.AMPHORA_INDEX: 0, + constants.TIMEOUT_DICT: timeout_dict}, + provides=constants.AMP_VRRP_INT)) + + amp_0_subflow.add(amphora_driver_tasks.AmphoraIndexVRRPUpdate( + name=sf_name + '-0-' + constants.AMP_VRRP_UPDATE, + requires=(constants.LOADBALANCER_ID, + constants.AMPHORAE_NETWORK_CONFIG, constants.AMPHORAE, + constants.AMP_VRRP_INT), + inject={constants.AMPHORA_INDEX: 0, + constants.TIMEOUT_DICT: timeout_dict})) + + amp_0_subflow.add(amphora_driver_tasks.AmphoraIndexVRRPStart( + name=sf_name + '-0-' + constants.AMP_VRRP_START, + requires=constants.AMPHORAE, + inject={constants.AMPHORA_INDEX: 0, + constants.TIMEOUT_DICT: timeout_dict})) + + amp_1_subflow = linear_flow.Flow('VRRP-amp-1-update-subflow') + + amp_1_subflow.add(amphora_driver_tasks.AmphoraIndexUpdateVRRPInterface( + name=sf_name + '-1-' + constants.AMP_UPDATE_VRRP_INTF, + requires=constants.AMPHORAE, + inject={constants.AMPHORA_INDEX: 1, + constants.TIMEOUT_DICT: timeout_dict}, + provides=constants.AMP_VRRP_INT)) + + amp_1_subflow.add(amphora_driver_tasks.AmphoraIndexVRRPUpdate( + name=sf_name + '-1-' + constants.AMP_VRRP_UPDATE, + requires=(constants.LOADBALANCER_ID, + constants.AMPHORAE_NETWORK_CONFIG, constants.AMPHORAE, + constants.AMP_VRRP_INT), + inject={constants.AMPHORA_INDEX: 1, + constants.TIMEOUT_DICT: timeout_dict})) + amp_1_subflow.add(amphora_driver_tasks.AmphoraIndexVRRPStart( + name=sf_name + '-1-' + constants.AMP_VRRP_START, + requires=constants.AMPHORAE, + inject={constants.AMPHORA_INDEX: 1, + constants.TIMEOUT_DICT: timeout_dict})) + + update_amps_subflow.add(amp_0_subflow) + update_amps_subflow.add(amp_1_subflow) + + vrrp_subflow.add(update_amps_subflow) + return vrrp_subflow def cert_rotate_amphora_flow(self): """Implement rotation for amphora's cert. - 1. Create a new certificate - 2. Upload the cert to amphora - 3. update the newly created certificate info to amphora - 4. update the cert_busy flag to be false after rotation + 1. Create a new certificate + 2. Upload the cert to amphora + 3. update the newly created certificate info to amphora + 4. update the cert_busy flag to be false after rotation :returns: The flow for updating an amphora """ @@ -644,3 +477,254 @@ class AmphoraFlows(object): requires=(constants.AMPHORA, constants.FLAVOR))) return update_amphora_flow + + def get_amphora_for_lb_failover_subflow( + self, prefix, role=constants.ROLE_STANDALONE, + failed_amp_vrrp_port_id=None, is_vrrp_ipv6=False, is_spare=False): + """Creates a new amphora that will be used in a failover flow. + + :requires: loadbalancer_id, flavor, vip, vip_sg_id, loadbalancer + :provides: amphora_id, amphora + :param prefix: The flow name prefix to use on the flow and tasks. + :param role: The role this amphora will have in the topology. + :param failed_amp_vrrp_port_id: The base port ID of the failed amp. + :param is_vrrp_ipv6: True if the base port IP is IPv6. + :param is_spare: True if we are getting a spare amphroa. + :return: A Taskflow sub-flow that will create the amphora. + """ + + sf_name = prefix + '-' + constants.CREATE_AMP_FOR_FAILOVER_SUBFLOW + + amp_for_failover_flow = linear_flow.Flow(sf_name) + + # Try to allocate or boot an amphora instance (unconfigured) + amp_for_failover_flow.add(self.get_amphora_for_lb_subflow( + prefix=prefix + '-' + constants.FAILOVER_LOADBALANCER_FLOW, + role=role, is_spare=is_spare)) + + # If we are getting a spare amphora, this is all we need to do. + if is_spare: + return amp_for_failover_flow + + # Create the VIP base (aka VRRP) port for the amphora. + amp_for_failover_flow.add(network_tasks.CreateVIPBasePort( + name=prefix + '-' + constants.CREATE_VIP_BASE_PORT, + requires=(constants.VIP, constants.VIP_SG_ID, + constants.AMPHORA_ID), + provides=constants.BASE_PORT)) + + # Attach the VIP base (aka VRRP) port to the amphora. + amp_for_failover_flow.add(compute_tasks.AttachPort( + name=prefix + '-' + constants.ATTACH_PORT, + requires=(constants.AMPHORA, constants.PORT), + rebind={constants.PORT: constants.BASE_PORT})) + + # Update the amphora database record with the VIP base port info. + amp_for_failover_flow.add(database_tasks.UpdateAmpFailoverDetails( + name=prefix + '-' + constants.UPDATE_AMP_FAILOVER_DETAILS, + requires=(constants.AMPHORA, constants.VIP, constants.BASE_PORT))) + + # Update the amphora networking for the plugged VIP port + amp_for_failover_flow.add(network_tasks.GetAmphoraNetworkConfigsByID( + name=prefix + '-' + constants.GET_AMPHORA_NETWORK_CONFIGS_BY_ID, + requires=(constants.LOADBALANCER_ID, constants.AMPHORA_ID), + provides=constants.AMPHORAE_NETWORK_CONFIG)) + + # Disable the base (vrrp) port on the failed amphora + # This prevents a DAD failure when bringing up the new amphora. + # Keepalived will handle this for act/stdby. + if (role == constants.ROLE_STANDALONE and failed_amp_vrrp_port_id and + is_vrrp_ipv6): + amp_for_failover_flow.add(network_tasks.AdminDownPort( + name=prefix + '-' + constants.ADMIN_DOWN_PORT, + inject={constants.PORT_ID: failed_amp_vrrp_port_id})) + + amp_for_failover_flow.add(amphora_driver_tasks.AmphoraPostVIPPlug( + name=prefix + '-' + constants.AMPHORA_POST_VIP_PLUG, + requires=(constants.AMPHORA, constants.LOADBALANCER, + constants.AMPHORAE_NETWORK_CONFIG))) + + # Plug member ports + amp_for_failover_flow.add(network_tasks.CalculateAmphoraDelta( + name=prefix + '-' + constants.CALCULATE_AMPHORA_DELTA, + requires=(constants.LOADBALANCER, constants.AMPHORA, + constants.AVAILABILITY_ZONE, constants.VRRP_PORT), + rebind={constants.VRRP_PORT: constants.BASE_PORT}, + provides=constants.DELTA)) + + amp_for_failover_flow.add(network_tasks.HandleNetworkDelta( + name=prefix + '-' + constants.HANDLE_NETWORK_DELTA, + requires=(constants.AMPHORA, constants.DELTA), + provides=constants.ADDED_PORTS)) + + amp_for_failover_flow.add(amphora_driver_tasks.AmphoraePostNetworkPlug( + name=prefix + '-' + constants.AMPHORAE_POST_NETWORK_PLUG, + requires=(constants.LOADBALANCER, constants.ADDED_PORTS))) + + return amp_for_failover_flow + + def get_failover_amphora_flow(self, failed_amphora, lb_amp_count): + """Get a Taskflow flow to failover an amphora. + + 1. Build a replacement amphora. + 2. Delete the old amphora. + 3. Update the amphorae listener configurations. + 4. Update the VRRP configurations if needed. + + :param failed_amphora: The amphora dict to failover. + :param lb_amp_count: The number of amphora on this load balancer. + :returns: The flow that will provide the failover. + """ + failover_amp_flow = linear_flow.Flow( + constants.FAILOVER_AMPHORA_FLOW) + + # Revert amphora to status ERROR if this flow goes wrong + failover_amp_flow.add(lifecycle_tasks.AmphoraToErrorOnRevertTask( + requires=constants.AMPHORA, + inject={constants.AMPHORA: failed_amphora})) + + if failed_amphora[constants.ROLE] in (constants.ROLE_MASTER, + constants.ROLE_BACKUP): + amp_role = 'master_or_backup' + elif failed_amphora[constants.ROLE] == constants.ROLE_STANDALONE: + amp_role = 'standalone' + elif failed_amphora[constants.ROLE] is None: + amp_role = 'spare' + else: + amp_role = 'undefined' + LOG.info("Performing failover for amphora: %s", + {"id": failed_amphora[constants.ID], + "load_balancer_id": failed_amphora.get( + constants.LOAD_BALANCER_ID), + "lb_network_ip": failed_amphora.get(constants.LB_NETWORK_IP), + "compute_id": failed_amphora.get(constants.COMPUTE_ID), + "role": amp_role}) + + failover_amp_flow.add(database_tasks.MarkAmphoraPendingDeleteInDB( + requires=constants.AMPHORA, + inject={constants.AMPHORA: failed_amphora})) + + failover_amp_flow.add(database_tasks.MarkAmphoraHealthBusy( + requires=constants.AMPHORA, + inject={constants.AMPHORA: failed_amphora})) + + failover_amp_flow.add(network_tasks.GetVIPSecurityGroupID( + requires=constants.LOADBALANCER_ID, + provides=constants.VIP_SG_ID)) + + is_spare = True + is_vrrp_ipv6 = False + if failed_amphora.get(constants.LOAD_BALANCER_ID): + is_spare = False + if failed_amphora.get(constants.VRRP_IP): + is_vrrp_ipv6 = utils.is_ipv6(failed_amphora[constants.VRRP_IP]) + + # Get a replacement amphora and plug all of the networking. + # + # Do this early as the compute services have been observed to be + # unreliable. The community decided the chance that deleting first + # would open resources for an instance is less likely than the + # compute service failing to boot an instance for other reasons. + + # TODO(johnsom) Move this back out to run for spares after + # delete amphora API is available. + failover_amp_flow.add(self.get_amphora_for_lb_failover_subflow( + prefix=constants.FAILOVER_LOADBALANCER_FLOW, + role=failed_amphora[constants.ROLE], + failed_amp_vrrp_port_id=failed_amphora.get( + constants.VRRP_PORT_ID), + is_vrrp_ipv6=is_vrrp_ipv6, + is_spare=is_spare)) + + failover_amp_flow.add( + self.get_delete_amphora_flow( + failed_amphora, + retry_attempts=CONF.controller_worker.amphora_delete_retries, + retry_interval=( + CONF.controller_worker.amphora_delete_retry_interval))) + failover_amp_flow.add( + database_tasks.DisableAmphoraHealthMonitoring( + requires=constants.AMPHORA, + inject={constants.AMPHORA: failed_amphora})) + + if not failed_amphora.get(constants.LOAD_BALANCER_ID): + # This is an unallocated amphora (spares pool), we are done. + return failover_amp_flow + + failover_amp_flow.add(database_tasks.GetLoadBalancer( + requires=constants.LOADBALANCER_ID, + inject={constants.LOADBALANCER_ID: + failed_amphora[constants.LOAD_BALANCER_ID]}, + provides=constants.LOADBALANCER)) + + failover_amp_flow.add(database_tasks.GetAmphoraeFromLoadbalancer( + name=constants.GET_AMPHORAE_FROM_LB, + requires=constants.LOADBALANCER_ID, + inject={constants.LOADBALANCER_ID: + failed_amphora[constants.LOAD_BALANCER_ID]}, + provides=constants.AMPHORAE)) + + # Setup timeouts for our requests to the amphorae + timeout_dict = { + constants.CONN_MAX_RETRIES: + CONF.haproxy_amphora.active_connection_max_retries, + constants.CONN_RETRY_INTERVAL: + CONF.haproxy_amphora.active_connection_rety_interval} + + # Listeners update needs to be run on all amphora to update + # their peer configurations. So parallelize this with an + # unordered subflow. + update_amps_subflow = unordered_flow.Flow( + constants.UPDATE_AMPS_SUBFLOW) + + for amp_index in range(0, lb_amp_count): + update_amps_subflow.add( + amphora_driver_tasks.AmphoraIndexListenerUpdate( + name=str(amp_index) + '-' + constants.AMP_LISTENER_UPDATE, + requires=(constants.LOADBALANCER, constants.AMPHORAE), + inject={constants.AMPHORA_INDEX: amp_index, + constants.TIMEOUT_DICT: timeout_dict})) + + failover_amp_flow.add(update_amps_subflow) + + # Configure and enable keepalived in the amphora + if lb_amp_count == 2: + failover_amp_flow.add( + self.get_vrrp_subflow(constants.GET_VRRP_SUBFLOW, + timeout_dict, create_vrrp_group=False)) + + # Reload the listener. This needs to be done here because + # it will create the required haproxy check scripts for + # the VRRP deployed above. + # A "U" or newer amphora-agent will remove the need for this + # task here. + # TODO(johnsom) Remove this in the "W" cycle + reload_listener_subflow = unordered_flow.Flow( + constants.AMPHORA_LISTENER_RELOAD_SUBFLOW) + + for amp_index in range(0, lb_amp_count): + reload_listener_subflow.add( + amphora_driver_tasks.AmphoraIndexListenersReload( + name=(str(amp_index) + '-' + + constants.AMPHORA_RELOAD_LISTENER), + requires=(constants.LOADBALANCER, constants.AMPHORAE), + inject={constants.AMPHORA_INDEX: amp_index, + constants.TIMEOUT_DICT: timeout_dict})) + + failover_amp_flow.add(reload_listener_subflow) + + # Remove any extraneous ports + # Note: Nova sometimes fails to delete ports attached to an instance. + # For example, if you create an LB with a listener, then + # 'openstack server delete' the amphora, you will see the vrrp + # port attached to that instance will remain after the instance + # is deleted. + # TODO(johnsom) Fix this as part of + # https://storyboard.openstack.org/#!/story/2007077 + + # Mark LB ACTIVE + failover_amp_flow.add( + database_tasks.MarkLBActiveInDB(mark_subobjects=True, + requires=constants.LOADBALANCER)) + + return failover_amp_flow diff --git a/octavia/controller/worker/v2/flows/flow_utils.py b/octavia/controller/worker/v2/flows/flow_utils.py index 58ad1bd3b5..f4de5cc58d 100644 --- a/octavia/controller/worker/v2/flows/flow_utils.py +++ b/octavia/controller/worker/v2/flows/flow_utils.py @@ -11,7 +11,7 @@ # License for the specific language governing permissions and limitations # under the License. -from octavia.common import constants +from octavia.api.drivers import utils as provider_utils from octavia.controller.worker.v2.flows import amphora_flows from octavia.controller.worker.v2.flows import health_monitor_flows from octavia.controller.worker.v2.flows import l7policy_flows @@ -41,16 +41,36 @@ def get_delete_load_balancer_flow(lb): return LB_FLOWS.get_delete_load_balancer_flow(lb) -def get_delete_listeners_store(lb): - return LB_FLOWS.get_delete_listeners_store(lb) +def get_listeners_on_lb(db_lb): + """Get a list of the listeners on a load balancer. + + :param db_lb: A load balancer database model object. + :returns: A list of provider dict format listeners. + """ + listener_dicts = [] + for listener in db_lb.listeners: + prov_listener = provider_utils.db_listener_to_provider_listener( + listener) + listener_dicts.append(prov_listener.to_dict()) + return listener_dicts -def get_delete_pools_store(lb): - return LB_FLOWS.get_delete_pools_store(lb) +def get_pools_on_lb(db_lb): + """Get a list of the pools on a load balancer. + + :param db_lb: A load balancer database model object. + :returns: A list of provider dict format pools. + """ + pool_dicts = [] + for pool in db_lb.pools: + prov_pool = provider_utils.db_pool_to_provider_pool(pool) + pool_dicts.append(prov_pool.to_dict()) + return pool_dicts -def get_cascade_delete_load_balancer_flow(lb): - return LB_FLOWS.get_cascade_delete_load_balancer_flow(lb) +def get_cascade_delete_load_balancer_flow(lb, listeners=(), pools=()): + return LB_FLOWS.get_cascade_delete_load_balancer_flow(lb, listeners, + pools) def get_update_load_balancer_flow(): @@ -61,12 +81,17 @@ def get_create_amphora_flow(): return AMP_FLOWS.get_create_amphora_flow() -def get_delete_amphora_flow(): - return AMP_FLOWS.get_delete_amphora_flow() +def get_delete_amphora_flow(amphora, retry_attempts=None, retry_interval=None): + return AMP_FLOWS.get_delete_amphora_flow(amphora, retry_attempts, + retry_interval) -def get_failover_flow(role=constants.ROLE_STANDALONE, load_balancer=None): - return AMP_FLOWS.get_failover_flow(role=role, load_balancer=load_balancer) +def get_failover_LB_flow(amps, lb): + return LB_FLOWS.get_failover_LB_flow(amps, lb) + + +def get_failover_amphora_flow(amphora_dict, lb_amp_count): + return AMP_FLOWS.get_failover_amphora_flow(amphora_dict, lb_amp_count) def cert_rotate_amphora_flow(): diff --git a/octavia/controller/worker/v2/flows/listener_flows.py b/octavia/controller/worker/v2/flows/listener_flows.py index f8ae183102..ae22835a96 100644 --- a/octavia/controller/worker/v2/flows/listener_flows.py +++ b/octavia/controller/worker/v2/flows/listener_flows.py @@ -83,24 +83,26 @@ class ListenerFlows(object): return delete_listener_flow - def get_delete_listener_internal_flow(self, listener_name): + def get_delete_listener_internal_flow(self, listener): """Create a flow to delete a listener and l7policies internally (will skip deletion on the amp and marking LB active) :returns: The flow for deleting a listener """ - delete_listener_flow = linear_flow.Flow(constants.DELETE_LISTENER_FLOW) + listener_id = listener[constants.LISTENER_ID] + delete_listener_flow = linear_flow.Flow( + constants.DELETE_LISTENER_FLOW + '-' + listener_id) # Should cascade delete all L7 policies delete_listener_flow.add(network_tasks.UpdateVIPForDelete( - name='delete_update_vip_' + listener_name, + name='delete_update_vip_' + listener_id, requires=constants.LOADBALANCER_ID)) delete_listener_flow.add(database_tasks.DeleteListenerInDB( - name='delete_listener_in_db_' + listener_name, + name='delete_listener_in_db_' + listener_id, requires=constants.LISTENER, - rebind={constants.LISTENER: listener_name})) + inject={constants.LISTENER: listener})) delete_listener_flow.add(database_tasks.DecrementListenerQuota( - name='decrement_listener_quota_' + listener_name, + name='decrement_listener_quota_' + listener_id, requires=constants.PROJECT_ID)) return delete_listener_flow diff --git a/octavia/controller/worker/v2/flows/load_balancer_flows.py b/octavia/controller/worker/v2/flows/load_balancer_flows.py index e601bae86e..17f5be9760 100644 --- a/octavia/controller/worker/v2/flows/load_balancer_flows.py +++ b/octavia/controller/worker/v2/flows/load_balancer_flows.py @@ -1,4 +1,5 @@ # Copyright 2015 Hewlett-Packard Development Company, L.P. +# Copyright 2020 Red Hat, Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -18,9 +19,9 @@ from oslo_log import log as logging from taskflow.patterns import linear_flow from taskflow.patterns import unordered_flow -from octavia.api.drivers import utils as provider_utils from octavia.common import constants from octavia.common import exceptions +from octavia.common import utils from octavia.controller.worker.v2.flows import amphora_flows from octavia.controller.worker.v2.flows import listener_flows from octavia.controller.worker.v2.flows import member_flows @@ -30,7 +31,6 @@ from octavia.controller.worker.v2.tasks import compute_tasks from octavia.controller.worker.v2.tasks import database_tasks from octavia.controller.worker.v2.tasks import lifecycle_tasks from octavia.controller.worker.v2.tasks import network_tasks -from octavia.db import api as db_apis from octavia.db import repositories as repo CONF = cfg.CONF @@ -72,7 +72,7 @@ class LoadBalancerFlows(object): requires=(constants.LOADBALANCER_ID, constants.VIP), provides=constants.LOADBALANCER)) lb_create_flow.add(network_tasks.UpdateVIPSecurityGroup( - requires=constants.LOADBALANCER)) + requires=constants.LOADBALANCER_ID)) lb_create_flow.add(network_tasks.GetSubnetFromVIP( requires=constants.LOADBALANCER, provides=constants.SUBNET)) @@ -97,9 +97,15 @@ class LoadBalancerFlows(object): return lb_create_flow def _create_single_topology(self): - return (self.amp_flows.get_amphora_for_lb_subflow( + sf_name = (constants.ROLE_STANDALONE + '-' + + constants.AMP_PLUG_NET_SUBFLOW) + amp_for_lb_net_flow = linear_flow.Flow(sf_name) + amp_for_lb_flow = self.amp_flows.get_amphora_for_lb_subflow( prefix=constants.ROLE_STANDALONE, - role=constants.ROLE_STANDALONE), ) + role=constants.ROLE_STANDALONE) + amp_for_lb_net_flow.add(amp_for_lb_flow) + amp_for_lb_net_flow.add(*self._get_amp_net_subflow(sf_name)) + return amp_for_lb_net_flow def _create_active_standby_topology( self, lf_name=constants.CREATE_LOADBALANCER_FLOW): @@ -128,16 +134,52 @@ class LoadBalancerFlows(object): f_name = constants.CREATE_LOADBALANCER_FLOW amps_flow = unordered_flow.Flow(f_name) - master_amp_sf = self.amp_flows.get_amphora_for_lb_subflow( - prefix=constants.ROLE_MASTER, role=constants.ROLE_MASTER - ) - backup_amp_sf = self.amp_flows.get_amphora_for_lb_subflow( - prefix=constants.ROLE_BACKUP, role=constants.ROLE_BACKUP) + master_sf_name = (constants.ROLE_MASTER + '-' + + constants.AMP_PLUG_NET_SUBFLOW) + master_amp_sf = linear_flow.Flow(master_sf_name) + master_amp_sf.add(self.amp_flows.get_amphora_for_lb_subflow( + prefix=constants.ROLE_MASTER, role=constants.ROLE_MASTER)) + master_amp_sf.add(*self._get_amp_net_subflow(master_sf_name)) + + backup_sf_name = (constants.ROLE_BACKUP + '-' + + constants.AMP_PLUG_NET_SUBFLOW) + backup_amp_sf = linear_flow.Flow(backup_sf_name) + backup_amp_sf.add(self.amp_flows.get_amphora_for_lb_subflow( + prefix=constants.ROLE_BACKUP, role=constants.ROLE_BACKUP)) + backup_amp_sf.add(*self._get_amp_net_subflow(backup_sf_name)) + amps_flow.add(master_amp_sf, backup_amp_sf) return flows + [amps_flow] + def _get_amp_net_subflow(self, sf_name): + flows = [] + flows.append(network_tasks.PlugVIPAmphora( + name=sf_name + '-' + constants.PLUG_VIP_AMPHORA, + requires=(constants.LOADBALANCER, constants.AMPHORA, + constants.SUBNET), + provides=constants.AMP_DATA)) + + flows.append(network_tasks.ApplyQosAmphora( + name=sf_name + '-' + constants.APPLY_QOS_AMP, + requires=(constants.LOADBALANCER, constants.AMP_DATA, + constants.UPDATE_DICT))) + flows.append(database_tasks.UpdateAmphoraVIPData( + name=sf_name + '-' + constants.UPDATE_AMPHORA_VIP_DATA, + requires=constants.AMP_DATA)) + flows.append(network_tasks.GetAmphoraNetworkConfigs( + name=sf_name + '-' + constants.GET_AMP_NETWORK_CONFIG, + requires=(constants.LOADBALANCER, constants.AMPHORA), + provides=constants.AMPHORA_NETWORK_CONFIG)) + flows.append(amphora_driver_tasks.AmphoraPostVIPPlug( + name=sf_name + '-' + constants.AMP_POST_VIP_PLUG, + rebind={constants.AMPHORAE_NETWORK_CONFIG: + constants.AMPHORA_NETWORK_CONFIG}, + requires=(constants.LOADBALANCER, + constants.AMPHORAE_NETWORK_CONFIG))) + return flows + def _create_listeners_flow(self): flows = [] flows.append( @@ -181,13 +223,6 @@ class LoadBalancerFlows(object): created/allocated amphorae. :return: Post amphorae association subflow """ - - # Note: If any task in this flow failed, the created amphorae will be - # left ''incorrectly'' allocated to the loadbalancer. Likely, - # the get_new_LB_networking_subflow is the most prune to failure - # shall deallocate the amphora from its loadbalancer and put it in a - # READY state. - sf_name = prefix + '-' + constants.POST_LB_AMP_ASSOCIATION_SUBFLOW post_create_LB_flow = linear_flow.Flow(sf_name) post_create_LB_flow.add( @@ -197,6 +232,9 @@ class LoadBalancerFlows(object): provides=constants.LOADBALANCER)) if topology == constants.TOPOLOGY_ACTIVE_STANDBY: + post_create_LB_flow.add(database_tasks.GetAmphoraeFromLoadbalancer( + requires=constants.LOADBALANCER_ID, + provides=constants.AMPHORAE)) vrrp_subflow = self.amp_flows.get_vrrp_subflow(prefix) post_create_LB_flow.add(vrrp_subflow) @@ -208,36 +246,19 @@ class LoadBalancerFlows(object): requires=constants.LOADBALANCER)) return post_create_LB_flow - def _get_delete_listeners_flow(self, lb): + def _get_delete_listeners_flow(self, listeners): """Sets up an internal delete flow - Because task flow doesn't support loops we store each listener - we want to delete in the store part and then rebind - :param lb: load balancer - :return: (flow, store) -- flow for the deletion and store with all - the listeners stored properly + :param listeners: A list of listener dicts + :return: The flow for the deletion """ - listeners_delete_flow = unordered_flow.Flow('listener_delete_flow') - db_lb = self.lb_repo.get(db_apis.get_session(), - id=lb[constants.LOADBALANCER_ID]) - for listener in db_lb.listeners: - listener_name = 'listener_' + listener.id + listeners_delete_flow = unordered_flow.Flow('listeners_delete_flow') + for listener in listeners: listeners_delete_flow.add( self.listener_flows.get_delete_listener_internal_flow( - listener_name)) + listener)) return listeners_delete_flow - def get_delete_listeners_store(self, lb): - store = {} - for listener in lb.listeners: - listener_name = 'listener_' + listener.id - prov_listener = provider_utils.db_listener_to_provider_listener( - listener) - store[listener_name] = prov_listener.to_dict() - store.update({constants.LOADBALANCER_ID: lb.id, - constants.PROJECT_ID: lb.project_id}) - return store - def get_delete_load_balancer_flow(self, lb): """Creates a flow to delete a load balancer. @@ -245,14 +266,7 @@ class LoadBalancerFlows(object): """ return self._get_delete_load_balancer_flow(lb, False) - def get_delete_pools_store(self, lb): - store = {} - for pool in lb.pools: - pool_name = 'pool' + pool.id - store[pool_name] = pool.id - return store - - def _get_delete_pools_flow(self, lb): + def _get_delete_pools_flow(self, pools): """Sets up an internal delete flow Because task flow doesn't support loops we store each pool @@ -262,16 +276,14 @@ class LoadBalancerFlows(object): the listeners stored properly """ pools_delete_flow = unordered_flow.Flow('pool_delete_flow') - db_lb = self.lb_repo.get(db_apis.get_session(), - id=lb[constants.LOADBALANCER_ID]) - for pool in db_lb.pools: - pool_name = 'pool' + pool.id + for pool in pools: pools_delete_flow.add( self.pool_flows.get_delete_pool_flow_internal( - pool_name)) + pool[constants.POOL_ID])) return pools_delete_flow - def _get_delete_load_balancer_flow(self, lb, cascade): + def _get_delete_load_balancer_flow(self, lb, cascade, + listeners=(), pools=()): delete_LB_flow = linear_flow.Flow(constants.DELETE_LOADBALANCER_FLOW) delete_LB_flow.add(lifecycle_tasks.LoadBalancerToErrorOnRevertTask( requires=constants.LOADBALANCER)) @@ -280,8 +292,8 @@ class LoadBalancerFlows(object): delete_LB_flow.add(database_tasks.MarkLBAmphoraeHealthBusy( requires=constants.LOADBALANCER)) if cascade: - listeners_delete = self._get_delete_listeners_flow(lb) - pools_delete = self._get_delete_pools_flow(lb) + listeners_delete = self._get_delete_listeners_flow(listeners) + pools_delete = self._get_delete_pools_flow(pools) delete_LB_flow.add(pools_delete) delete_LB_flow.add(listeners_delete) delete_LB_flow.add(network_tasks.UnplugVIP( @@ -300,47 +312,14 @@ class LoadBalancerFlows(object): requires=constants.PROJECT_ID)) return delete_LB_flow - def get_cascade_delete_load_balancer_flow(self, lb): + def get_cascade_delete_load_balancer_flow(self, lb, listeners, pools): """Creates a flow to delete a load balancer. :returns: The flow for deleting a load balancer """ - return self._get_delete_load_balancer_flow(lb, True) - - def get_new_LB_networking_subflow(self): - """Create a sub-flow to setup networking. - - :returns: The flow to setup networking for a new amphora - """ - - new_LB_net_subflow = linear_flow.Flow(constants. - LOADBALANCER_NETWORKING_SUBFLOW) - new_LB_net_subflow.add(network_tasks.AllocateVIP( - requires=constants.LOADBALANCER, - provides=constants.VIP)) - new_LB_net_subflow.add(database_tasks.UpdateVIPAfterAllocation( - requires=(constants.LOADBALANCER_ID, constants.VIP), - provides=constants.LOADBALANCER)) - new_LB_net_subflow.add(network_tasks.PlugVIP( - requires=constants.LOADBALANCER, - provides=constants.AMPS_DATA)) - new_LB_net_subflow.add(network_tasks.ApplyQos( - requires=(constants.LOADBALANCER, constants.AMPS_DATA, - constants.UPDATE_DICT))) - new_LB_net_subflow.add(database_tasks.UpdateAmphoraeVIPData( - requires=constants.AMPS_DATA)) - new_LB_net_subflow.add(database_tasks.ReloadLoadBalancer( - name=constants.RELOAD_LB_AFTER_PLUG_VIP, - requires=constants.LOADBALANCER_ID, - provides=constants.LOADBALANCER)) - new_LB_net_subflow.add(network_tasks.GetAmphoraeNetworkConfigs( - requires=constants.LOADBALANCER, - provides=constants.AMPHORAE_NETWORK_CONFIG)) - new_LB_net_subflow.add(amphora_driver_tasks.AmphoraePostVIPPlug( - requires=(constants.LOADBALANCER, - constants.AMPHORAE_NETWORK_CONFIG))) - - return new_LB_net_subflow + return self._get_delete_load_balancer_flow(lb, True, + listeners=listeners, + pools=pools) def get_update_load_balancer_flow(self): """Creates a flow to update a load balancer. @@ -360,3 +339,341 @@ class LoadBalancerFlows(object): requires=constants.LOADBALANCER)) return update_LB_flow + + def get_failover_LB_flow(self, amps, lb): + """Failover a load balancer. + + 1. Validate the VIP port is correct and present. + 2. Build a replacement amphora. + 3. Delete the failed amphora. + 4. Configure the replacement amphora listeners. + 5. Configure VRRP for the listeners. + 6. Build the second replacement amphora. + 7. Delete the second failed amphora. + 8. Delete any extraneous amphora. + 9. Configure the listeners on the new amphorae. + 10. Configure the VRRP on the new amphorae. + 11. Reload the listener configurations to pick up VRRP changes. + 12. Mark the load balancer back to ACTIVE. + + :returns: The flow that will provide the failover. + """ + lb_topology = lb[constants.FLAVOR][constants.LOADBALANCER_TOPOLOGY] + # Pick one amphora to be failed over if any exist. + failed_amp = None + if amps: + failed_amp = amps.pop() + + failover_LB_flow = linear_flow.Flow( + constants.FAILOVER_LOADBALANCER_FLOW) + + # Revert LB to provisioning_status ERROR if this flow goes wrong + failover_LB_flow.add(lifecycle_tasks.LoadBalancerToErrorOnRevertTask( + requires=constants.LOADBALANCER)) + + # Setup timeouts for our requests to the amphorae + timeout_dict = { + constants.CONN_MAX_RETRIES: + CONF.haproxy_amphora.active_connection_max_retries, + constants.CONN_RETRY_INTERVAL: + CONF.haproxy_amphora.active_connection_rety_interval} + + if failed_amp: + failed_amp_role = failed_amp.get(constants.ROLE) + if failed_amp_role in (constants.ROLE_MASTER, + constants.ROLE_BACKUP): + amp_role = 'master_or_backup' + elif failed_amp_role == constants.ROLE_STANDALONE: + amp_role = 'standalone' + elif failed_amp_role is None: + amp_role = 'spare' + else: + amp_role = 'undefined' + LOG.info("Performing failover for amphora: %s", + {"id": failed_amp.get(constants.ID), + "load_balancer_id": lb.get(constants.ID), + "lb_network_ip": failed_amp.get(constants.LB_NETWORK_IP), + "compute_id": failed_amp.get(constants.COMPUTE_ID), + "role": amp_role}) + + failover_LB_flow.add(database_tasks.MarkAmphoraPendingDeleteInDB( + requires=constants.AMPHORA, + inject={constants.AMPHORA: failed_amp})) + + failover_LB_flow.add(database_tasks.MarkAmphoraHealthBusy( + requires=constants.AMPHORA, + inject={constants.AMPHORA: failed_amp})) + + # Check that the VIP port exists and is ok + failover_LB_flow.add( + network_tasks.AllocateVIP(requires=constants.LOADBALANCER, + provides=constants.VIP)) + + # Update the database with the VIP information + failover_LB_flow.add(database_tasks.UpdateVIPAfterAllocation( + requires=(constants.LOADBALANCER_ID, constants.VIP), + provides=constants.LOADBALANCER)) + + # Make sure the SG has the correct rules and re-apply to the + # VIP port. It is not used on the VIP port, but will help lock + # the SG as in use. + failover_LB_flow.add(network_tasks.UpdateVIPSecurityGroup( + requires=constants.LOADBALANCER_ID, provides=constants.VIP_SG_ID)) + + new_amp_role = constants.ROLE_STANDALONE + if lb_topology == constants.TOPOLOGY_ACTIVE_STANDBY: + new_amp_role = constants.ROLE_BACKUP + + # Get a replacement amphora and plug all of the networking. + # + # Do this early as the compute services have been observed to be + # unreliable. The community decided the chance that deleting first + # would open resources for an instance is less likely than the compute + # service failing to boot an instance for other reasons. + if failed_amp: + failed_vrrp_is_ipv6 = False + if failed_amp.get(constants.VRRP_IP): + failed_vrrp_is_ipv6 = utils.is_ipv6( + failed_amp[constants.VRRP_IP]) + failover_LB_flow.add( + self.amp_flows.get_amphora_for_lb_failover_subflow( + prefix=constants.FAILOVER_LOADBALANCER_FLOW, + role=new_amp_role, + failed_amp_vrrp_port_id=failed_amp.get( + constants.VRRP_PORT_ID), + is_vrrp_ipv6=failed_vrrp_is_ipv6)) + else: + failover_LB_flow.add( + self.amp_flows.get_amphora_for_lb_failover_subflow( + prefix=constants.FAILOVER_LOADBALANCER_FLOW, + role=new_amp_role)) + + if lb_topology == constants.TOPOLOGY_ACTIVE_STANDBY: + failover_LB_flow.add(database_tasks.MarkAmphoraBackupInDB( + name=constants.MARK_AMP_BACKUP_INDB, + requires=constants.AMPHORA)) + + # Delete the failed amp + if failed_amp: + failover_LB_flow.add( + self.amp_flows.get_delete_amphora_flow(failed_amp)) + + # Update the data stored in the flow from the database + failover_LB_flow.add(database_tasks.ReloadLoadBalancer( + requires=constants.LOADBALANCER_ID, + provides=constants.LOADBALANCER)) + + # Configure the listener(s) + # We will run update on this amphora again later if this is + # an active/standby load balancer because we want this amp + # functional as soon as possible. It must run again to update + # the configurations for the new peers. + failover_LB_flow.add(amphora_driver_tasks.AmpListenersUpdate( + name=constants.AMP_LISTENER_UPDATE, + requires=(constants.LOADBALANCER, constants.AMPHORA), + inject={constants.TIMEOUT_DICT: timeout_dict})) + + # Bring up the new "backup" amphora VIP now to reduce the outage + # on the final failover. This dropped the outage from 8-9 seconds + # to less than one in my lab. + # This does mean some steps have to be repeated later to reconfigure + # for the second amphora as a peer. + if lb_topology == constants.TOPOLOGY_ACTIVE_STANDBY: + + failover_LB_flow.add(database_tasks.CreateVRRPGroupForLB( + name=new_amp_role + '-' + constants.CREATE_VRRP_GROUP_FOR_LB, + requires=constants.LOADBALANCER_ID)) + + failover_LB_flow.add(network_tasks.GetAmphoraNetworkConfigsByID( + name=(new_amp_role + '-' + + constants.GET_AMPHORA_NETWORK_CONFIGS_BY_ID), + requires=(constants.LOADBALANCER_ID, constants.AMPHORA_ID), + provides=constants.FIRST_AMP_NETWORK_CONFIGS)) + + failover_LB_flow.add( + amphora_driver_tasks.AmphoraUpdateVRRPInterface( + name=new_amp_role + '-' + constants.AMP_UPDATE_VRRP_INTF, + requires=constants.AMPHORA, + inject={constants.TIMEOUT_DICT: timeout_dict}, + provides=constants.FIRST_AMP_VRRP_INTERFACE)) + + failover_LB_flow.add(amphora_driver_tasks.AmphoraVRRPUpdate( + name=new_amp_role + '-' + constants.AMP_VRRP_UPDATE, + requires=(constants.LOADBALANCER_ID, constants.AMPHORA), + rebind={constants.AMPHORAE_NETWORK_CONFIG: + constants.FIRST_AMP_NETWORK_CONFIGS, + constants.AMP_VRRP_INT: + constants.FIRST_AMP_VRRP_INTERFACE}, + inject={constants.TIMEOUT_DICT: timeout_dict})) + + failover_LB_flow.add(amphora_driver_tasks.AmphoraVRRPStart( + name=new_amp_role + '-' + constants.AMP_VRRP_START, + requires=constants.AMPHORA, + inject={constants.TIMEOUT_DICT: timeout_dict})) + + # Start the listener. This needs to be done here because + # it will create the required haproxy check scripts for + # the VRRP deployed above. + # A "V" or newer amphora-agent will remove the need for this + # task here. + # TODO(johnsom) Remove this in the "X" cycle + failover_LB_flow.add(amphora_driver_tasks.ListenersStart( + name=new_amp_role + '-' + constants.AMP_LISTENER_START, + requires=(constants.LOADBALANCER, constants.AMPHORA))) + + # #### Work on standby amphora if needed ##### + + new_amp_role = constants.ROLE_MASTER + failed_amp = None + if amps: + failed_amp = amps.pop() + + if failed_amp: + failed_amp_role = failed_amp.get(constants.ROLE) + if failed_amp_role in (constants.ROLE_MASTER, + constants.ROLE_BACKUP): + amp_role = 'master_or_backup' + elif failed_amp_role == constants.ROLE_STANDALONE: + amp_role = 'standalone' + elif failed_amp_role is None: + amp_role = 'spare' + else: + amp_role = 'undefined' + LOG.info("Performing failover for amphora: %s", + {"id": failed_amp.get(constants.ID), + "load_balancer_id": lb.get(constants.ID), + "lb_network_ip": failed_amp.get( + constants.LB_NETWORK_IP), + "compute_id": failed_amp.get(constants.COMPUTE_ID), + "role": amp_role}) + + failover_LB_flow.add( + database_tasks.MarkAmphoraPendingDeleteInDB( + name=(new_amp_role + '-' + + constants.MARK_AMPHORA_PENDING_DELETE), + requires=constants.AMPHORA, + inject={constants.AMPHORA: failed_amp})) + + failover_LB_flow.add(database_tasks.MarkAmphoraHealthBusy( + name=(new_amp_role + '-' + + constants.MARK_AMPHORA_HEALTH_BUSY), + requires=constants.AMPHORA, + inject={constants.AMPHORA: failed_amp})) + + # Get a replacement amphora and plug all of the networking. + # + # Do this early as the compute services have been observed to be + # unreliable. The community decided the chance that deleting first + # would open resources for an instance is less likely than the + # compute service failing to boot an instance for other reasons. + failover_LB_flow.add( + self.amp_flows.get_amphora_for_lb_failover_subflow( + prefix=(new_amp_role + '-' + + constants.FAILOVER_LOADBALANCER_FLOW), + role=new_amp_role)) + + failover_LB_flow.add(database_tasks.MarkAmphoraMasterInDB( + name=constants.MARK_AMP_MASTER_INDB, + requires=constants.AMPHORA)) + + # Delete the failed amp + if failed_amp: + failover_LB_flow.add( + self.amp_flows.get_delete_amphora_flow( + failed_amp)) + failover_LB_flow.add( + database_tasks.DisableAmphoraHealthMonitoring( + name=(new_amp_role + '-' + + constants.DISABLE_AMP_HEALTH_MONITORING), + requires=constants.AMPHORA, + inject={constants.AMPHORA: failed_amp})) + + # Remove any extraneous amphora + # Note: This runs in all topology situations. + # It should run before the act/stdby final listener update so + # that we don't bother attempting to update dead amphorae. + delete_extra_amps_flow = unordered_flow.Flow( + constants.DELETE_EXTRA_AMPHORAE_FLOW) + for amp in amps: + LOG.debug('Found extraneous amphora %s on load balancer %s. ' + 'Deleting.', amp.get(constants.ID), lb.get(id)) + delete_extra_amps_flow.add( + self.amp_flows.get_delete_amphora_flow(amp)) + + failover_LB_flow.add(delete_extra_amps_flow) + + if lb_topology == constants.TOPOLOGY_ACTIVE_STANDBY: + # Update the data stored in the flow from the database + failover_LB_flow.add(database_tasks.ReloadLoadBalancer( + name=new_amp_role + '-' + constants.RELOAD_LB_AFTER_AMP_ASSOC, + requires=constants.LOADBALANCER_ID, + provides=constants.LOADBALANCER)) + + failover_LB_flow.add(database_tasks.GetAmphoraeFromLoadbalancer( + name=new_amp_role + '-' + constants.GET_AMPHORAE_FROM_LB, + requires=constants.LOADBALANCER_ID, + provides=constants.AMPHORAE)) + + # Listeners update needs to be run on all amphora to update + # their peer configurations. So parallelize this with an + # unordered subflow. + update_amps_subflow = unordered_flow.Flow( + constants.UPDATE_AMPS_SUBFLOW) + + # Setup parallel flows for each amp. We don't know the new amp + # details at flow creation time, so setup a subflow for each + # amp on the LB, they let the task index into a list of amps + # to find the amphora it should work on. + update_amps_subflow.add( + amphora_driver_tasks.AmphoraIndexListenerUpdate( + name=(constants.AMPHORA + '-0-' + + constants.AMP_LISTENER_UPDATE), + requires=(constants.LOADBALANCER, constants.AMPHORAE), + inject={constants.AMPHORA_INDEX: 0, + constants.TIMEOUT_DICT: timeout_dict})) + update_amps_subflow.add( + amphora_driver_tasks.AmphoraIndexListenerUpdate( + name=(constants.AMPHORA + '-1-' + + constants.AMP_LISTENER_UPDATE), + requires=(constants.LOADBALANCER, constants.AMPHORAE), + inject={constants.AMPHORA_INDEX: 1, + constants.TIMEOUT_DICT: timeout_dict})) + + failover_LB_flow.add(update_amps_subflow) + + # Configure and enable keepalived in the amphora + failover_LB_flow.add(self.amp_flows.get_vrrp_subflow( + new_amp_role + '-' + constants.GET_VRRP_SUBFLOW, + timeout_dict, create_vrrp_group=False)) + + # #### End of standby #### + + # Reload the listener. This needs to be done here because + # it will create the required haproxy check scripts for + # the VRRP deployed above. + # A "V" or newer amphora-agent will remove the need for this + # task here. + # TODO(johnsom) Remove this in the "X" cycle + failover_LB_flow.add( + amphora_driver_tasks.AmphoraIndexListenersReload( + name=(new_amp_role + '-' + + constants.AMPHORA_RELOAD_LISTENER), + requires=(constants.LOADBALANCER, constants.AMPHORAE), + inject={constants.AMPHORA_INDEX: 1, + constants.TIMEOUT_DICT: timeout_dict})) + + # Remove any extraneous ports + # Note: Nova sometimes fails to delete ports attached to an instance. + # For example, if you create an LB with a listener, then + # 'openstack server delete' the amphora, you will see the vrrp + # port attached to that instance will remain after the instance + # is deleted. + # TODO(johnsom) Fix this as part of + # https://storyboard.openstack.org/#!/story/2007077 + + # Mark LB ACTIVE + failover_LB_flow.add( + database_tasks.MarkLBActiveInDB(mark_subobjects=True, + requires=constants.LOADBALANCER)) + + return failover_LB_flow diff --git a/octavia/controller/worker/v2/flows/pool_flows.py b/octavia/controller/worker/v2/flows/pool_flows.py index b055cba74b..42d1c3595f 100644 --- a/octavia/controller/worker/v2/flows/pool_flows.py +++ b/octavia/controller/worker/v2/flows/pool_flows.py @@ -74,22 +74,23 @@ class PoolFlows(object): :returns: The flow for deleting a pool """ - delete_pool_flow = linear_flow.Flow(constants.DELETE_POOL_FLOW) + delete_pool_flow = linear_flow.Flow(constants.DELETE_POOL_FLOW + '-' + + pool_id) # health monitor should cascade # members should cascade delete_pool_flow.add(database_tasks.MarkPoolPendingDeleteInDB( name='mark_pool_pending_delete_in_db_' + pool_id, requires=constants.POOL_ID, - rebind={constants.POOL_ID: pool_id})) + inject={constants.POOL_ID: pool_id})) delete_pool_flow.add(database_tasks.CountPoolChildrenForQuota( name='count_pool_children_for_quota_' + pool_id, requires=constants.POOL_ID, provides=constants.POOL_CHILD_COUNT, - rebind={constants.POOL_ID: pool_id})) + inject={constants.POOL_ID: pool_id})) delete_pool_flow.add(database_tasks.DeletePoolInDB( name='delete_pool_in_db_' + pool_id, requires=constants.POOL_ID, - rebind={constants.POOL_ID: pool_id})) + inject={constants.POOL_ID: pool_id})) delete_pool_flow.add(database_tasks.DecrementPoolQuota( name='decrement_pool_quota_' + pool_id, requires=[constants.PROJECT_ID, constants.POOL_CHILD_COUNT])) diff --git a/octavia/controller/worker/v2/tasks/amphora_driver_tasks.py b/octavia/controller/worker/v2/tasks/amphora_driver_tasks.py index daf070e5a6..0402ecf04f 100644 --- a/octavia/controller/worker/v2/tasks/amphora_driver_tasks.py +++ b/octavia/controller/worker/v2/tasks/amphora_driver_tasks.py @@ -23,7 +23,6 @@ from taskflow.types import failure from octavia.amphorae.backends.agent import agent_jinja_cfg from octavia.amphorae.driver_exceptions import exceptions as driver_except -from octavia.api.drivers import utils as provider_utils from octavia.common import constants from octavia.common import utils from octavia.controller.worker import task_utils as task_utilities @@ -74,21 +73,46 @@ class AmpRetry(retry.Times): class AmpListenersUpdate(BaseAmphoraTask): """Task to update the listeners on one amphora.""" + def execute(self, loadbalancer, amphora, timeout_dict=None): + # Note, we don't want this to cause a revert as it may be used + # in a failover flow with both amps failing. Skip it and let + # health manager fix it. + # TODO(johnsom) Optimize this to use the dicts and not need the + # DB lookups + db_amp = self.amphora_repo.get(db_apis.get_session(), + id=amphora[constants.ID]) + try: + db_lb = self.loadbalancer_repo.get( + db_apis.get_session(), + id=loadbalancer[constants.LOADBALANCER_ID]) + self.amphora_driver.update_amphora_listeners( + db_lb, db_amp, timeout_dict) + except Exception as e: + LOG.error('Failed to update listeners on amphora %s. Skipping ' + 'this amphora as it is failing to update due to: %s', + db_amp.id, str(e)) + self.amphora_repo.update(db_apis.get_session(), db_amp.id, + status=constants.ERROR) + + +class AmphoraIndexListenerUpdate(BaseAmphoraTask): + """Task to update the listeners on one amphora.""" + def execute(self, loadbalancer, amphora_index, amphorae, timeout_dict=()): # Note, we don't want this to cause a revert as it may be used # in a failover flow with both amps failing. Skip it and let # health manager fix it. try: - db_amphorae = [] - for amp in amphorae: - db_amp = self.amphora_repo.get(db_apis.get_session(), - id=amp[constants.ID]) - db_amphorae.append(db_amp) + # TODO(johnsom) Optimize this to use the dicts and not need the + # DB lookups + db_amp = self.amphora_repo.get( + db_apis.get_session(), + id=amphorae[amphora_index][constants.ID]) db_lb = self.loadbalancer_repo.get( db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID]) self.amphora_driver.update_amphora_listeners( - db_lb, db_amphorae[amphora_index], timeout_dict) + db_lb, db_amp, timeout_dict) except Exception as e: amphora_id = amphorae[amphora_index].get(constants.ID) LOG.error('Failed to update listeners on amphora %s. Skipping ' @@ -148,6 +172,35 @@ class ListenersStart(BaseAmphoraTask): self.task_utils.mark_listener_prov_status_error(listener.id) +class AmphoraIndexListenersReload(BaseAmphoraTask): + """Task to reload all listeners on an amphora.""" + + def execute(self, loadbalancer, amphora_index, amphorae, + timeout_dict=None): + """Execute listener reload routines for listeners on an amphora.""" + if amphorae is None: + return + # TODO(johnsom) Optimize this to use the dicts and not need the + # DB lookups + db_amp = self.amphora_repo.get( + db_apis.get_session(), id=amphorae[amphora_index][constants.ID]) + db_lb = self.loadbalancer_repo.get( + db_apis.get_session(), + id=loadbalancer[constants.LOADBALANCER_ID]) + if db_lb.listeners: + self.amphora_driver.reload(db_lb, db_amp, timeout_dict) + + def revert(self, loadbalancer, *args, **kwargs): + """Handle failed listeners reloads.""" + + LOG.warning("Reverting listener reload.") + db_lb = self.loadbalancer_repo.get( + db_apis.get_session(), + id=loadbalancer[constants.LOADBALANCER_ID]) + for listener in db_lb.listeners: + self.task_utils.mark_listener_prov_status_error(listener.id) + + class ListenerDelete(BaseAmphoraTask): """Task to delete the listener on the vip.""" @@ -332,96 +385,149 @@ class AmphoraCertUpload(BaseAmphoraTask): db_amp, fer.decrypt(server_pem.encode('utf-8'))) +# TODO(johnsom) REMOVE ME! class AmphoraUpdateVRRPInterface(BaseAmphoraTask): """Task to get and update the VRRP interface device name from amphora.""" - def execute(self, loadbalancer): - """Execute post_vip_routine.""" - amps = [] - db_lb = self.loadbalancer_repo.get( - db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID]) - timeout_dict = { - constants.CONN_MAX_RETRIES: - CONF.haproxy_amphora.active_connection_max_retries, - constants.CONN_RETRY_INTERVAL: - CONF.haproxy_amphora.active_connection_rety_interval} - for amp in filter( - lambda amp: amp.status == constants.AMPHORA_ALLOCATED, - db_lb.amphorae): + def execute(self, amphora, timeout_dict=None): + try: + # TODO(johnsom) Optimize this to use the dicts and not need the + # DB lookups + db_amp = self.amphora_repo.get(db_apis.get_session(), + id=amphora[constants.ID]) + interface = self.amphora_driver.get_interface_from_ip( + db_amp, db_amp.vrrp_ip, timeout_dict=timeout_dict) + except Exception as e: + # This can occur when an active/standby LB has no listener + LOG.error('Failed to get amphora VRRP interface on amphora ' + '%s. Skipping this amphora as it is failing due to: ' + '%s', amphora.get(constants.ID), str(e)) + self.amphora_repo.update(db_apis.get_session(), + amphora.get(constants.ID), + status=constants.ERROR) + return None - try: - interface = self.amphora_driver.get_interface_from_ip( - amp, amp.vrrp_ip, timeout_dict=timeout_dict) - except Exception as e: - # This can occur when an active/standby LB has no listener - LOG.error('Failed to get amphora VRRP interface on amphora ' - '%s. Skipping this amphora as it is failing due to: ' - '%s', amp.id, str(e)) - self.amphora_repo.update(db_apis.get_session(), amp.id, - status=constants.ERROR) - continue + self.amphora_repo.update(db_apis.get_session(), amphora[constants.ID], + vrrp_interface=interface) + return interface - self.amphora_repo.update(db_apis.get_session(), amp.id, - vrrp_interface=interface) - amps.append(self.amphora_repo.get(db_apis.get_session(), - id=amp.id)) - db_lb.amphorae = amps - return provider_utils.db_loadbalancer_to_provider_loadbalancer( - db_lb).to_dict() - def revert(self, result, loadbalancer, *args, **kwargs): - """Handle a failed amphora vip plug notification.""" - if isinstance(result, failure.Failure): - return - LOG.warning("Reverting Get Amphora VRRP Interface.") - db_lb = self.loadbalancer_repo.get( - db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID]) - for amp in filter( - lambda amp: amp.status == constants.AMPHORA_ALLOCATED, - db_lb.amphorae): +class AmphoraIndexUpdateVRRPInterface(BaseAmphoraTask): + """Task to get and update the VRRP interface device name from amphora.""" - try: - self.amphora_repo.update(db_apis.get_session(), amp.id, - vrrp_interface=None) - except Exception as e: - LOG.error("Failed to update amphora %(amp)s " - "VRRP interface to None due to: %(except)s", - {'amp': amp.id, 'except': e}) + def execute(self, amphora_index, amphorae, timeout_dict=None): + amphora_id = amphorae[amphora_index][constants.ID] + try: + # TODO(johnsom) Optimize this to use the dicts and not need the + # DB lookups + db_amp = self.amphora_repo.get(db_apis.get_session(), + id=amphora_id) + interface = self.amphora_driver.get_interface_from_ip( + db_amp, db_amp.vrrp_ip, timeout_dict=timeout_dict) + except Exception as e: + # This can occur when an active/standby LB has no listener + LOG.error('Failed to get amphora VRRP interface on amphora ' + '%s. Skipping this amphora as it is failing due to: ' + '%s', amphora_id, str(e)) + self.amphora_repo.update(db_apis.get_session(), amphora_id, + status=constants.ERROR) + return None + + self.amphora_repo.update(db_apis.get_session(), amphora_id, + vrrp_interface=interface) + return interface class AmphoraVRRPUpdate(BaseAmphoraTask): - """Task to update the VRRP configuration of the loadbalancer amphorae.""" + """Task to update the VRRP configuration of an amphora.""" - def execute(self, loadbalancer, amphorae_network_config): + def execute(self, loadbalancer_id, amphorae_network_config, amphora, + amp_vrrp_int, timeout_dict=None): """Execute update_vrrp_conf.""" - db_lb = self.loadbalancer_repo.get( - db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID]) - self.amphora_driver.update_vrrp_conf(db_lb, - amphorae_network_config) - LOG.debug("Uploaded VRRP configuration of loadbalancer %s amphorae", - loadbalancer[constants.LOADBALANCER_ID]) + # Note, we don't want this to cause a revert as it may be used + # in a failover flow with both amps failing. Skip it and let + # health manager fix it. + amphora_id = amphora[constants.ID] + try: + # TODO(johnsom) Optimize this to use the dicts and not need the + # DB lookups + db_amp = self.amphora_repo.get(db_apis.get_session(), + id=amphora_id) + loadbalancer = self.loadbalancer_repo.get(db_apis.get_session(), + id=loadbalancer_id) + db_amp.vrrp_interface = amp_vrrp_int + self.amphora_driver.update_vrrp_conf( + loadbalancer, amphorae_network_config, db_amp, timeout_dict) + except Exception as e: + LOG.error('Failed to update VRRP configuration amphora %s. ' + 'Skipping this amphora as it is failing to update due ' + 'to: %s', amphora_id, str(e)) + self.amphora_repo.update(db_apis.get_session(), amphora_id, + status=constants.ERROR) + + LOG.debug("Uploaded VRRP configuration of amphora %s.", amphora_id) -class AmphoraVRRPStop(BaseAmphoraTask): - """Task to stop keepalived of all amphorae of a LB.""" +class AmphoraIndexVRRPUpdate(BaseAmphoraTask): + """Task to update the VRRP configuration of an amphora.""" - def execute(self, loadbalancer): - db_lb = self.loadbalancer_repo.get( - db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID]) - self.amphora_driver.stop_vrrp_service(db_lb) - LOG.debug("Stopped VRRP of loadbalancer %s amphorae", - loadbalancer[constants.LOADBALANCER_ID]) + def execute(self, loadbalancer_id, amphorae_network_config, amphora_index, + amphorae, amp_vrrp_int, timeout_dict=None): + """Execute update_vrrp_conf.""" + # Note, we don't want this to cause a revert as it may be used + # in a failover flow with both amps failing. Skip it and let + # health manager fix it. + amphora_id = amphorae[amphora_index][constants.ID] + try: + # TODO(johnsom) Optimize this to use the dicts and not need the + # DB lookups + db_amp = self.amphora_repo.get(db_apis.get_session(), + id=amphora_id) + loadbalancer = self.loadbalancer_repo.get(db_apis.get_session(), + id=loadbalancer_id) + db_amp.vrrp_interface = amp_vrrp_int + self.amphora_driver.update_vrrp_conf( + loadbalancer, amphorae_network_config, db_amp, timeout_dict) + except Exception as e: + LOG.error('Failed to update VRRP configuration amphora %s. ' + 'Skipping this amphora as it is failing to update due ' + 'to: %s', amphora_id, str(e)) + self.amphora_repo.update(db_apis.get_session(), amphora_id, + status=constants.ERROR) + + LOG.debug("Uploaded VRRP configuration of amphora %s.", amphora_id) class AmphoraVRRPStart(BaseAmphoraTask): - """Task to start keepalived of all amphorae of a LB.""" + """Task to start keepalived on an amphora. - def execute(self, loadbalancer): - db_lb = self.loadbalancer_repo.get( - db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID]) - self.amphora_driver.start_vrrp_service(db_lb) - LOG.debug("Started VRRP of loadbalancer %s amphorae", - loadbalancer[constants.LOADBALANCER_ID]) + This will reload keepalived if it is already running. + """ + + def execute(self, amphora, timeout_dict=None): + # TODO(johnsom) Optimize this to use the dicts and not need the + # DB lookups + db_amp = self.amphora_repo.get( + db_apis.get_session(), id=amphora[constants.ID]) + self.amphora_driver.start_vrrp_service(db_amp, timeout_dict) + LOG.debug("Started VRRP on amphora %s.", amphora[constants.ID]) + + +class AmphoraIndexVRRPStart(BaseAmphoraTask): + """Task to start keepalived on an amphora. + + This will reload keepalived if it is already running. + """ + + def execute(self, amphora_index, amphorae, timeout_dict=None): + # TODO(johnsom) Optimize this to use the dicts and not need the + # DB lookups + db_amp = self.amphora_repo.get( + db_apis.get_session(), id=amphorae[amphora_index][constants.ID]) + + self.amphora_driver.start_vrrp_service(db_amp, timeout_dict) + LOG.debug("Started VRRP on amphora %s.", + amphorae[amphora_index][constants.ID]) class AmphoraComputeConnectivityWait(BaseAmphoraTask): diff --git a/octavia/controller/worker/v2/tasks/compute_tasks.py b/octavia/controller/worker/v2/tasks/compute_tasks.py index faac953b74..6b76588e54 100644 --- a/octavia/controller/worker/v2/tasks/compute_tasks.py +++ b/octavia/controller/worker/v2/tasks/compute_tasks.py @@ -21,6 +21,7 @@ from oslo_log import log as logging from stevedore import driver as stevedore_driver from taskflow import task from taskflow.types import failure +import tenacity from octavia.amphorae.backends.agent import agent_jinja_cfg from octavia.common import constants @@ -53,10 +54,9 @@ class BaseComputeTask(task.Task): class ComputeCreate(BaseComputeTask): """Create the compute instance for a new amphora.""" - def execute(self, amphora_id, config_drive_files=None, + def execute(self, amphora_id, server_group_id, config_drive_files=None, build_type_priority=constants.LB_CREATE_NORMAL_PRIORITY, - server_group_id=None, ports=None, flavor=None, - availability_zone=None): + ports=None, flavor=None, availability_zone=None): """Create an amphora :param availability_zone: availability zone metadata dictionary @@ -151,10 +151,9 @@ class ComputeCreate(BaseComputeTask): class CertComputeCreate(ComputeCreate): - def execute(self, amphora_id, server_pem, + def execute(self, amphora_id, server_pem, server_group_id, build_type_priority=constants.LB_CREATE_NORMAL_PRIORITY, - server_group_id=None, ports=None, flavor=None, - availability_zone=None): + ports=None, flavor=None, availability_zone=None): """Create an amphora :param availability_zone: availability zone metadata dictionary @@ -199,16 +198,53 @@ class DeleteAmphoraeOnLoadBalancer(BaseComputeTask): class ComputeDelete(BaseComputeTask): - def execute(self, amphora): - LOG.debug("Compute Delete execute for amphora with id %s", - amphora.get(constants.ID)) + @tenacity.retry(retry=tenacity.retry_if_exception_type(), + stop=tenacity.stop_after_attempt(CONF.compute.max_retries), + wait=tenacity.wait_exponential( + multiplier=CONF.compute.retry_backoff, + min=CONF.compute.retry_interval, + max=CONF.compute.retry_max), reraise=True) + def execute(self, amphora, passive_failure=False): + + amphora_id = amphora.get(constants.ID) + compute_id = amphora[constants.COMPUTE_ID] + + if self.execute.retry.statistics.get(constants.ATTEMPT_NUMBER, 1) == 1: + LOG.debug('Compute delete execute for amphora with ID %s and ' + 'compute ID: %s', amphora_id, compute_id) + else: + LOG.warning('Retrying compute delete of %s attempt %s of %s.', + compute_id, + self.execute.retry.statistics[ + constants.ATTEMPT_NUMBER], + self.execute.retry.stop.max_attempt_number) + # Let the Taskflow engine know we are working and alive + # Don't use get with a default for 'attempt_number', we need to fail + # if that number is missing. + self.update_progress( + self.execute.retry.statistics[constants.ATTEMPT_NUMBER] / + self.execute.retry.stop.max_attempt_number) try: - self.compute.delete(amphora[constants.COMPUTE_ID]) + self.compute.delete(compute_id) except Exception: - LOG.exception("Compute delete for amphora id: %s failed", - amphora.get(constants.ID)) - raise + if (self.execute.retry.statistics[constants.ATTEMPT_NUMBER] != + self.execute.retry.stop.max_attempt_number): + LOG.warning('Compute delete for amphora id: %s failed. ' + 'Retrying.', amphora_id) + raise + if passive_failure: + LOG.exception('Compute delete for compute ID: %s on amphora ' + 'ID: %s failed. This resource will be abandoned ' + 'and should manually be cleaned up once the ' + 'compute service is functional.', + compute_id, amphora_id) + else: + LOG.exception('Compute delete for compute ID: %s on amphora ' + 'ID: %s failed. The compute service has failed. ' + 'Aborting and reverting.', compute_id, + amphora_id) + raise class ComputeActiveWait(BaseComputeTask): @@ -281,3 +317,33 @@ class NovaServerGroupDelete(BaseComputeTask): self.compute.delete_server_group(server_group_id) else: return + + +class AttachPort(BaseComputeTask): + def execute(self, amphora, port): + """Attach a port to an amphora instance. + + :param amphora: The amphora to attach the port to. + :param port: The port to attach to the amphora. + :returns: None + """ + LOG.debug('Attaching port: %s to compute: %s', + port[constants.ID], amphora[constants.COMPUTE_ID]) + self.compute.attach_network_or_port(amphora[constants.COMPUTE_ID], + port_id=port[constants.ID]) + + def revert(self, amphora, port, *args, **kwargs): + """Revert our port attach. + + :param amphora: The amphora to detach the port from. + :param port: The port to attach to the amphora. + """ + LOG.warning('Reverting port: %s attach to compute: %s', + port[constants.ID], amphora[constants.COMPUTE_ID]) + try: + self.compute.detach_port(amphora[constants.COMPUTE_ID], + port[constants.ID]) + except Exception as e: + LOG.error('Failed to detach port %s from compute %s for revert ' + 'due to %s.', port[constants.ID], + amphora[constants.COMPUTE_ID], str(e)) diff --git a/octavia/controller/worker/v2/tasks/database_tasks.py b/octavia/controller/worker/v2/tasks/database_tasks.py index f2422ea032..76702699fc 100644 --- a/octavia/controller/worker/v2/tasks/database_tasks.py +++ b/octavia/controller/worker/v2/tasks/database_tasks.py @@ -280,6 +280,7 @@ class DeleteListenerInDB(BaseDatabaseTask): :returns: None """ + # TODO(johnsom) Fix this, it doesn't revert anything LOG.warning("Reverting mark listener delete in DB for listener id %s", listener[constants.LISTENER_ID]) @@ -490,22 +491,26 @@ class UpdateAmphoraVIPData(BaseDatabaseTask): class UpdateAmpFailoverDetails(BaseDatabaseTask): """Update amphora failover details in the database.""" - def execute(self, amphora, amp_data): + def execute(self, amphora, vip, base_port): """Update amphora failover details in the database. :param amphora: The amphora to update - :param amp_data: data_models.Amphora object with update data + :param vip: The VIP object associated with this amphora. + :param base_port: The base port object associated with the amphora. :returns: None """ # role and vrrp_priority will be updated later. self.repos.amphora.update( db_apis.get_session(), amphora.get(constants.ID), - vrrp_ip=amp_data[constants.VRRP_IP], - ha_ip=amp_data[constants.HA_IP], - vrrp_port_id=amp_data[constants.VRRP_PORT_ID], - ha_port_id=amp_data[constants.HA_PORT_ID], - vrrp_id=amp_data[constants.VRRP_ID]) + # TODO(johnsom) We should do a better job getting the fixed_ip + # as this could be a problem with dual stack. + # Fix this during the multi-vip patch. + vrrp_ip=base_port[constants.FIXED_IPS][0][constants.IP_ADDRESS], + ha_ip=vip[constants.IP_ADDRESS], + vrrp_port_id=base_port[constants.ID], + ha_port_id=vip[constants.PORT_ID], + vrrp_id=1) class AssociateFailoverAmphoraWithLBID(BaseDatabaseTask): @@ -595,27 +600,25 @@ class MapLoadbalancerToAmphora(BaseDatabaseTask): class _MarkAmphoraRoleAndPriorityInDB(BaseDatabaseTask): """Alter the amphora role and priority in DB.""" - def _execute(self, amphora, amp_role, vrrp_priority): + def _execute(self, amphora_id, amp_role, vrrp_priority): """Alter the amphora role and priority in DB. - :param amphora: Amphora to update. + :param amphora_id: Amphora ID to update. :param amp_role: Amphora role to be set. :param vrrp_priority: VRRP priority to set. :returns: None """ LOG.debug("Mark %(role)s in DB for amphora: %(amp)s", - {constants.ROLE: amp_role, 'amp': amphora[constants.ID]}) - self.amphora_repo.update(db_apis.get_session(), - amphora[constants.ID], - role=amp_role, - vrrp_priority=vrrp_priority) + {constants.ROLE: amp_role, 'amp': amphora_id}) + self.amphora_repo.update(db_apis.get_session(), amphora_id, + role=amp_role, vrrp_priority=vrrp_priority) - def _revert(self, result, amphora, *args, **kwargs): + def _revert(self, result, amphora_id, *args, **kwargs): """Removes role and vrrp_priority association. :param result: Result of the association. - :param amphora: Amphora which role/vrrp_priority association - failed. + :param amphora_id: Amphora ID which role/vrrp_priority association + failed. :returns: None """ @@ -623,17 +626,14 @@ class _MarkAmphoraRoleAndPriorityInDB(BaseDatabaseTask): return LOG.warning("Reverting amphora role in DB for amp id %(amp)s", - {'amp': amphora[constants.ID]}) + {'amp': amphora_id}) try: - self.amphora_repo.update(db_apis.get_session(), - amphora[constants.ID], - role=None, - vrrp_priority=None) + self.amphora_repo.update(db_apis.get_session(), amphora_id, + role=None, vrrp_priority=None) except Exception as e: LOG.error("Failed to update amphora %(amp)s " "role and vrrp_priority to None due to: " - "%(except)s", {'amp': amphora[constants.ID], - 'except': e}) + "%(except)s", {'amp': amphora_id, 'except': e}) class MarkAmphoraMasterInDB(_MarkAmphoraRoleAndPriorityInDB): @@ -646,7 +646,8 @@ class MarkAmphoraMasterInDB(_MarkAmphoraRoleAndPriorityInDB): :returns: None """ amp_role = constants.ROLE_MASTER - self._execute(amphora, amp_role, constants.ROLE_MASTER_PRIORITY) + self._execute(amphora[constants.ID], amp_role, + constants.ROLE_MASTER_PRIORITY) def revert(self, result, amphora, *args, **kwargs): """Removes amphora role association. @@ -654,7 +655,7 @@ class MarkAmphoraMasterInDB(_MarkAmphoraRoleAndPriorityInDB): :param amphora: Amphora to update role. :returns: None """ - self._revert(result, amphora, *args, **kwargs) + self._revert(result, amphora[constants.ID], *args, **kwargs) class MarkAmphoraBackupInDB(_MarkAmphoraRoleAndPriorityInDB): @@ -667,7 +668,8 @@ class MarkAmphoraBackupInDB(_MarkAmphoraRoleAndPriorityInDB): :returns: None """ amp_role = constants.ROLE_BACKUP - self._execute(amphora, amp_role, constants.ROLE_BACKUP_PRIORITY) + self._execute(amphora[constants.ID], amp_role, + constants.ROLE_BACKUP_PRIORITY) def revert(self, result, amphora, *args, **kwargs): """Removes amphora role association. @@ -675,7 +677,7 @@ class MarkAmphoraBackupInDB(_MarkAmphoraRoleAndPriorityInDB): :param amphora: Amphora to update role. :returns: None """ - self._revert(result, amphora, *args, **kwargs) + self._revert(result, amphora[constants.ID], *args, **kwargs) class MarkAmphoraStandAloneInDB(_MarkAmphoraRoleAndPriorityInDB): @@ -688,7 +690,7 @@ class MarkAmphoraStandAloneInDB(_MarkAmphoraRoleAndPriorityInDB): :returns: None """ amp_role = constants.ROLE_STANDALONE - self._execute(amphora, amp_role, None) + self._execute(amphora[constants.ID], amp_role, None) def revert(self, result, amphora, *args, **kwargs): """Removes amphora role association. @@ -696,7 +698,7 @@ class MarkAmphoraStandAloneInDB(_MarkAmphoraRoleAndPriorityInDB): :param amphora: Amphora to update role. :returns: None """ - self._revert(result, amphora, *args, **kwargs) + self._revert(result, amphora[constants.ID], *args, **kwargs) class MarkAmphoraAllocatedInDB(BaseDatabaseTask): @@ -809,10 +811,10 @@ class MarkAmphoraDeletedInDB(BaseDatabaseTask): LOG.debug("Mark DELETED in DB for amphora: %(amp)s with " "compute id %(comp)s", - {'amp': amphora.get(constants.ID), + {'amp': amphora[constants.ID], 'comp': amphora[constants.COMPUTE_ID]}) self.amphora_repo.update(db_apis.get_session(), - amphora.get(constants.ID), + amphora[constants.ID], status=constants.DELETED) def revert(self, amphora, *args, **kwargs): @@ -824,10 +826,10 @@ class MarkAmphoraDeletedInDB(BaseDatabaseTask): LOG.warning("Reverting mark amphora deleted in DB " "for amp id %(amp)s and compute id %(comp)s", - {'amp': amphora.get(constants.ID), + {'amp': amphora[constants.ID], 'comp': amphora[constants.COMPUTE_ID]}) - self.task_utils.mark_amphora_status_error(amphora.get(constants.ID)) + self.task_utils.mark_amphora_status_error(amphora[constants.ID]) class MarkAmphoraPendingDeleteInDB(BaseDatabaseTask): @@ -845,10 +847,10 @@ class MarkAmphoraPendingDeleteInDB(BaseDatabaseTask): LOG.debug("Mark PENDING DELETE in DB for amphora: %(amp)s " "with compute id %(id)s", - {'amp': amphora.get(constants.ID), + {'amp': amphora[constants.ID], 'id': amphora[constants.COMPUTE_ID]}) self.amphora_repo.update(db_apis.get_session(), - amphora.get(constants.ID), + amphora[constants.ID], status=constants.PENDING_DELETE) def revert(self, amphora, *args, **kwargs): @@ -860,9 +862,9 @@ class MarkAmphoraPendingDeleteInDB(BaseDatabaseTask): LOG.warning("Reverting mark amphora pending delete in DB " "for amp id %(amp)s and compute id %(comp)s", - {'amp': amphora.get(constants.ID), + {'amp': amphora[constants.ID], 'comp': amphora[constants.COMPUTE_ID]}) - self.task_utils.mark_amphora_status_error(amphora.get(constants.ID)) + self.task_utils.mark_amphora_status_error(amphora[constants.ID]) class MarkAmphoraPendingUpdateInDB(BaseDatabaseTask): @@ -1111,7 +1113,7 @@ class MarkLBActiveInDB(BaseDatabaseTask): def _mark_member_status(self, member, status): self.member_repo.update( - db_apis.get_session(), member[constants.MEMBER_ID], + db_apis.get_session(), member.id, provisioning_status=status) def revert(self, loadbalancer, *args, **kwargs): @@ -1691,17 +1693,17 @@ class GetAmphoraDetails(BaseDatabaseTask): class GetAmphoraeFromLoadbalancer(BaseDatabaseTask): - """Task to pull the listeners from a loadbalancer.""" + """Task to pull the amphorae from a loadbalancer.""" - def execute(self, loadbalancer): + def execute(self, loadbalancer_id): """Pull the amphorae from a loadbalancer. - :param loadbalancer: Load balancer which listeners are required + :param loadbalancer_id: Load balancer ID to get amphorae from :returns: A list of Listener objects """ amphorae = [] - db_lb = self.repos.load_balancer.get( - db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID]) + db_lb = self.repos.load_balancer.get(db_apis.get_session(), + id=loadbalancer_id) for amp in db_lb.amphorae: a = self.amphora_repo.get(db_apis.get_session(), id=amp.id, show_deleted=False) @@ -1746,29 +1748,45 @@ class GetVipFromLoadbalancer(BaseDatabaseTask): return db_lb.vip.to_dict(recurse=True) +class GetLoadBalancer(BaseDatabaseTask): + """Get an load balancer object from the database.""" + + def execute(self, loadbalancer_id, *args, **kwargs): + """Get an load balancer object from the database. + + :param loadbalancer_id: The load balancer ID to lookup + :returns: The load balancer object + """ + + LOG.debug("Get load balancer from DB for load balancer id: %s", + loadbalancer_id) + db_lb = self.loadbalancer_repo.get(db_apis.get_session(), + id=loadbalancer_id) + provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( + db_lb) + return provider_lb.to_dict() + + class CreateVRRPGroupForLB(BaseDatabaseTask): """Create a VRRP group for a load balancer.""" - def execute(self, loadbalancer): + def execute(self, loadbalancer_id): """Create a VRRP group for a load balancer. - :param loadbalancer: Load balancer for which a VRRP group + :param loadbalancer_id: Load balancer ID for which a VRRP group should be created - :returns: Updated load balancer """ try: self.repos.vrrpgroup.create( db_apis.get_session(), - load_balancer_id=loadbalancer[constants.LOADBALANCER_ID], - vrrp_group_name=str( - loadbalancer[constants.LOADBALANCER_ID]).replace('-', ''), + load_balancer_id=loadbalancer_id, + vrrp_group_name=str(loadbalancer_id).replace('-', ''), vrrp_auth_type=constants.VRRP_AUTH_DEFAULT, vrrp_auth_pass=uuidutils.generate_uuid().replace('-', '')[0:7], advert_int=CONF.keepalived_vrrp.vrrp_advert_int) except odb_exceptions.DBDuplicateEntry: LOG.debug('VRRP_GROUP entry already exists for load balancer, ' 'skipping create.') - return loadbalancer class DisableAmphoraHealthMonitoring(BaseDatabaseTask): @@ -1784,7 +1802,7 @@ class DisableAmphoraHealthMonitoring(BaseDatabaseTask): :param amphora: The amphora to disable health monitoring for :returns: None """ - self._delete_from_amp_health(amphora.get(constants.ID)) + self._delete_from_amp_health(amphora[constants.ID]) class DisableLBAmphoraeHealthMonitoring(BaseDatabaseTask): @@ -1819,7 +1837,7 @@ class MarkAmphoraHealthBusy(BaseDatabaseTask): :param amphora: The amphora to mark amphora health busy :returns: None """ - self._mark_amp_health_busy(amphora.get(constants.ID)) + self._mark_amp_health_busy(amphora[constants.ID]) class MarkLBAmphoraeHealthBusy(BaseDatabaseTask): diff --git a/octavia/controller/worker/v2/tasks/network_tasks.py b/octavia/controller/worker/v2/tasks/network_tasks.py index 80eb43b4d2..a5bd3cea89 100644 --- a/octavia/controller/worker/v2/tasks/network_tasks.py +++ b/octavia/controller/worker/v2/tasks/network_tasks.py @@ -12,11 +12,14 @@ # License for the specific language governing permissions and limitations # under the License. # +import time from oslo_config import cfg from oslo_log import log as logging +from oslo_utils import excutils from taskflow import task from taskflow.types import failure +import tenacity from octavia.common import constants from octavia.common import data_models @@ -52,21 +55,27 @@ class CalculateAmphoraDelta(BaseNetworkTask): default_provides = constants.DELTA - def execute(self, loadbalancer, amphora, availability_zone): + def execute(self, loadbalancer, amphora, availability_zone, + vrrp_port=None): LOG.debug("Calculating network delta for amphora id: %s", amphora.get(constants.ID)) + if vrrp_port is None: + vrrp_port = self.network_driver.get_port( + amphora[constants.VRRP_PORT_ID]) + vrrp_port_network_id = vrrp_port.network_id + else: + vrrp_port_network_id = vrrp_port[constants.NETWORK_ID] + # Figure out what networks we want # seed with lb network(s) - vrrp_port = self.network_driver.get_port( - amphora[constants.VRRP_PORT_ID]) if availability_zone: management_nets = ( [availability_zone.get(constants.MANAGEMENT_NETWORK)] or CONF.controller_worker.amp_boot_network_list) else: management_nets = CONF.controller_worker.amp_boot_network_list - desired_network_ids = {vrrp_port.network_id}.union(management_nets) + desired_network_ids = {vrrp_port_network_id}.union(management_nets) db_lb = self.loadbalancer_repo.get( db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID]) for pool in db_lb.pools: @@ -84,7 +93,7 @@ class CalculateAmphoraDelta(BaseNetworkTask): del_ids = set(actual_network_nics) - desired_network_ids delete_nics = list( - actual_network_nics[net_id] for net_id in del_ids) + n_data_models.Interface(network_id=net_id) for net_id in del_ids) add_ids = desired_network_ids - set(actual_network_nics) add_nics = list(n_data_models.Interface( @@ -353,7 +362,8 @@ class PlugVIP(BaseNetworkTask): LOG.debug("Plumbing VIP for loadbalancer id: %s", loadbalancer[constants.LOADBALANCER_ID]) db_lb = self.loadbalancer_repo.get( - db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID]) + db_apis.get_session(), + id=loadbalancer[constants.LOADBALANCER_ID]) amps_data = self.network_driver.plug_vip(db_lb, db_lb.vip) return [amp.to_dict() for amp in amps_data] @@ -367,7 +377,8 @@ class PlugVIP(BaseNetworkTask): loadbalancer[constants.LOADBALANCER_ID]) db_lb = self.loadbalancer_repo.get( - db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID]) + db_apis.get_session(), + id=loadbalancer[constants.LOADBALANCER_ID]) try: # Make sure we have the current port IDs for cleanup for amp_data in result: @@ -388,13 +399,12 @@ class PlugVIP(BaseNetworkTask): class UpdateVIPSecurityGroup(BaseNetworkTask): """Task to setup SG for LB.""" - def execute(self, loadbalancer): + def execute(self, loadbalancer_id): """Task to setup SG for LB.""" - LOG.debug("Setup SG for loadbalancer id: %s", - loadbalancer[constants.LOADBALANCER_ID]) + LOG.debug("Setup SG for loadbalancer id: %s", loadbalancer_id) db_lb = self.loadbalancer_repo.get( - db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID]) + db_apis.get_session(), id=loadbalancer_id) self.network_driver.update_vip_sg(db_lb, db_lb.vip) @@ -411,7 +421,7 @@ class GetSubnetFromVIP(BaseNetworkTask): loadbalancer['vip_subnet_id']).to_dict() -class PlugVIPAmpphora(BaseNetworkTask): +class PlugVIPAmphora(BaseNetworkTask): """Task to plumb a VIP.""" def execute(self, loadbalancer, amphora, subnet): @@ -561,13 +571,29 @@ class GetAmphoraNetworkConfigs(BaseNetworkTask): return provider_dict +class GetAmphoraNetworkConfigsByID(BaseNetworkTask): + """Task to retrieve amphora network details.""" + + def execute(self, loadbalancer_id, amphora_id=None): + LOG.debug("Retrieving vip network details.") + loadbalancer = self.loadbalancer_repo.get(db_apis.get_session(), + id=loadbalancer_id) + amphora = self.amphora_repo.get(db_apis.get_session(), id=amphora_id) + db_configs = self.network_driver.get_network_configs(loadbalancer, + amphora=amphora) + provider_dict = {} + for amp_id, amp_conf in db_configs.items(): + provider_dict[amp_id] = amp_conf.to_dict(recurse=True) + return provider_dict + + class GetAmphoraeNetworkConfigs(BaseNetworkTask): """Task to retrieve amphorae network details.""" - def execute(self, loadbalancer): + def execute(self, loadbalancer_id): LOG.debug("Retrieving vip network details.") db_lb = self.loadbalancer_repo.get( - db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID]) + db_apis.get_session(), id=loadbalancer_id) db_configs = self.network_driver.get_network_configs(db_lb) provider_dict = {} for amp_id, amp_conf in db_configs.items(): @@ -625,56 +651,13 @@ class PlugPorts(BaseNetworkTask): self.network_driver.plug_port(db_amp, port) -class PlugVIPPort(BaseNetworkTask): - """Task to plug a VIP into a compute instance.""" - - def execute(self, amphora, amphorae_network_config): - vrrp_port = amphorae_network_config.get( - amphora.get(constants.ID))[constants.VRRP_PORT] - LOG.debug('Plugging VIP VRRP port ID: %(port_id)s into compute ' - 'instance: %(compute_id)s.', - {constants.PORT_ID: vrrp_port.get(constants.ID), - constants.COMPUTE_ID: amphora[constants.COMPUTE_ID]}) - db_vrrp_port = self.network_driver.get_port( - vrrp_port.get(constants.ID)) - db_amp = self.amphora_repo.get(db_apis.get_session(), - id=amphora[constants.ID]) - self.network_driver.plug_port(db_amp, db_vrrp_port) - - def revert(self, result, amphora, amphorae_network_config, - *args, **kwargs): - vrrp_port = None - try: - vrrp_port = amphorae_network_config.get( - amphora.get(constants.ID))[constants.VRRP_PORT] - db_vrrp_port = self.network_driver.get_port( - vrrp_port.get(constants.ID)) - db_amp = self.amphora_repo.get(db_apis.get_session(), - id=amphora[constants.ID]) - self.network_driver.unplug_port(db_amp, db_vrrp_port) - except Exception: - LOG.warning('Failed to unplug vrrp port: %(port)s from amphora: ' - '%(amp)s', - {'port': vrrp_port, 'amp': amphora[constants.ID]}) - - -class WaitForPortDetach(BaseNetworkTask): - """Task to wait for the neutron ports to detach from an amphora.""" - - def execute(self, amphora): - LOG.debug('Waiting for ports to detach from amphora: %(amp_id)s.', - {'amp_id': amphora.get(constants.ID)}) - db_amp = self.amphora_repo.get(db_apis.get_session(), - id=amphora.get(constants.ID)) - self.network_driver.wait_for_port_detach(db_amp) - - class ApplyQos(BaseNetworkTask): """Apply Quality of Services to the VIP""" def _apply_qos_on_vrrp_ports(self, loadbalancer, amps_data, qos_policy_id, is_revert=False, request_qos_id=None): """Call network driver to apply QoS Policy on the vrrp ports.""" + if not amps_data: db_lb = self.loadbalancer_repo.get( db_apis.get_session(), @@ -688,12 +671,21 @@ class ApplyQos(BaseNetworkTask): def execute(self, loadbalancer, amps_data=None, update_dict=None): """Apply qos policy on the vrrp ports which are related with vip.""" - qos_policy_id = loadbalancer['vip_qos_policy_id'] + db_lb = self.loadbalancer_repo.get( + db_apis.get_session(), + id=loadbalancer[constants.LOADBALANCER_ID]) + + qos_policy_id = db_lb.vip.qos_policy_id if not qos_policy_id and ( not update_dict or ( 'vip' not in update_dict or 'qos_policy_id' not in update_dict[constants.VIP])): return + if update_dict and update_dict.get(constants.VIP): + vip_dict = update_dict[constants.VIP] + if vip_dict.get(constants.QOS_POLICY_ID): + qos_policy_id = vip_dict[constants.QOS_POLICY_ID] + self._apply_qos_on_vrrp_ports(loadbalancer, amps_data, qos_policy_id) def revert(self, result, loadbalancer, amps_data=None, update_dict=None, @@ -756,3 +748,147 @@ class ApplyQosAmphora(BaseNetworkTask): LOG.error('Failed to remove QoS policy: %s from port: %s due ' 'to error: %s', orig_qos_id, amp_data[constants.VRRP_PORT_ID], e) + + +class DeletePort(BaseNetworkTask): + """Task to delete a network port.""" + + @tenacity.retry(retry=tenacity.retry_if_exception_type(), + stop=tenacity.stop_after_attempt( + CONF.networking.max_retries), + wait=tenacity.wait_exponential( + multiplier=CONF.networking.retry_backoff, + min=CONF.networking.retry_interval, + max=CONF.networking.retry_max), reraise=True) + def execute(self, port_id, passive_failure=False): + """Delete the network port.""" + if port_id is None: + return + if self.execute.retry.statistics.get(constants.ATTEMPT_NUMBER, 1) == 1: + LOG.debug("Deleting network port %s", port_id) + else: + LOG.warning('Retrying network port %s delete attempt %s of %s.', + port_id, + self.execute.retry.statistics[ + constants.ATTEMPT_NUMBER], + self.execute.retry.stop.max_attempt_number) + # Let the Taskflow engine know we are working and alive + # Don't use get with a default for 'attempt_number', we need to fail + # if that number is missing. + self.update_progress( + self.execute.retry.statistics[constants.ATTEMPT_NUMBER] / + self.execute.retry.stop.max_attempt_number) + try: + self.network_driver.delete_port(port_id) + except Exception: + if (self.execute.retry.statistics[constants.ATTEMPT_NUMBER] != + self.execute.retry.stop.max_attempt_number): + LOG.warning('Network port delete for port id: %s failed. ' + 'Retrying.', port_id) + raise + if passive_failure: + LOG.exception('Network port delete for port ID: %s failed. ' + 'This resource will be abandoned and should ' + 'manually be cleaned up once the ' + 'network service is functional.', port_id) + # Let's at least attempt to disable it so if the instance + # comes back from the dead it doesn't conflict with anything. + try: + self.network_driver.admin_down_port(port_id) + LOG.info('Successfully disabled (admin down) network port ' + '%s that failed to delete.', port_id) + except Exception: + LOG.warning('Attempt to disable (admin down) network port ' + '%s failed. The network service has failed. ' + 'Continuing.', port_id) + else: + LOG.exception('Network port delete for port ID: %s failed. ' + 'The network service has failed. ' + 'Aborting and reverting.', port_id) + raise + + +class CreateVIPBasePort(BaseNetworkTask): + """Task to create the VIP base port for an amphora.""" + + @tenacity.retry(retry=tenacity.retry_if_exception_type(), + stop=tenacity.stop_after_attempt( + CONF.networking.max_retries), + wait=tenacity.wait_exponential( + multiplier=CONF.networking.retry_backoff, + min=CONF.networking.retry_interval, + max=CONF.networking.retry_max), reraise=True) + def execute(self, vip, vip_sg_id, amphora_id): + port_name = constants.AMP_BASE_PORT_PREFIX + amphora_id + fixed_ips = [{constants.SUBNET_ID: vip[constants.SUBNET_ID]}] + sg_id = [] + if vip_sg_id: + sg_id = [vip_sg_id] + port = self.network_driver.create_port( + vip[constants.NETWORK_ID], name=port_name, fixed_ips=fixed_ips, + secondary_ips=[vip[constants.IP_ADDRESS]], + security_group_ids=sg_id, + qos_policy_id=vip[constants.QOS_POLICY_ID]) + LOG.info('Created port %s with ID %s for amphora %s', + port_name, port.id, amphora_id) + return port.to_dict(recurse=True) + + def revert(self, result, vip, vip_sg_id, amphora_id, *args, **kwargs): + if isinstance(result, failure.Failure): + return + try: + port_name = constants.AMP_BASE_PORT_PREFIX + amphora_id + for port in result: + self.network_driver.delete_port(port.id) + LOG.info('Deleted port %s with ID %s for amphora %s due to a ' + 'revert.', port_name, port.id, amphora_id) + except Exception as e: + LOG.error('Failed to delete port %s. Resources may still be in ' + 'use for a port intended for amphora %s due to error ' + '%s. Search for a port named %s', + result, amphora_id, str(e), port_name) + + +class AdminDownPort(BaseNetworkTask): + + def execute(self, port_id): + try: + self.network_driver.set_port_admin_state_up(port_id, False) + except base.PortNotFound: + return + for i in range(CONF.networking.max_retries): + port = self.network_driver.get_port(port_id) + if port.status == constants.DOWN: + LOG.debug('Disabled port: %s', port_id) + return + LOG.debug('Port %s is %s instead of DOWN, waiting.', + port_id, port.status) + time.sleep(CONF.networking.retry_interval) + LOG.error('Port %s failed to go DOWN. Port status is still %s. ' + 'Ignoring and continuing.', port_id, port.status) + + def revert(self, result, port_id, *args, **kwargs): + if isinstance(result, failure.Failure): + return + try: + self.network_driver.set_port_admin_state_up(port_id, True) + except Exception as e: + LOG.error('Failed to bring port %s admin up on revert due to: %s.', + port_id, str(e)) + + +class GetVIPSecurityGroupID(BaseNetworkTask): + + def execute(self, loadbalancer_id): + sg_name = utils.get_vip_security_group_name(loadbalancer_id) + try: + security_group = self.network_driver.get_security_group(sg_name) + if security_group: + return security_group.id + except base.SecurityGroupNotFound: + with excutils.save_and_reraise_exception() as ctxt: + if self.network_driver.sec_grp_enabled: + LOG.error('VIP security group %s was not found.', sg_name) + else: + ctxt.reraise = False + return None diff --git a/octavia/controller/worker/v2/tasks/retry_tasks.py b/octavia/controller/worker/v2/tasks/retry_tasks.py new file mode 100644 index 0000000000..04efa852e9 --- /dev/null +++ b/octavia/controller/worker/v2/tasks/retry_tasks.py @@ -0,0 +1,73 @@ +# Copyright 2019 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import time + +from oslo_log import log as logging +from taskflow import retry + +LOG = logging.getLogger(__name__) + + +class SleepingRetryTimesController(retry.Times): + """A retry controller to attempt subflow retries a number of times. + + This retry controller overrides the Times on_failure to inject a + sleep interval between retries. + It also adds a log message when all of the retries are exhausted. + + :param attempts: number of attempts to retry the associated subflow + before giving up + :type attempts: int + :param name: Meaningful name for this atom, should be something that is + distinguishable and understandable for notification, + debugging, storing and any other similar purposes. + :param provides: A set, string or list of items that + this will be providing (or could provide) to others, used + to correlate and associate the thing/s this atom + produces, if it produces anything at all. + :param requires: A set or list of required inputs for this atom's + ``execute`` method. + :param rebind: A dict of key/value pairs used to define argument + name conversions for inputs to this atom's ``execute`` + method. + :param revert_all: when provided this will cause the full flow to revert + when the number of attempts that have been tried + has been reached (when false, it will only locally + revert the associated subflow) + :type revert_all: bool + :param interval: Interval, in seconds, between retry attempts. + :type interval: int + """ + + def __init__(self, attempts=1, name=None, provides=None, requires=None, + auto_extract=True, rebind=None, revert_all=False, interval=1): + super().__init__(attempts, name, provides, requires, auto_extract, + rebind, revert_all) + self._interval = interval + + def on_failure(self, history, *args, **kwargs): + if len(history) < self._attempts: + LOG.warning('%s attempt %s of %s failed. Sleeping %s seconds and ' + 'retrying.', + self.name[self.name.startswith('retry-') and + len('retry-'):], len(history), + self._attempts, self._interval) + time.sleep(self._interval) + return retry.RETRY + return self._revert_action + + def revert(self, history, *args, **kwargs): + LOG.error('%s retries with interval %s seconds have failed for %s. ' + 'Giving up.', len(history), self._interval, self.name) diff --git a/octavia/tests/unit/controller/worker/v1/flows/test_amphora_flows.py b/octavia/tests/unit/controller/worker/v1/flows/test_amphora_flows.py index 69f798b301..dd26fd331a 100644 --- a/octavia/tests/unit/controller/worker/v1/flows/test_amphora_flows.py +++ b/octavia/tests/unit/controller/worker/v1/flows/test_amphora_flows.py @@ -315,21 +315,9 @@ class TestAmphoraFlows(base.TestCase): self.assertIsInstance(amp_flow, flow.Flow) - # TODO(johnsom) Uncomment after amphora failover builds a replacement - # amphora. - # self.assertIn(constants.AVAILABILITY_ZONE, amp_flow.requires) - # self.assertIn(constants.BUILD_TYPE_PRIORITY, amp_flow.requires) - # self.assertIn(constants.FLAVOR, amp_flow.requires) - # self.assertEqual(5, len(amp_flow.requires)) self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires) - # self.assertIn(constants.AMPHORA, amp_flow.provides) - # self.assertIn(constants.AMPHORA_ID, amp_flow.provides) - # self.assertIn(constants.COMPUTE_ID, amp_flow.provides) - # self.assertIn(constants.COMPUTE_OBJ, amp_flow.provides) - # self.assertIn(constants.SERVER_PEM, amp_flow.provides) - # self.assertIn(constants.VIP_SG_ID, amp_flow.provides) - # self.assertEqual(6, len(amp_flow.provides)) + self.assertIn(constants.VIP_SG_ID, amp_flow.provides) self.assertEqual(1, len(amp_flow.requires)) self.assertEqual(1, len(amp_flow.provides)) @@ -340,14 +328,10 @@ class TestAmphoraFlows(base.TestCase): self.assertIsInstance(amp_flow, flow.Flow) - # TODO(johnsom) Uncomment after amphora failover builds a replacement - # amphora. - # self.assertIn(constants.AVAILABILITY_ZONE, amp_flow.requires) - # self.assertIn(constants.BUILD_TYPE_PRIORITY, amp_flow.requires) - # self.assertEqual(5, len(amp_flow.requires)) - # self.assertEqual(6, len(amp_flow.provides)) self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires) + self.assertIn(constants.VIP_SG_ID, amp_flow.provides) + self.assertEqual(1, len(amp_flow.requires)) self.assertEqual(1, len(amp_flow.provides)) diff --git a/octavia/tests/unit/controller/worker/v1/tasks/test_amphora_driver_tasks.py b/octavia/tests/unit/controller/worker/v1/tasks/test_amphora_driver_tasks.py index ab72029ac7..c614eb5921 100644 --- a/octavia/tests/unit/controller/worker/v1/tasks/test_amphora_driver_tasks.py +++ b/octavia/tests/unit/controller/worker/v1/tasks/test_amphora_driver_tasks.py @@ -203,13 +203,13 @@ class TestAmphoraDriverTasks(base.TestCase): # Test no listeners mock_lb.listeners = None - listeners_reload_obj.execute(mock_lb, None, 0) + listeners_reload_obj.execute(mock_lb, 0, None) mock_driver.reload.assert_not_called() # Test with listeners mock_driver.start.reset_mock() mock_lb.listeners = [mock_listener] - listeners_reload_obj.execute(mock_lb, [amphora_mock], 0, + listeners_reload_obj.execute(mock_lb, 0, [amphora_mock], timeout_dict=self.timeout_dict) mock_driver.reload.assert_called_once_with(mock_lb, amphora_mock, self.timeout_dict) @@ -620,7 +620,7 @@ class TestAmphoraDriverTasks(base.TestCase): amphora_update_vrrp_interface_obj = ( amphora_driver_tasks.AmphoraIndexUpdateVRRPInterface()) amphora_update_vrrp_interface_obj.execute( - [_amphora_mock], 0, timeout_dict) + 0, [_amphora_mock], timeout_dict) mock_driver.get_interface_from_ip.assert_called_once_with( _amphora_mock, _amphora_mock.vrrp_ip, timeout_dict=timeout_dict) mock_amphora_repo_update.assert_called_once_with( @@ -629,7 +629,7 @@ class TestAmphoraDriverTasks(base.TestCase): # Test with an exception mock_amphora_repo_update.reset_mock() amphora_update_vrrp_interface_obj.execute( - [_amphora_mock], 0, timeout_dict) + 0, [_amphora_mock], timeout_dict) mock_amphora_repo_update.assert_called_once_with( _session_mock, _amphora_mock.id, status=constants.ERROR) @@ -692,19 +692,6 @@ class TestAmphoraDriverTasks(base.TestCase): mock_amphora_repo_update.assert_called_once_with( _session_mock, _amphora_mock.id, status=constants.ERROR) - def test_amphora_vrrp_stop(self, - mock_driver, - mock_generate_uuid, - mock_log, - mock_get_session, - mock_listener_repo_get, - mock_listener_repo_update, - mock_amphora_repo_update): - amphora_vrrp_stop_obj = ( - amphora_driver_tasks.AmphoraVRRPStop()) - amphora_vrrp_stop_obj.execute(_LB_mock) - mock_driver.stop_vrrp_service.assert_called_once_with(_LB_mock) - def test_amphora_vrrp_start(self, mock_driver, mock_generate_uuid, diff --git a/octavia/tests/unit/controller/worker/v1/test_controller_worker.py b/octavia/tests/unit/controller/worker/v1/test_controller_worker.py index e63783afd8..eeafb79cb4 100644 --- a/octavia/tests/unit/controller/worker/v1/test_controller_worker.py +++ b/octavia/tests/unit/controller/worker/v1/test_controller_worker.py @@ -1571,54 +1571,6 @@ class TestControllerWorker(base.TestCase): assert_called_once_with(FAKE_FLOW, store=expected_stored_params)) _flow_mock.run.assert_called_once_with() - @mock.patch( - 'octavia.db.repositories.AmphoraRepository.get_lb_for_amphora', - return_value=None) - @mock.patch('octavia.controller.worker.v1.flows.' - 'amphora_flows.AmphoraFlows.get_failover_amphora_flow', - return_value=_flow_mock) - def test_failover_spare_amphora(self, - mock_get_failover_flow, - mock_get_lb_for_amphora, - mock_api_get_session, - mock_dyn_log_listener, - mock_taskflow_load, - mock_pool_repo_get, - mock_member_repo_get, - mock_l7rule_repo_get, - mock_l7policy_repo_get, - mock_listener_repo_get, - mock_lb_repo_get, - mock_health_mon_repo_get, - mock_amp_repo_get): - - _flow_mock.reset_mock() - - # simulate a spare amphora (amphora not attached to any load_balancer) - mock_amphora = mock.MagicMock() - mock_amphora.id = AMP_ID - mock_amphora.status = constants.AMPHORA_READY - mock_amphora.load_balancer_id = None - mock_amp_repo_get.return_value = mock_amphora - - cw = controller_worker.ControllerWorker() - cw.failover_amphora(AMP_ID) - - (base_taskflow.BaseTaskFlowEngine.taskflow_load. - assert_called_once_with( - _flow_mock, - store={constants.LOADBALANCER: None, - constants.LOADBALANCER_ID: None, - constants.BUILD_TYPE_PRIORITY: - constants.LB_CREATE_FAILOVER_PRIORITY, - constants.FLAVOR: {}, - constants.SERVER_GROUP_ID: None, - constants.AVAILABILITY_ZONE: {}, - constants.VIP: None - })) - - _flow_mock.run.assert_called_once_with() - @mock.patch('octavia.db.repositories.AmphoraHealthRepository.delete') def test_failover_deleted_amphora(self, mock_delete, diff --git a/octavia/tests/unit/controller/worker/v2/flows/test_amphora_flows.py b/octavia/tests/unit/controller/worker/v2/flows/test_amphora_flows.py index 3965848803..8cf8996513 100644 --- a/octavia/tests/unit/controller/worker/v2/flows/test_amphora_flows.py +++ b/octavia/tests/unit/controller/worker/v2/flows/test_amphora_flows.py @@ -16,6 +16,7 @@ from unittest import mock from oslo_config import cfg from oslo_config import fixture as oslo_fixture +from oslo_utils import uuidutils from taskflow.patterns import linear_flow as flow from octavia.common import constants @@ -42,6 +43,7 @@ class TestAmphoraFlows(base.TestCase): self.amp1 = data_models.Amphora(id=1) self.amp2 = data_models.Amphora(id=2) self.amp3 = data_models.Amphora(id=3, status=constants.DELETED) + self.amp4 = data_models.Amphora(id=uuidutils.generate_uuid()) self.lb = data_models.LoadBalancer( id=4, amphorae=[self.amp1, self.amp2, self.amp3]) @@ -62,7 +64,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.AVAILABILITY_ZONE, amp_flow.requires) self.assertEqual(5, len(amp_flow.provides)) - self.assertEqual(3, len(amp_flow.requires)) + self.assertEqual(4, len(amp_flow.requires)) def test_get_create_amphora_flow_cert(self, mock_get_net_driver): self.AmpFlow = amphora_flows.AmphoraFlows() @@ -82,7 +84,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.AVAILABILITY_ZONE, amp_flow.requires) self.assertEqual(5, len(amp_flow.provides)) - self.assertEqual(3, len(amp_flow.requires)) + self.assertEqual(4, len(amp_flow.requires)) def test_get_create_amphora_for_lb_flow(self, mock_get_net_driver): @@ -103,7 +105,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.SERVER_PEM, amp_flow.provides) self.assertEqual(5, len(amp_flow.provides)) - self.assertEqual(4, len(amp_flow.requires)) + self.assertEqual(5, len(amp_flow.requires)) def test_get_cert_create_amphora_for_lb_flow(self, mock_get_net_driver): @@ -126,7 +128,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.SERVER_PEM, amp_flow.provides) self.assertEqual(5, len(amp_flow.provides)) - self.assertEqual(4, len(amp_flow.requires)) + self.assertEqual(5, len(amp_flow.requires)) def test_get_cert_master_create_amphora_for_lb_flow( self, mock_get_net_driver): @@ -150,7 +152,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.SERVER_PEM, amp_flow.provides) self.assertEqual(5, len(amp_flow.provides)) - self.assertEqual(4, len(amp_flow.requires)) + self.assertEqual(5, len(amp_flow.requires)) def test_get_cert_master_rest_anti_affinity_create_amphora_for_lb_flow( self, mock_get_net_driver): @@ -200,7 +202,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.SERVER_PEM, amp_flow.provides) self.assertEqual(5, len(amp_flow.provides)) - self.assertEqual(4, len(amp_flow.requires)) + self.assertEqual(5, len(amp_flow.requires)) def test_get_cert_bogus_create_amphora_for_lb_flow( self, mock_get_net_driver): @@ -223,7 +225,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.SERVER_PEM, amp_flow.provides) self.assertEqual(5, len(amp_flow.provides)) - self.assertEqual(4, len(amp_flow.requires)) + self.assertEqual(5, len(amp_flow.requires)) def test_get_cert_backup_rest_anti_affinity_create_amphora_for_lb_flow( self, mock_get_net_driver): @@ -253,14 +255,14 @@ class TestAmphoraFlows(base.TestCase): def test_get_delete_amphora_flow(self, mock_get_net_driver): - amp_flow = self.AmpFlow.get_delete_amphora_flow() + amp_flow = self.AmpFlow.get_delete_amphora_flow(self.amp4.to_dict()) self.assertIsInstance(amp_flow, flow.Flow) - self.assertIn(constants.AMPHORA, amp_flow.requires) + # This flow injects the required data at flow compile time. self.assertEqual(0, len(amp_flow.provides)) - self.assertEqual(1, len(amp_flow.requires)) + self.assertEqual(0, len(amp_flow.requires)) def test_allocate_amp_to_lb_decider(self, mock_get_net_driver): history = mock.MagicMock() @@ -282,102 +284,103 @@ class TestAmphoraFlows(base.TestCase): result = self.AmpFlow._create_new_amp_for_lb_decider(history) self.assertFalse(result) - @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') - @mock.patch('octavia.db.api.get_session', return_value=mock.MagicMock()) - def test_get_failover_flow_allocated(self, mock_session, mock_get_lb, - mock_get_net_driver): - mock_get_lb.return_value = self.lb - provider_lb = {constants.LOADBALANCER_ID: '1234'} - amp_flow = self.AmpFlow.get_failover_flow( - load_balancer=provider_lb) + def test_get_failover_flow_act_stdby(self, mock_get_net_driver): + failed_amphora = data_models.Amphora( + id=uuidutils.generate_uuid(), role=constants.ROLE_MASTER, + load_balancer_id=uuidutils.generate_uuid()).to_dict() + + amp_flow = self.AmpFlow.get_failover_amphora_flow( + failed_amphora, 2) self.assertIsInstance(amp_flow, flow.Flow) - self.assertIn(constants.FAILED_AMPHORA, amp_flow.requires) + self.assertIn(constants.AVAILABILITY_ZONE, amp_flow.requires) + self.assertIn(constants.BUILD_TYPE_PRIORITY, amp_flow.requires) + self.assertIn(constants.FLAVOR, amp_flow.requires) + self.assertIn(constants.LOADBALANCER, amp_flow.requires) self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires) + self.assertIn(constants.VIP, amp_flow.requires) - self.assertIn(constants.AMP_DATA, amp_flow.provides) + self.assertIn(constants.ADDED_PORTS, amp_flow.provides) + self.assertIn(constants.AMP_VRRP_INT, amp_flow.provides) self.assertIn(constants.AMPHORA, amp_flow.provides) self.assertIn(constants.AMPHORA_ID, amp_flow.provides) + self.assertIn(constants.AMPHORAE, amp_flow.provides) self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, amp_flow.provides) + self.assertIn(constants.BASE_PORT, amp_flow.provides) self.assertIn(constants.COMPUTE_ID, amp_flow.provides) self.assertIn(constants.COMPUTE_OBJ, amp_flow.provides) - self.assertIn(constants.LISTENERS, amp_flow.provides) + self.assertIn(constants.DELTA, amp_flow.provides) self.assertIn(constants.LOADBALANCER, amp_flow.provides) + self.assertIn(constants.SERVER_PEM, amp_flow.provides) + self.assertIn(constants.VIP_SG_ID, amp_flow.provides) - self.assertEqual(5, len(amp_flow.requires)) - self.assertEqual(12, len(amp_flow.provides)) + self.assertEqual(7, len(amp_flow.requires)) + self.assertEqual(13, len(amp_flow.provides)) - amp_flow = self.AmpFlow.get_failover_flow( - role=constants.ROLE_MASTER, load_balancer=provider_lb) + def test_get_failover_flow_standalone(self, mock_get_net_driver): + failed_amphora = data_models.Amphora( + id=uuidutils.generate_uuid(), role=constants.ROLE_STANDALONE, + load_balancer_id=uuidutils.generate_uuid(), + vrrp_ip='2001:3b8::32').to_dict() + + amp_flow = self.AmpFlow.get_failover_amphora_flow( + failed_amphora, 1) self.assertIsInstance(amp_flow, flow.Flow) - self.assertIn(constants.FAILED_AMPHORA, amp_flow.requires) + self.assertIn(constants.AVAILABILITY_ZONE, amp_flow.requires) + self.assertIn(constants.BUILD_TYPE_PRIORITY, amp_flow.requires) + self.assertIn(constants.FLAVOR, amp_flow.requires) + self.assertIn(constants.LOADBALANCER, amp_flow.requires) self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires) + self.assertIn(constants.VIP, amp_flow.requires) - self.assertIn(constants.AMP_DATA, amp_flow.provides) + self.assertIn(constants.ADDED_PORTS, amp_flow.provides) self.assertIn(constants.AMPHORA, amp_flow.provides) self.assertIn(constants.AMPHORA_ID, amp_flow.provides) + self.assertIn(constants.AMPHORAE, amp_flow.provides) self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, amp_flow.provides) + self.assertIn(constants.BASE_PORT, amp_flow.provides) self.assertIn(constants.COMPUTE_ID, amp_flow.provides) self.assertIn(constants.COMPUTE_OBJ, amp_flow.provides) - self.assertIn(constants.LISTENERS, amp_flow.provides) + self.assertIn(constants.DELTA, amp_flow.provides) self.assertIn(constants.LOADBALANCER, amp_flow.provides) + self.assertIn(constants.SERVER_PEM, amp_flow.provides) + self.assertIn(constants.VIP_SG_ID, amp_flow.provides) - self.assertEqual(5, len(amp_flow.requires)) + self.assertEqual(7, len(amp_flow.requires)) self.assertEqual(12, len(amp_flow.provides)) - amp_flow = self.AmpFlow.get_failover_flow( - role=constants.ROLE_BACKUP, load_balancer=provider_lb) + def test_get_failover_flow_bogus_role(self, mock_get_net_driver): + failed_amphora = data_models.Amphora(id=uuidutils.generate_uuid(), + role='bogus').to_dict() + + amp_flow = self.AmpFlow.get_failover_amphora_flow( + failed_amphora, 1) self.assertIsInstance(amp_flow, flow.Flow) - self.assertIn(constants.FAILED_AMPHORA, amp_flow.requires) self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires) - self.assertIn(constants.AMP_DATA, amp_flow.provides) - self.assertIn(constants.AMPHORA, amp_flow.provides) - self.assertIn(constants.AMPHORA_ID, amp_flow.provides) - self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, amp_flow.provides) - self.assertIn(constants.COMPUTE_ID, amp_flow.provides) - self.assertIn(constants.COMPUTE_OBJ, amp_flow.provides) - self.assertIn(constants.LISTENERS, amp_flow.provides) - self.assertIn(constants.LOADBALANCER, amp_flow.provides) + self.assertIn(constants.VIP_SG_ID, amp_flow.provides) - self.assertEqual(5, len(amp_flow.requires)) - self.assertEqual(12, len(amp_flow.provides)) - - amp_flow = self.AmpFlow.get_failover_flow( - role='BOGUSROLE', load_balancer=provider_lb) - - self.assertIsInstance(amp_flow, flow.Flow) - - self.assertIn(constants.FAILED_AMPHORA, amp_flow.requires) - self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires) - - self.assertIn(constants.AMP_DATA, amp_flow.provides) - self.assertIn(constants.AMPHORA, amp_flow.provides) - self.assertIn(constants.AMPHORA_ID, amp_flow.provides) - self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, amp_flow.provides) - self.assertIn(constants.COMPUTE_ID, amp_flow.provides) - self.assertIn(constants.COMPUTE_OBJ, amp_flow.provides) - self.assertIn(constants.LISTENERS, amp_flow.provides) - self.assertIn(constants.LOADBALANCER, amp_flow.provides) - - self.assertEqual(5, len(amp_flow.requires)) - self.assertEqual(12, len(amp_flow.provides)) + self.assertEqual(1, len(amp_flow.requires)) + self.assertEqual(1, len(amp_flow.provides)) def test_get_failover_flow_spare(self, mock_get_net_driver): - amp_flow = self.AmpFlow.get_failover_flow() + amp_flow = self.AmpFlow.get_failover_amphora_flow( + self.amp4.to_dict(), 0) self.assertIsInstance(amp_flow, flow.Flow) - self.assertIn(constants.FAILED_AMPHORA, amp_flow.requires) + self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires) + + self.assertIn(constants.VIP_SG_ID, amp_flow.provides) self.assertEqual(1, len(amp_flow.requires)) - self.assertEqual(0, len(amp_flow.provides)) + self.assertEqual(1, len(amp_flow.provides)) def test_cert_rotate_amphora_flow(self, mock_get_net_driver): self.AmpFlow = amphora_flows.AmphoraFlows() @@ -396,12 +399,30 @@ class TestAmphoraFlows(base.TestCase): self.assertIsInstance(vrrp_subflow, flow.Flow) - self.assertIn(constants.LOADBALANCER, vrrp_subflow.provides) + self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, vrrp_subflow.provides) + self.assertIn(constants.AMP_VRRP_INT, vrrp_subflow.provides) - self.assertIn(constants.LOADBALANCER, vrrp_subflow.requires) + self.assertIn(constants.LOADBALANCER_ID, vrrp_subflow.requires) + self.assertIn(constants.AMPHORAE, vrrp_subflow.requires) self.assertEqual(2, len(vrrp_subflow.provides)) - self.assertEqual(1, len(vrrp_subflow.requires)) + self.assertEqual(2, len(vrrp_subflow.requires)) + + def test_get_vrrp_subflow_dont_create_vrrp_group( + self, mock_get_net_driver): + vrrp_subflow = self.AmpFlow.get_vrrp_subflow('123', + create_vrrp_group=False) + + self.assertIsInstance(vrrp_subflow, flow.Flow) + + self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, vrrp_subflow.provides) + self.assertIn(constants.AMP_VRRP_INT, vrrp_subflow.provides) + + self.assertIn(constants.LOADBALANCER_ID, vrrp_subflow.requires) + self.assertIn(constants.AMPHORAE, vrrp_subflow.requires) + + self.assertEqual(2, len(vrrp_subflow.provides)) + self.assertEqual(2, len(vrrp_subflow.requires)) def test_get_post_map_lb_subflow(self, mock_get_net_driver): @@ -414,9 +435,8 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.FLAVOR, amp_flow.requires) self.assertIn(constants.AMPHORA, amp_flow.requires) - self.assertIn(constants.AMPHORA, amp_flow.provides) - self.assertEqual(1, len(amp_flow.provides)) + self.assertEqual(0, len(amp_flow.provides)) self.assertEqual(2, len(amp_flow.requires)) amp_flow = self.AmpFlow._get_post_map_lb_subflow( @@ -426,9 +446,8 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.FLAVOR, amp_flow.requires) self.assertIn(constants.AMPHORA, amp_flow.requires) - self.assertIn(constants.AMPHORA, amp_flow.provides) - self.assertEqual(1, len(amp_flow.provides)) + self.assertEqual(0, len(amp_flow.provides)) self.assertEqual(2, len(amp_flow.requires)) amp_flow = self.AmpFlow._get_post_map_lb_subflow( @@ -438,9 +457,8 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.FLAVOR, amp_flow.requires) self.assertIn(constants.AMPHORA, amp_flow.requires) - self.assertIn(constants.AMPHORA, amp_flow.provides) - self.assertEqual(1, len(amp_flow.provides)) + self.assertEqual(0, len(amp_flow.provides)) self.assertEqual(2, len(amp_flow.requires)) amp_flow = self.AmpFlow._get_post_map_lb_subflow( @@ -450,9 +468,8 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.FLAVOR, amp_flow.requires) self.assertIn(constants.AMPHORA, amp_flow.requires) - self.assertIn(constants.AMPHORA, amp_flow.provides) - self.assertEqual(1, len(amp_flow.provides)) + self.assertEqual(0, len(amp_flow.provides)) self.assertEqual(2, len(amp_flow.requires)) def test_update_amphora_config_flow(self, mock_get_net_driver): diff --git a/octavia/tests/unit/controller/worker/v2/flows/test_listener_flows.py b/octavia/tests/unit/controller/worker/v2/flows/test_listener_flows.py index 709c91b9df..45ae52f378 100644 --- a/octavia/tests/unit/controller/worker/v2/flows/test_listener_flows.py +++ b/octavia/tests/unit/controller/worker/v2/flows/test_listener_flows.py @@ -14,6 +14,7 @@ # from unittest import mock +from oslo_utils import uuidutils from taskflow.patterns import linear_flow as flow from octavia.common import constants @@ -57,15 +58,16 @@ class TestListenerFlows(base.TestCase): self.assertEqual(0, len(listener_flow.provides)) def test_get_delete_listener_internal_flow(self, mock_get_net_driver): + fake_listener = {constants.LISTENER_ID: uuidutils.generate_uuid()} listener_flow = self.ListenerFlow.get_delete_listener_internal_flow( - 'test-listener') + fake_listener) self.assertIsInstance(listener_flow, flow.Flow) - self.assertIn('test-listener', listener_flow.requires) + self.assertIn(constants.LOADBALANCER_ID, listener_flow.requires) self.assertIn(constants.PROJECT_ID, listener_flow.requires) - self.assertEqual(3, len(listener_flow.requires)) + self.assertEqual(2, len(listener_flow.requires)) self.assertEqual(0, len(listener_flow.provides)) def test_get_update_listener_flow(self, mock_get_net_driver): diff --git a/octavia/tests/unit/controller/worker/v2/flows/test_load_balancer_flows.py b/octavia/tests/unit/controller/worker/v2/flows/test_load_balancer_flows.py index ba2127ea95..df3ad0957d 100644 --- a/octavia/tests/unit/controller/worker/v2/flows/test_load_balancer_flows.py +++ b/octavia/tests/unit/controller/worker/v2/flows/test_load_balancer_flows.py @@ -16,10 +16,12 @@ from unittest import mock from oslo_config import cfg from oslo_config import fixture as oslo_fixture +from oslo_utils import uuidutils from taskflow.patterns import linear_flow as flow from octavia.common import constants from octavia.common import exceptions +from octavia.controller.worker.v2.flows import flow_utils from octavia.controller.worker.v2.flows import load_balancer_flows import octavia.tests.unit.base as base @@ -106,34 +108,38 @@ class TestLoadBalancerFlows(base.TestCase): lb_mock = mock.Mock() listener_mock = mock.Mock() listener_mock.id = '123' - listener_dict = {constants.LISTENER_ID: '123'} listener_mock.to_dict.return_value = {'id': '123'} lb_mock.listeners = [listener_mock] lb_mock.id = '321' lb_mock.project_id = '876' pool_mock = mock.Mock() pool_mock.id = '345' + pool_mock.to_dict.return_value = {constants.ID: pool_mock.id} + pool_mock.listeners = None + pool_mock.health_monitor = None + pool_mock.members = None lb_mock.pools = [pool_mock] l7_mock = mock.Mock() l7_mock.id = '678' listener_mock.l7policies = [l7_mock] mock_get_lb.return_value = lb_mock lb_dict = {constants.LOADBALANCER_ID: lb_mock.id} - store = self.LBFlow.get_delete_listeners_store(lb_mock) - store.update(self.LBFlow.get_delete_pools_store(lb_mock)) + + listeners = flow_utils.get_listeners_on_lb(lb_mock) + pools = flow_utils.get_pools_on_lb(lb_mock) + lb_flow = self.LBFlow.get_cascade_delete_load_balancer_flow( - lb_dict) + lb_dict, listeners, pools) self.assertIsInstance(lb_flow, flow.Flow) - self.assertEqual({'listener_123': listener_dict, - constants.LOADBALANCER_ID: lb_mock.id, - constants.PROJECT_ID: lb_mock.project_id, - 'pool345': pool_mock.id}, store) self.assertIn(constants.LOADBALANCER, lb_flow.requires) + self.assertIn(constants.LOADBALANCER_ID, lb_flow.requires) + self.assertIn(constants.PROJECT_ID, lb_flow.requires) + self.assertIn(constants.SERVER_GROUP_ID, lb_flow.requires) self.assertEqual(1, len(lb_flow.provides)) - self.assertEqual(6, len(lb_flow.requires)) + self.assertEqual(4, len(lb_flow.requires)) def test_get_update_load_balancer_flow(self, mock_get_net_driver): @@ -168,10 +174,14 @@ class TestLoadBalancerFlows(base.TestCase): self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires) self.assertIn(constants.UPDATE_DICT, amp_flow.requires) + + self.assertIn(constants.AMPHORAE, amp_flow.provides) + self.assertIn(constants.AMP_VRRP_INT, amp_flow.provides) + self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, amp_flow.provides) self.assertIn(constants.LOADBALANCER, amp_flow.provides) - self.assertEqual(2, len(amp_flow.provides)) - self.assertEqual(2, len(amp_flow.requires)) + self.assertEqual(2, len(amp_flow.requires), amp_flow.requires) + self.assertEqual(4, len(amp_flow.provides), amp_flow.provides) # Test mark_active=False amp_flow = self.LBFlow.get_post_lb_amp_association_flow( @@ -181,10 +191,14 @@ class TestLoadBalancerFlows(base.TestCase): self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires) self.assertIn(constants.UPDATE_DICT, amp_flow.requires) + + self.assertIn(constants.AMPHORAE, amp_flow.provides) + self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, amp_flow.provides) + self.assertIn(constants.AMP_VRRP_INT, amp_flow.provides) self.assertIn(constants.LOADBALANCER, amp_flow.provides) - self.assertEqual(2, len(amp_flow.provides)) - self.assertEqual(2, len(amp_flow.requires)) + self.assertEqual(2, len(amp_flow.requires), amp_flow.requires) + self.assertEqual(4, len(amp_flow.provides), amp_flow.provides) def test_get_create_load_balancer_flows_single_listeners( self, mock_get_net_driver): @@ -200,16 +214,18 @@ class TestLoadBalancerFlows(base.TestCase): self.assertIn(constants.LISTENERS, create_flow.provides) self.assertIn(constants.AMPHORA, create_flow.provides) self.assertIn(constants.AMPHORA_ID, create_flow.provides) + self.assertIn(constants.AMPHORA_NETWORK_CONFIG, create_flow.provides) + self.assertIn(constants.AMP_DATA, create_flow.provides) self.assertIn(constants.COMPUTE_ID, create_flow.provides) self.assertIn(constants.COMPUTE_OBJ, create_flow.provides) self.assertIn(constants.LOADBALANCER, create_flow.provides) self.assertIn(constants.DELTAS, create_flow.provides) self.assertIn(constants.ADDED_PORTS, create_flow.provides) + self.assertIn(constants.SERVER_PEM, create_flow.provides) + self.assertIn(constants.SUBNET, create_flow.provides) self.assertIn(constants.VIP, create_flow.provides) - self.assertIn(constants.AMP_DATA, create_flow.provides) - self.assertIn(constants.AMPHORA_NETWORK_CONFIG, create_flow.provides) - self.assertEqual(5, len(create_flow.requires)) + self.assertEqual(6, len(create_flow.requires)) self.assertEqual(13, len(create_flow.provides), create_flow.provides) @@ -221,22 +237,232 @@ class TestLoadBalancerFlows(base.TestCase): ) ) self.assertIsInstance(create_flow, flow.Flow) + self.assertIn(constants.AVAILABILITY_ZONE, create_flow.requires) + self.assertIn(constants.BUILD_TYPE_PRIORITY, create_flow.requires) + self.assertIn(constants.FLAVOR, create_flow.requires) self.assertIn(constants.LOADBALANCER_ID, create_flow.requires) + self.assertIn(constants.SERVER_GROUP_ID, create_flow.requires) self.assertIn(constants.UPDATE_DICT, create_flow.requires) - self.assertIn(constants.LISTENERS, create_flow.provides) + self.assertIn(constants.ADDED_PORTS, create_flow.provides) + self.assertIn(constants.AMP_DATA, create_flow.provides) + self.assertIn(constants.AMP_VRRP_INT, create_flow.provides) self.assertIn(constants.AMPHORA, create_flow.provides) + self.assertIn(constants.AMPHORAE, create_flow.provides) self.assertIn(constants.AMPHORA_ID, create_flow.provides) + self.assertIn(constants.AMPHORA_NETWORK_CONFIG, create_flow.provides) + self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, create_flow.provides) self.assertIn(constants.COMPUTE_ID, create_flow.provides) self.assertIn(constants.COMPUTE_OBJ, create_flow.provides) - self.assertIn(constants.LOADBALANCER, create_flow.provides) self.assertIn(constants.DELTAS, create_flow.provides) - self.assertIn(constants.ADDED_PORTS, create_flow.provides) + self.assertIn(constants.LOADBALANCER, create_flow.provides) + self.assertIn(constants.LISTENERS, create_flow.provides) + self.assertIn(constants.SERVER_PEM, create_flow.provides) + self.assertIn(constants.SUBNET, create_flow.provides) self.assertIn(constants.VIP, create_flow.provides) - self.assertIn(constants.AMP_DATA, create_flow.provides) - self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, - create_flow.provides) - self.assertEqual(5, len(create_flow.requires)) - self.assertEqual(14, len(create_flow.provides), + self.assertEqual(6, len(create_flow.requires), create_flow.requires) + self.assertEqual(16, len(create_flow.provides), create_flow.provides) + + def _test_get_failover_LB_flow_single(self, amphorae): + lb_mock = mock.MagicMock() + lb_mock.id = uuidutils.generate_uuid() + lb_mock.topology = constants.TOPOLOGY_SINGLE + + failover_flow = self.LBFlow.get_failover_LB_flow(amphorae, lb_mock) + + self.assertIsInstance(failover_flow, flow.Flow) + + self.assertIn(constants.AVAILABILITY_ZONE, failover_flow.requires) + self.assertIn(constants.BUILD_TYPE_PRIORITY, failover_flow.requires) + self.assertIn(constants.FLAVOR, failover_flow.requires) + self.assertIn(constants.LOADBALANCER, failover_flow.requires) + self.assertIn(constants.LOADBALANCER_ID, failover_flow.requires) + + self.assertIn(constants.ADDED_PORTS, failover_flow.provides) + self.assertIn(constants.AMPHORA, failover_flow.provides) + self.assertIn(constants.AMPHORA_ID, failover_flow.provides) + self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, + failover_flow.provides) + self.assertIn(constants.BASE_PORT, failover_flow.provides) + self.assertIn(constants.COMPUTE_ID, failover_flow.provides) + self.assertIn(constants.COMPUTE_OBJ, failover_flow.provides) + self.assertIn(constants.DELTA, failover_flow.provides) + self.assertIn(constants.LOADBALANCER, failover_flow.provides) + self.assertIn(constants.SERVER_PEM, failover_flow.provides) + self.assertIn(constants.VIP, failover_flow.provides) + self.assertIn(constants.VIP_SG_ID, failover_flow.provides) + + self.assertEqual(6, len(failover_flow.requires), + failover_flow.requires) + self.assertEqual(12, len(failover_flow.provides), + failover_flow.provides) + + def test_get_failover_LB_flow_no_amps_single(self, mock_get_net_driver): + self._test_get_failover_LB_flow_single([]) + + def test_get_failover_LB_flow_one_amp_single(self, mock_get_net_driver): + amphora_dict = {constants.ID: uuidutils.generate_uuid(), + constants.ROLE: constants.ROLE_STANDALONE, + constants.COMPUTE_ID: uuidutils.generate_uuid(), + constants.VRRP_PORT_ID: None, constants.VRRP_IP: None} + + self._test_get_failover_LB_flow_single([amphora_dict]) + + def test_get_failover_LB_flow_one_spare_amp_single(self, + mock_get_net_driver): + amphora_dict = {constants.ID: uuidutils.generate_uuid(), + constants.ROLE: 'bogus', + constants.COMPUTE_ID: uuidutils.generate_uuid(), + constants.VRRP_PORT_ID: None, constants.VRRP_IP: None} + + self._test_get_failover_LB_flow_single([amphora_dict]) + + def test_get_failover_LB_flow_one_bogus_amp_single(self, + mock_get_net_driver): + amphora_dict = {constants.ID: uuidutils.generate_uuid(), + constants.ROLE: 'bogus', + constants.COMPUTE_ID: uuidutils.generate_uuid(), + constants.VRRP_PORT_ID: None, constants.VRRP_IP: None} + + self._test_get_failover_LB_flow_single([amphora_dict]) + + def test_get_failover_LB_flow_two_amp_single(self, mock_get_net_driver): + amphora_dict = {constants.ID: uuidutils.generate_uuid()} + amphora2_dict = {constants.ID: uuidutils.generate_uuid(), + constants.ROLE: constants.ROLE_STANDALONE, + constants.COMPUTE_ID: uuidutils.generate_uuid(), + constants.VRRP_PORT_ID: None, constants.VRRP_IP: None} + + self._test_get_failover_LB_flow_single([amphora_dict, amphora2_dict]) + + def _test_get_failover_LB_flow_no_amps_act_stdby(self, amphorae): + lb_mock = mock.MagicMock() + lb_mock.id = uuidutils.generate_uuid() + lb_mock.topology = constants.TOPOLOGY_ACTIVE_STANDBY + + failover_flow = self.LBFlow.get_failover_LB_flow(amphorae, lb_mock) + + self.assertIsInstance(failover_flow, flow.Flow) + + self.assertIn(constants.AVAILABILITY_ZONE, failover_flow.requires) + self.assertIn(constants.BUILD_TYPE_PRIORITY, failover_flow.requires) + self.assertIn(constants.FLAVOR, failover_flow.requires) + self.assertIn(constants.LOADBALANCER, failover_flow.requires) + self.assertIn(constants.LOADBALANCER_ID, failover_flow.requires) + + self.assertIn(constants.ADDED_PORTS, failover_flow.provides) + self.assertIn(constants.AMPHORA, failover_flow.provides) + self.assertIn(constants.AMPHORA_ID, failover_flow.provides) + self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, + failover_flow.provides) + self.assertIn(constants.BASE_PORT, failover_flow.provides) + self.assertIn(constants.COMPUTE_ID, failover_flow.provides) + self.assertIn(constants.COMPUTE_OBJ, failover_flow.provides) + self.assertIn(constants.DELTA, failover_flow.provides) + self.assertIn(constants.LOADBALANCER, failover_flow.provides) + self.assertIn(constants.SERVER_PEM, failover_flow.provides) + self.assertIn(constants.VIP, failover_flow.provides) + self.assertIn(constants.VIP_SG_ID, failover_flow.provides) + + self.assertEqual(6, len(failover_flow.requires), + failover_flow.requires) + self.assertEqual(12, len(failover_flow.provides), + failover_flow.provides) + + def test_get_failover_LB_flow_no_amps_act_stdby(self, mock_get_net_driver): + self._test_get_failover_LB_flow_no_amps_act_stdby([]) + + def test_get_failover_LB_flow_one_amps_act_stdby(self, amphorae): + amphora_dict = {constants.ID: uuidutils.generate_uuid(), + constants.ROLE: constants.ROLE_MASTER, + constants.COMPUTE_ID: uuidutils.generate_uuid(), + constants.VRRP_PORT_ID: None, constants.VRRP_IP: None} + + self._test_get_failover_LB_flow_no_amps_act_stdby([amphora_dict]) + + def test_get_failover_LB_flow_two_amps_act_stdby(self, + mock_get_net_driver): + amphora_dict = {constants.ID: uuidutils.generate_uuid(), + constants.ROLE: constants.ROLE_MASTER, + constants.COMPUTE_ID: uuidutils.generate_uuid(), + constants.VRRP_PORT_ID: uuidutils.generate_uuid(), + constants.VRRP_IP: '192.0.2.46'} + amphora2_dict = {constants.ID: uuidutils.generate_uuid(), + constants.ROLE: constants.ROLE_BACKUP, + constants.COMPUTE_ID: uuidutils.generate_uuid(), + constants.VRRP_PORT_ID: uuidutils.generate_uuid(), + constants.VRRP_IP: '2001:db8::46'} + + self._test_get_failover_LB_flow_no_amps_act_stdby([amphora_dict, + amphora2_dict]) + + def test_get_failover_LB_flow_three_amps_act_stdby(self, + mock_get_net_driver): + amphora_dict = {constants.ID: uuidutils.generate_uuid(), + constants.ROLE: constants.ROLE_MASTER, + constants.COMPUTE_ID: uuidutils.generate_uuid(), + constants.VRRP_PORT_ID: uuidutils.generate_uuid(), + constants.VRRP_IP: '192.0.2.46'} + amphora2_dict = {constants.ID: uuidutils.generate_uuid(), + constants.ROLE: constants.ROLE_BACKUP, + constants.COMPUTE_ID: uuidutils.generate_uuid(), + constants.VRRP_PORT_ID: uuidutils.generate_uuid(), + constants.VRRP_IP: '2001:db8::46'} + amphora3_dict = {constants.ID: uuidutils.generate_uuid(), + constants.ROLE: 'bogus', + constants.COMPUTE_ID: uuidutils.generate_uuid(), + constants.VRRP_PORT_ID: None, constants.VRRP_IP: None} + + self._test_get_failover_LB_flow_no_amps_act_stdby( + [amphora_dict, amphora2_dict, amphora3_dict]) + + def test_get_failover_LB_flow_two_amps_bogus_act_stdby( + self, mock_get_net_driver): + amphora_dict = {constants.ID: uuidutils.generate_uuid(), + constants.ROLE: 'bogus', + constants.COMPUTE_ID: uuidutils.generate_uuid(), + constants.VRRP_PORT_ID: uuidutils.generate_uuid(), + constants.VRRP_IP: '192.0.2.46'} + amphora2_dict = {constants.ID: uuidutils.generate_uuid(), + constants.ROLE: constants.ROLE_MASTER, + constants.COMPUTE_ID: uuidutils.generate_uuid(), + constants.VRRP_PORT_ID: uuidutils.generate_uuid(), + constants.VRRP_IP: '2001:db8::46'} + + self._test_get_failover_LB_flow_no_amps_act_stdby([amphora_dict, + amphora2_dict]) + + def test_get_failover_LB_flow_two_amps_spare_act_stdby( + self, mock_get_net_driver): + amphora_dict = {constants.ID: uuidutils.generate_uuid(), + constants.ROLE: None, + constants.COMPUTE_ID: uuidutils.generate_uuid(), + constants.VRRP_PORT_ID: uuidutils.generate_uuid(), + constants.VRRP_IP: '192.0.2.46'} + amphora2_dict = {constants.ID: uuidutils.generate_uuid(), + constants.ROLE: constants.ROLE_MASTER, + constants.COMPUTE_ID: uuidutils.generate_uuid(), + constants.VRRP_PORT_ID: uuidutils.generate_uuid(), + constants.VRRP_IP: '2001:db8::46'} + + self._test_get_failover_LB_flow_no_amps_act_stdby([amphora_dict, + amphora2_dict]) + + def test_get_failover_LB_flow_two_amps_standalone_act_stdby( + self, mock_get_net_driver): + amphora_dict = {constants.ID: uuidutils.generate_uuid(), + constants.ROLE: constants.ROLE_STANDALONE, + constants.COMPUTE_ID: uuidutils.generate_uuid(), + constants.VRRP_PORT_ID: uuidutils.generate_uuid(), + constants.VRRP_IP: '192.0.2.46'} + + amphora2_dict = {constants.ID: uuidutils.generate_uuid(), + constants.ROLE: constants.ROLE_MASTER, + constants.COMPUTE_ID: uuidutils.generate_uuid(), + constants.VRRP_PORT_ID: uuidutils.generate_uuid(), + constants.VRRP_IP: '2001:db8::46'} + + self._test_get_failover_LB_flow_no_amps_act_stdby([amphora_dict, + amphora2_dict]) diff --git a/octavia/tests/unit/controller/worker/v2/flows/test_pool_flows.py b/octavia/tests/unit/controller/worker/v2/flows/test_pool_flows.py index 6cbca3bbec..80d39d0186 100644 --- a/octavia/tests/unit/controller/worker/v2/flows/test_pool_flows.py +++ b/octavia/tests/unit/controller/worker/v2/flows/test_pool_flows.py @@ -60,10 +60,9 @@ class TestPoolFlows(base.TestCase): pool_flow = self.PoolFlow.get_delete_pool_flow_internal('test') self.assertIsInstance(pool_flow, flow.Flow) - self.assertIn('test', pool_flow.requires) self.assertIn(constants.PROJECT_ID, pool_flow.requires) - self.assertEqual(2, len(pool_flow.requires)) + self.assertEqual(1, len(pool_flow.requires)) self.assertEqual(1, len(pool_flow.provides)) def test_get_update_pool_flow(self): diff --git a/octavia/tests/unit/controller/worker/v2/tasks/test_amphora_driver_tasks.py b/octavia/tests/unit/controller/worker/v2/tasks/test_amphora_driver_tasks.py index daf927e3d8..058e7fc4b2 100644 --- a/octavia/tests/unit/controller/worker/v2/tasks/test_amphora_driver_tasks.py +++ b/octavia/tests/unit/controller/worker/v2/tasks/test_amphora_driver_tasks.py @@ -41,6 +41,7 @@ FAKE_CONFIG_FILE = 'fake config file' _db_amphora_mock = mock.MagicMock() _db_amphora_mock.id = AMP_ID _db_amphora_mock.status = constants.AMPHORA_ALLOCATED +_db_amphora_mock.vrrp_ip = '198.51.100.65' _amphora_mock = { constants.ID: AMP_ID, constants.STATUS: constants.AMPHORA_ALLOCATED, @@ -81,37 +82,64 @@ class TestAmphoraDriverTasks(base.TestCase): active_connection_rety_interval=CONN_RETRY_INTERVAL) conf.config(group="controller_worker", loadbalancer_topology=constants.TOPOLOGY_SINGLE) + self.timeout_dict = {constants.REQ_CONN_TIMEOUT: 1, + constants.REQ_READ_TIMEOUT: 2, + constants.CONN_MAX_RETRIES: 3, + constants.CONN_RETRY_INTERVAL: 4} super().setUp() @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') - def test_amp_listener_update(self, - mock_lb_get, - mock_driver, - mock_generate_uuid, - mock_log, - mock_get_session, - mock_listener_repo_get, - mock_listener_repo_update, - mock_amphora_repo_get, - mock_amphora_repo_update): + def test_amp_listeners_update(self, + mock_lb_get, + mock_driver, + mock_generate_uuid, + mock_log, + mock_get_session, + mock_listener_repo_get, + mock_listener_repo_update, + mock_amphora_repo_get, + mock_amphora_repo_update): - timeout_dict = {constants.REQ_CONN_TIMEOUT: 1, - constants.REQ_READ_TIMEOUT: 2, - constants.CONN_MAX_RETRIES: 3, - constants.CONN_RETRY_INTERVAL: 4} mock_amphora_repo_get.return_value = _db_amphora_mock mock_lb_get.return_value = _db_load_balancer_mock amp_list_update_obj = amphora_driver_tasks.AmpListenersUpdate() - amp_list_update_obj.execute(_LB_mock, 0, - [_amphora_mock], timeout_dict) + amp_list_update_obj.execute(_LB_mock, _amphora_mock, self.timeout_dict) mock_driver.update_amphora_listeners.assert_called_once_with( - _db_load_balancer_mock, _db_amphora_mock, timeout_dict) + _db_load_balancer_mock, _db_amphora_mock, self.timeout_dict) + + mock_driver.update_amphora_listeners.side_effect = Exception('boom') + + amp_list_update_obj.execute(_LB_mock, _amphora_mock, self.timeout_dict) + + mock_amphora_repo_update.assert_called_once_with( + _session_mock, AMP_ID, status=constants.ERROR) + + @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') + def test_amp_index_listener_update(self, + mock_lb_get, + mock_driver, + mock_generate_uuid, + mock_log, + mock_get_session, + mock_listener_repo_get, + mock_listener_repo_update, + mock_amphora_repo_get, + mock_amphora_repo_update): + + mock_amphora_repo_get.return_value = _db_amphora_mock + mock_lb_get.return_value = _db_load_balancer_mock + amp_list_update_obj = amphora_driver_tasks.AmphoraIndexListenerUpdate() + amp_list_update_obj.execute(_LB_mock, 0, [_amphora_mock], + self.timeout_dict) + + mock_driver.update_amphora_listeners.assert_called_once_with( + _db_load_balancer_mock, _db_amphora_mock, self.timeout_dict) mock_driver.update_amphora_listeners.side_effect = Exception('boom') amp_list_update_obj.execute(_LB_mock, 0, - [_amphora_mock], timeout_dict) + [_amphora_mock], self.timeout_dict) mock_amphora_repo_update.assert_called_once_with( _session_mock, AMP_ID, status=constants.ERROR) @@ -155,6 +183,40 @@ class TestAmphoraDriverTasks(base.TestCase): self.assertEqual(2, repo.ListenerRepository.update.call_count) self.assertIsNone(amp) + @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') + @mock.patch('octavia.controller.worker.task_utils.TaskUtils.' + 'mark_listener_prov_status_error') + def test_amphora_index_listeners_reload( + self, mock_prov_status_error, mock_lb_repo_get, + mock_driver, mock_generate_uuid, mock_log, mock_get_session, + mock_listener_repo_get, mock_listener_repo_update, + mock_amphora_repo_get, mock_amphora_repo_update): + amphora_mock = mock.MagicMock() + listeners_reload_obj = ( + amphora_driver_tasks.AmphoraIndexListenersReload()) + mock_lb = mock.MagicMock() + mock_listener = mock.MagicMock() + mock_listener.id = '12345' + mock_amphora_repo_get.return_value = amphora_mock + mock_lb_repo_get.return_value = mock_lb + + # Test no listeners + mock_lb.listeners = None + listeners_reload_obj.execute(mock_lb, 0, None) + mock_driver.reload.assert_not_called() + + # Test with listeners + mock_driver.start.reset_mock() + mock_lb.listeners = [mock_listener] + listeners_reload_obj.execute(mock_lb, 0, [amphora_mock], + timeout_dict=self.timeout_dict) + mock_driver.reload.assert_called_once_with(mock_lb, amphora_mock, + self.timeout_dict) + # Test revert + mock_lb.listeners = [mock_listener] + listeners_reload_obj.revert(mock_lb) + mock_prov_status_error.assert_called_once_with('12345') + @mock.patch('octavia.controller.worker.task_utils.TaskUtils.' 'mark_listener_prov_status_error') @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') @@ -293,6 +355,12 @@ class TestAmphoraDriverTasks(base.TestCase): status=constants.ERROR) self.assertIsNone(amp) + # Test revert when this task failed + repo.AmphoraRepository.update.reset_mock() + amp = amphora_finalize_obj.revert( + failure.Failure.from_exception(Exception('boom')), _amphora_mock) + repo.AmphoraRepository.update.assert_not_called() + def test_amphora_post_network_plug(self, mock_driver, mock_generate_uuid, @@ -335,6 +403,12 @@ class TestAmphoraDriverTasks(base.TestCase): self.assertIsNone(amp) + # Test revert when this task failed + repo.AmphoraRepository.update.reset_mock() + amp = amphora_post_network_plug_obj.revert( + failure.Failure.from_exception(Exception('boom')), _amphora_mock) + repo.AmphoraRepository.update.assert_not_called() + @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') def test_amphorae_post_network_plug(self, mock_lb_get, mock_driver, @@ -365,6 +439,14 @@ class TestAmphoraDriverTasks(base.TestCase): assert_called_once_with(_db_amphora_mock, network_data_models.Port(**port_mock))) + # Test with no ports to plug + mock_driver.post_network_plug.reset_mock() + + _deltas_mock = {'0': [port_mock]} + + amphora_post_network_plug_obj.execute(_LB_mock, _deltas_mock) + mock_driver.post_network_plug.assert_not_called() + # Test revert amp = amphora_post_network_plug_obj.revert(None, _LB_mock, _deltas_mock) @@ -387,6 +469,13 @@ class TestAmphoraDriverTasks(base.TestCase): self.assertIsNone(amp) + # Test revert when this task failed + repo.AmphoraRepository.update.reset_mock() + amp = amphora_post_network_plug_obj.revert( + failure.Failure.from_exception(Exception('boom')), _amphora_mock, + None) + repo.AmphoraRepository.update.assert_not_called() + @mock.patch('octavia.db.repositories.LoadBalancerRepository.update') @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') def test_amphora_post_vip_plug(self, mock_lb_get, @@ -446,6 +535,13 @@ class TestAmphoraDriverTasks(base.TestCase): self.assertIsNone(amp) + # Test revert when this task failed + repo.AmphoraRepository.update.reset_mock() + amp = amphora_post_vip_plug_obj.revert( + failure.Failure.from_exception(Exception('boom')), _amphora_mock, + None) + repo.AmphoraRepository.update.assert_not_called() + @mock.patch('octavia.db.repositories.LoadBalancerRepository.update') @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') def test_amphorae_post_vip_plug(self, mock_lb_get, @@ -495,6 +591,13 @@ class TestAmphoraDriverTasks(base.TestCase): self.assertIsNone(amp) + # Test revert when this task failed + repo.AmphoraRepository.update.reset_mock() + amp = amphora_post_vip_plug_obj.revert( + failure.Failure.from_exception(Exception('boom')), _amphora_mock, + None) + repo.AmphoraRepository.update.assert_not_called() + def test_amphora_cert_upload(self, mock_driver, mock_generate_uuid, @@ -515,9 +618,7 @@ class TestAmphoraDriverTasks(base.TestCase): mock_driver.upload_cert_amp.assert_called_once_with( _db_amphora_mock, fer.decrypt(pem_file_mock.encode('utf-8'))) - @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') def test_amphora_update_vrrp_interface(self, - mock_lb_get, mock_driver, mock_generate_uuid, mock_log, @@ -526,45 +627,57 @@ class TestAmphoraDriverTasks(base.TestCase): mock_listener_repo_update, mock_amphora_repo_get, mock_amphora_repo_update): - _db_load_balancer_mock.amphorae = _amphorae_mock - mock_lb_get.return_value = _db_load_balancer_mock + FAKE_INTERFACE = 'fake0' + mock_amphora_repo_get.return_value = _db_amphora_mock + mock_driver.get_interface_from_ip.side_effect = [FAKE_INTERFACE, + Exception('boom')] timeout_dict = {constants.CONN_MAX_RETRIES: CONN_MAX_RETRIES, constants.CONN_RETRY_INTERVAL: CONN_RETRY_INTERVAL} amphora_update_vrrp_interface_obj = ( amphora_driver_tasks.AmphoraUpdateVRRPInterface()) - amphora_update_vrrp_interface_obj.execute(_LB_mock) + amphora_update_vrrp_interface_obj.execute(_amphora_mock, timeout_dict) mock_driver.get_interface_from_ip.assert_called_once_with( _db_amphora_mock, _db_amphora_mock.vrrp_ip, timeout_dict=timeout_dict) + mock_amphora_repo_update.assert_called_once_with( + _session_mock, _db_amphora_mock.id, vrrp_interface=FAKE_INTERFACE) - # Test revert - mock_driver.reset_mock() - _db_load_balancer_mock.amphorae = _amphorae_mock - - amphora_update_vrrp_interface_obj.revert("BADRESULT", _LB_mock) - mock_amphora_repo_update.assert_called_with(_session_mock, - _db_amphora_mock.id, - vrrp_interface=None) - - mock_driver.reset_mock() + # Test with an exception mock_amphora_repo_update.reset_mock() - _db_load_balancer_mock.amphorae = _amphorae_mock + amphora_update_vrrp_interface_obj.execute(_amphora_mock, timeout_dict) + mock_amphora_repo_update.assert_called_once_with( + _session_mock, _db_amphora_mock.id, status=constants.ERROR) - failure_obj = failure.Failure.from_exception(Exception("TESTEXCEPT")) - amphora_update_vrrp_interface_obj.revert(failure_obj, _LB_mock) - self.assertFalse(mock_amphora_repo_update.called) + def test_amphora_index_update_vrrp_interface( + self, mock_driver, mock_generate_uuid, mock_log, mock_get_session, + mock_listener_repo_get, mock_listener_repo_update, + mock_amphora_repo_get, mock_amphora_repo_update): + mock_amphora_repo_get.return_value = _db_amphora_mock + FAKE_INTERFACE = 'fake0' + mock_driver.get_interface_from_ip.side_effect = [FAKE_INTERFACE, + Exception('boom')] - # Test revert with exception - mock_driver.reset_mock() + timeout_dict = {constants.CONN_MAX_RETRIES: CONN_MAX_RETRIES, + constants.CONN_RETRY_INTERVAL: CONN_RETRY_INTERVAL} + + amphora_update_vrrp_interface_obj = ( + amphora_driver_tasks.AmphoraIndexUpdateVRRPInterface()) + amphora_update_vrrp_interface_obj.execute( + 0, [_amphora_mock], timeout_dict) + mock_driver.get_interface_from_ip.assert_called_once_with( + _db_amphora_mock, _db_amphora_mock.vrrp_ip, + timeout_dict=timeout_dict) + mock_amphora_repo_update.assert_called_once_with( + _session_mock, _db_amphora_mock.id, vrrp_interface=FAKE_INTERFACE) + + # Test with an exception mock_amphora_repo_update.reset_mock() - mock_amphora_repo_update.side_effect = Exception('fail') - - amphora_update_vrrp_interface_obj.revert("BADRESULT", _LB_mock) - mock_amphora_repo_update.assert_called_with(_session_mock, - _db_amphora_mock.id, - vrrp_interface=None) + amphora_update_vrrp_interface_obj.execute( + 0, [_amphora_mock], timeout_dict) + mock_amphora_repo_update.assert_called_once_with( + _session_mock, _db_amphora_mock.id, status=constants.ERROR) @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') def test_amphora_vrrp_update(self, @@ -578,34 +691,59 @@ class TestAmphoraDriverTasks(base.TestCase): mock_amphora_repo_get, mock_amphora_repo_update): amphorae_network_config = mock.MagicMock() + mock_driver.update_vrrp_conf.side_effect = [mock.DEFAULT, + Exception('boom')] mock_lb_get.return_value = _db_load_balancer_mock + mock_amphora_repo_get.return_value = _db_amphora_mock amphora_vrrp_update_obj = ( amphora_driver_tasks.AmphoraVRRPUpdate()) - amphora_vrrp_update_obj.execute(_LB_mock, amphorae_network_config) + amphora_vrrp_update_obj.execute(LB_ID, amphorae_network_config, + _amphora_mock, 'fakeint0') mock_driver.update_vrrp_conf.assert_called_once_with( - _db_load_balancer_mock, amphorae_network_config) + _db_load_balancer_mock, amphorae_network_config, + _db_amphora_mock, None) + + # Test with an exception + mock_amphora_repo_update.reset_mock() + amphora_vrrp_update_obj.execute(LB_ID, amphorae_network_config, + _amphora_mock, 'fakeint0') + mock_amphora_repo_update.assert_called_once_with( + _session_mock, _db_amphora_mock.id, status=constants.ERROR) @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') - def test_amphora_vrrp_stop(self, - mock_lb_get, - mock_driver, - mock_generate_uuid, - mock_log, - mock_get_session, - mock_listener_repo_get, - mock_listener_repo_update, - mock_amphora_repo_get, - mock_amphora_repo_update): - amphora_vrrp_stop_obj = ( - amphora_driver_tasks.AmphoraVRRPStop()) + def test_amphora_index_vrrp_update(self, + mock_lb_get, + mock_driver, + mock_generate_uuid, + mock_log, + mock_get_session, + mock_listener_repo_get, + mock_listener_repo_update, + mock_amphora_repo_get, + mock_amphora_repo_update): + amphorae_network_config = mock.MagicMock() + mock_driver.update_vrrp_conf.side_effect = [mock.DEFAULT, + Exception('boom')] mock_lb_get.return_value = _db_load_balancer_mock - amphora_vrrp_stop_obj.execute(_LB_mock) - mock_driver.stop_vrrp_service.assert_called_once_with( - _db_load_balancer_mock) + mock_amphora_repo_get.return_value = _db_amphora_mock + amphora_vrrp_update_obj = ( + amphora_driver_tasks.AmphoraIndexVRRPUpdate()) + + amphora_vrrp_update_obj.execute(LB_ID, amphorae_network_config, + 0, [_amphora_mock], 'fakeint0', + timeout_dict=self.timeout_dict) + mock_driver.update_vrrp_conf.assert_called_once_with( + _db_load_balancer_mock, amphorae_network_config, _db_amphora_mock, + self.timeout_dict) + + # Test with an exception + mock_amphora_repo_update.reset_mock() + amphora_vrrp_update_obj.execute(LB_ID, amphorae_network_config, + 0, [_amphora_mock], 'fakeint0') + mock_amphora_repo_update.assert_called_once_with( + _session_mock, _db_amphora_mock.id, status=constants.ERROR) - @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') def test_amphora_vrrp_start(self, - mock_lb_get, mock_driver, mock_generate_uuid, mock_log, @@ -614,12 +752,30 @@ class TestAmphoraDriverTasks(base.TestCase): mock_listener_repo_update, mock_amphora_repo_get, mock_amphora_repo_update): + mock_amphora_repo_get.return_value = _db_amphora_mock amphora_vrrp_start_obj = ( amphora_driver_tasks.AmphoraVRRPStart()) - mock_lb_get.return_value = _db_load_balancer_mock - amphora_vrrp_start_obj.execute(_LB_mock) + amphora_vrrp_start_obj.execute(_amphora_mock, + timeout_dict=self.timeout_dict) mock_driver.start_vrrp_service.assert_called_once_with( - _db_load_balancer_mock) + _db_amphora_mock, self.timeout_dict) + + def test_amphora_index_vrrp_start(self, + mock_driver, + mock_generate_uuid, + mock_log, + mock_get_session, + mock_listener_repo_get, + mock_listener_repo_update, + mock_amphora_repo_get, + mock_amphora_repo_update): + mock_amphora_repo_get.return_value = _db_amphora_mock + amphora_vrrp_start_obj = ( + amphora_driver_tasks.AmphoraIndexVRRPStart()) + amphora_vrrp_start_obj.execute(0, [_amphora_mock], + timeout_dict=self.timeout_dict) + mock_driver.start_vrrp_service.assert_called_once_with( + _db_amphora_mock, self.timeout_dict) def test_amphora_compute_connectivity_wait(self, mock_driver, diff --git a/octavia/tests/unit/controller/worker/v2/tasks/test_compute_tasks.py b/octavia/tests/unit/controller/worker/v2/tasks/test_compute_tasks.py index 7ea980ddfa..2febd35782 100644 --- a/octavia/tests/unit/controller/worker/v2/tasks/test_compute_tasks.py +++ b/octavia/tests/unit/controller/worker/v2/tasks/test_compute_tasks.py @@ -18,6 +18,7 @@ from cryptography import fernet from oslo_config import cfg from oslo_config import fixture as oslo_fixture from oslo_utils import uuidutils +import tenacity from octavia.common import constants from octavia.common import exceptions @@ -178,7 +179,8 @@ class TestComputeTasks(base.TestCase): mock_driver.build.return_value = COMPUTE_ID # Test execute() - compute_id = createcompute.execute(_db_amphora_mock.id, ports=[_port]) + compute_id = createcompute.execute(_db_amphora_mock.id, ports=[_port], + server_group_id=None) # Validate that the build method was called properly mock_driver.build.assert_called_once_with( @@ -508,20 +510,54 @@ class TestComputeTasks(base.TestCase): def test_delete_amphorae_on_load_balancer(self, mock_lb_get, mock_session, mock_driver): + mock_driver.delete.side_effect = [mock.DEFAULT, + exceptions.OctaviaException('boom')] delete_amps = compute_tasks.DeleteAmphoraeOnLoadBalancer() + mock_lb_get.return_value = _db_load_balancer_mock delete_amps.execute(_load_balancer_mock) mock_driver.delete.assert_called_once_with(COMPUTE_ID) + # Test compute driver exception is raised + self.assertRaises(exceptions.OctaviaException, delete_amps.execute, + _load_balancer_mock) + @mock.patch('stevedore.driver.DriverManager.driver') def test_compute_delete(self, mock_driver): + mock_driver.delete.side_effect = [ + mock.DEFAULT, exceptions.OctaviaException('boom'), + mock.DEFAULT, exceptions.OctaviaException('boom'), + exceptions.OctaviaException('boom'), + exceptions.OctaviaException('boom'), + exceptions.OctaviaException('boom')] delete_compute = compute_tasks.ComputeDelete() + + # Limit the retry attempts for the test run to save time + delete_compute.execute.retry.stop = tenacity.stop_after_attempt(2) + delete_compute.execute(_amphora_mock) mock_driver.delete.assert_called_once_with(COMPUTE_ID) + # Test retry after a compute exception + mock_driver.reset_mock() + delete_compute.execute(_amphora_mock) + mock_driver.delete.assert_has_calls([mock.call(COMPUTE_ID), + mock.call(COMPUTE_ID)]) + + # Test passive failure + mock_driver.reset_mock() + delete_compute.execute(_amphora_mock, passive_failure=True) + mock_driver.delete.assert_has_calls([mock.call(COMPUTE_ID), + mock.call(COMPUTE_ID)]) + + # Test non-passive failure + mock_driver.reset_mock() + self.assertRaises(exceptions.OctaviaException, delete_compute.execute, + _amphora_mock, passive_failure=False) + @mock.patch('stevedore.driver.DriverManager.driver') def test_nova_server_group_create(self, mock_driver): nova_sever_group_obj = compute_tasks.NovaServerGroupCreate() @@ -567,3 +603,32 @@ class TestComputeTasks(base.TestCase): sg_id = None nova_sever_group_obj.execute(sg_id) self.assertFalse(mock_driver.delete_server_group.called, sg_id) + + @mock.patch('stevedore.driver.DriverManager.driver') + def test_attach_port(self, mock_driver): + COMPUTE_ID = uuidutils.generate_uuid() + PORT_ID = uuidutils.generate_uuid() + amphora_dict = {constants.COMPUTE_ID: COMPUTE_ID} + port_dict = {constants.ID: PORT_ID} + + attach_port_obj = compute_tasks.AttachPort() + + # Test execute + attach_port_obj.execute(amphora_dict, port_dict) + + mock_driver.attach_network_or_port.assert_called_once_with( + COMPUTE_ID, port_id=PORT_ID) + + # Test revert + mock_driver.reset_mock() + + attach_port_obj.revert(amphora_dict, port_dict) + + mock_driver.detach_port.assert_called_once_with(COMPUTE_ID, PORT_ID) + + # Test rever exception + mock_driver.reset_mock() + mock_driver.detach_port.side_effect = [Exception('boom')] + + # should not raise + attach_port_obj.revert(amphora_dict, port_dict) diff --git a/octavia/tests/unit/controller/worker/v2/tasks/test_database_tasks.py b/octavia/tests/unit/controller/worker/v2/tasks/test_database_tasks.py index cdbffad15d..5641b572dc 100644 --- a/octavia/tests/unit/controller/worker/v2/tasks/test_database_tasks.py +++ b/octavia/tests/unit/controller/worker/v2/tasks/test_database_tasks.py @@ -46,10 +46,10 @@ HA_PORT_ID = uuidutils.generate_uuid() L7POLICY_ID = uuidutils.generate_uuid() L7RULE_ID = uuidutils.generate_uuid() VIP_IP = '192.0.5.2' +VRRP_ID = 1 VRRP_IP = '192.0.5.3' HA_IP = '192.0.5.4' AMP_ROLE = 'FAKE_ROLE' -VRRP_ID = random.randrange(255) VRRP_PRIORITY = random.randrange(100) CACHED_ZONE = 'zone1' IMAGE_ID = uuidutils.generate_uuid() @@ -542,8 +542,15 @@ class TestDatabaseTasks(base.TestCase): mock_amphora_repo_update, mock_amphora_repo_delete): + amphora_dict = {constants.ID: AMP_ID} + vip_dict = {constants.IP_ADDRESS: HA_IP, + constants.PORT_ID: HA_PORT_ID} + fixed_ips = [{constants.IP_ADDRESS: VRRP_IP}] + base_port_dict = {constants.ID: VRRP_PORT_ID, + constants.FIXED_IPS: fixed_ips} + update_amp_fo_details = database_tasks.UpdateAmpFailoverDetails() - update_amp_fo_details.execute(self.amphora, self.amphora) + update_amp_fo_details.execute(amphora_dict, vip_dict, base_port_dict) mock_amphora_repo_update.assert_called_once_with( 'TEST', @@ -1337,16 +1344,16 @@ class TestDatabaseTasks(base.TestCase): mock_amphora_repo_update, mock_amphora_repo_delete): unused_pool = data_models.Pool(id='unused_pool') - members1 = [{constants.MEMBER_ID: 'member1'}, - {constants.MEMBER_ID: 'member2'}] + members1 = [data_models.Member(id='member1'), + data_models.Member(id='member2')] health_monitor = data_models.HealthMonitor(id='hm1') default_pool = data_models.Pool(id='default_pool', members=members1, health_monitor=health_monitor) listener1 = data_models.Listener(id='listener1', default_pool=default_pool) - members2 = [{constants.MEMBER_ID: 'member3'}, - {constants.MEMBER_ID: 'member4'}] + members2 = [data_models.Member(id='member3'), + data_models.Member(id='member4')] redirect_pool = data_models.Pool(id='redirect_pool', members=members2) l7rules = [data_models.L7Rule(id='rule1')] @@ -1954,6 +1961,22 @@ class TestDatabaseTasks(base.TestCase): result = get_vip_from_lb_obj.execute(self.loadbalancer_mock) self.assertEqual(_vip_mock.to_dict(), result) + @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') + def test_get_loadbalancer(self, mock_lb_get, mock_generate_uuid, mock_LOG, + mock_get_session, mock_loadbalancer_repo_update, + mock_listener_repo_update, + mock_amphora_repo_update, + mock_amphora_repo_delete): + LB_ID = uuidutils.generate_uuid() + get_loadbalancer_obj = database_tasks.GetLoadBalancer() + + mock_lb_get.return_value = _db_loadbalancer_mock + + result = get_loadbalancer_obj.execute(LB_ID) + + self.assertEqual(self.loadbalancer_mock, result) + mock_lb_get.assert_called_once_with('TEST', id=LB_ID) + @mock.patch('octavia.db.repositories.VRRPGroupRepository.create') def test_create_vrrp_group_for_lb(self, mock_vrrp_group_create, @@ -1968,7 +1991,7 @@ class TestDatabaseTasks(base.TestCase): mock_get_session.side_effect = ['TEST', odb_exceptions.DBDuplicateEntry] create_vrrp_group = database_tasks.CreateVRRPGroupForLB() - create_vrrp_group.execute(self.loadbalancer_mock) + create_vrrp_group.execute(LB_ID) mock_vrrp_group_create.assert_called_once_with( 'TEST', load_balancer_id=LB_ID, vrrp_group_name=LB_ID.replace('-', ''), diff --git a/octavia/tests/unit/controller/worker/v2/tasks/test_network_tasks.py b/octavia/tests/unit/controller/worker/v2/tasks/test_network_tasks.py index 858d95bc4e..b9e3fa32cd 100644 --- a/octavia/tests/unit/controller/worker/v2/tasks/test_network_tasks.py +++ b/octavia/tests/unit/controller/worker/v2/tasks/test_network_tasks.py @@ -18,10 +18,12 @@ from oslo_config import cfg from oslo_config import fixture as oslo_fixture from oslo_utils import uuidutils from taskflow.types import failure +import tenacity from octavia.api.drivers import utils as provider_utils from octavia.common import constants from octavia.common import data_models as o_data_models +from octavia.common import exceptions from octavia.controller.worker.v2.tasks import network_tasks from octavia.network import base as net_base from octavia.network import data_models @@ -81,6 +83,11 @@ class TestNetworkTasks(base.TestCase): self.db_amphora_mock.id = AMPHORA_ID self.db_amphora_mock.compute_id = COMPUTE_ID self.db_amphora_mock.status = constants.AMPHORA_ALLOCATED + self.boot_net_id = NETWORK_ID + conf = self.useFixture(oslo_fixture.Config(cfg.CONF)) + conf.config(group="controller_worker", + amp_boot_network_list=[self.boot_net_id]) + conf.config(group="networking", max_retries=1) self.amphora_mock = {constants.ID: AMPHORA_ID, constants.COMPUTE_ID: COMPUTE_ID, constants.LB_NETWORK_IP: IP_ADDRESS, @@ -94,10 +101,85 @@ class TestNetworkTasks(base.TestCase): } conf = oslo_fixture.Config(cfg.CONF) - conf.config(group="controller_worker", amp_boot_network_list=['netid']) + conf.config(group="controller_worker", + amp_boot_network_list=[self.boot_net_id]) super().setUp() + @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') + @mock.patch('octavia.db.api.get_session', return_value=_session_mock) + def test_calculate_amphora_delta(self, mock_get_session, mock_lb_repo_get, + mock_get_net_driver): + LB_ID = uuidutils.generate_uuid() + DELETE_NETWORK_ID = uuidutils.generate_uuid() + MEMBER_NETWORK_ID = uuidutils.generate_uuid() + MEMBER_SUBNET_ID = uuidutils.generate_uuid() + VRRP_PORT_ID = uuidutils.generate_uuid() + mock_driver = mock.MagicMock() + mock_get_net_driver.return_value = mock_driver + member_mock = mock.MagicMock() + member_mock.subnet_id = MEMBER_SUBNET_ID + pool_mock = mock.MagicMock() + pool_mock.members = [member_mock] + lb_mock = mock.MagicMock() + lb_mock.pools = [pool_mock] + lb_dict = {constants.LOADBALANCER_ID: LB_ID} + amphora_dict = {constants.ID: AMPHORA_ID, + constants.COMPUTE_ID: COMPUTE_ID, + constants.VRRP_PORT_ID: VRRP_PORT_ID} + vrrp_port_mock = mock.MagicMock() + vrrp_port_mock.network_id = self.boot_net_id + vrrp_port_dict = {constants.NETWORK_ID: self.boot_net_id} + mock_subnet = mock.MagicMock() + mock_subnet.network_id = MEMBER_NETWORK_ID + nic1_delete_mock = mock.MagicMock() + nic1_delete_mock.network_id = DELETE_NETWORK_ID + nic2_keep_mock = mock.MagicMock() + nic2_keep_mock.network_id = self.boot_net_id + + mock_lb_repo_get.return_value = lb_mock + mock_driver.get_port.return_value = vrrp_port_mock + mock_driver.get_subnet.return_value = mock_subnet + mock_driver.get_plugged_networks.return_value = [nic1_delete_mock, + nic2_keep_mock] + + calc_amp_delta = network_tasks.CalculateAmphoraDelta() + + # Test vrrp_port_id is None + result = calc_amp_delta.execute(lb_dict, amphora_dict, {}) + + self.assertEqual(AMPHORA_ID, result[constants.AMPHORA_ID]) + self.assertEqual(COMPUTE_ID, result[constants.COMPUTE_ID]) + self.assertEqual(1, len(result[constants.ADD_NICS])) + self.assertEqual(MEMBER_NETWORK_ID, + result[constants.ADD_NICS][0][constants.NETWORK_ID]) + self.assertEqual(1, len(result[constants.DELETE_NICS])) + self.assertEqual( + DELETE_NETWORK_ID, + result[constants.DELETE_NICS][0][constants.NETWORK_ID]) + mock_driver.get_port.assert_called_once_with(VRRP_PORT_ID) + mock_driver.get_subnet.assert_called_once_with(MEMBER_SUBNET_ID) + mock_driver.get_plugged_networks.assert_called_once_with(COMPUTE_ID) + + # Test with vrrp_port_id + mock_driver.reset_mock() + + result = calc_amp_delta.execute(lb_dict, amphora_dict, {}, + vrrp_port=vrrp_port_dict) + + self.assertEqual(AMPHORA_ID, result[constants.AMPHORA_ID]) + self.assertEqual(COMPUTE_ID, result[constants.COMPUTE_ID]) + self.assertEqual(1, len(result[constants.ADD_NICS])) + self.assertEqual(MEMBER_NETWORK_ID, + result[constants.ADD_NICS][0][constants.NETWORK_ID]) + self.assertEqual(1, len(result[constants.DELETE_NICS])) + self.assertEqual( + DELETE_NETWORK_ID, + result[constants.DELETE_NICS][0][constants.NETWORK_ID]) + mock_driver.get_port.assert_not_called() + mock_driver.get_subnet.assert_called_once_with(MEMBER_SUBNET_ID) + mock_driver.get_plugged_networks.assert_called_once_with(COMPUTE_ID) + @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') @mock.patch('octavia.db.api.get_session', return_value=_session_mock) def test_calculate_delta(self, mock_get_session, mock_get_lb, @@ -110,9 +192,9 @@ class TestNetworkTasks(base.TestCase): constants.VRRP_PORT_ID: PORT_ID} mock_get_net_driver.return_value = mock_driver mock_driver.get_plugged_networks.return_value = [ - data_models.Interface(network_id='netid')] + data_models.Interface(network_id=self.boot_net_id)] mock_driver.get_port.return_value = data_models.Port( - network_id='netid') + network_id=self.boot_net_id) EMPTY = {} empty_deltas = {self.db_amphora_mock.id: data_models.Delta( amphora_id=AMPHORA_ID, @@ -179,7 +261,7 @@ class TestNetworkTasks(base.TestCase): pool_mock.members = [member_mock] mock_driver.get_plugged_networks.return_value = [ data_models.Interface(network_id=3), - data_models.Interface(network_id='netid')] + data_models.Interface(network_id=self.boot_net_id)] self.assertEqual(empty_deltas, calc_delta.execute(self.load_balancer_mock, {})) @@ -192,7 +274,7 @@ class TestNetworkTasks(base.TestCase): pool_mock.members = [member_mock] mock_driver.get_plugged_networks.return_value = [ data_models.Interface(network_id=2), - data_models.Interface(network_id='netid')] + data_models.Interface(network_id=self.boot_net_id)] ndm = data_models.Delta(amphora_id=self.db_amphora_mock.id, compute_id=self.db_amphora_mock.compute_id, @@ -210,7 +292,7 @@ class TestNetworkTasks(base.TestCase): pool_mock.members = [] mock_driver.get_plugged_networks.return_value = [ data_models.Interface(network_id=2), - data_models.Interface(network_id='netid') + data_models.Interface(network_id=self.boot_net_id) ] ndm = data_models.Delta(amphora_id=self.db_amphora_mock.id, @@ -648,6 +730,7 @@ class TestNetworkTasks(base.TestCase): t_constants.MOCK_QOS_POLICY_ID1, mock.ANY) self.assertEqual(2, mock_driver.apply_qos_on_port.call_count) + mock_get_lb.return_value = null_qos_lb mock_driver.reset_mock() update_dict = {} net.execute(null_qos_lb_dict, update_dict=update_dict) @@ -685,7 +768,7 @@ class TestNetworkTasks(base.TestCase): net.revert(None, pr_tm_dict, update_dict=update_dict) mock_driver.apply_qos_on_port.assert_called_with( t_constants.MOCK_QOS_POLICY_ID2, mock.ANY) - self.assertEqual(2, mock_driver.apply_qos_on_port.call_count) + self.assertEqual(1, mock_driver.apply_qos_on_port.call_count) @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') @mock.patch('octavia.db.api.get_session', return_value=_session_mock) @@ -770,6 +853,28 @@ class TestNetworkTasks(base.TestCase): net_task.execute(listener) mock_driver.update_vip.assert_called_once_with(lb, for_delete=True) + @mock.patch('octavia.db.api.get_session', return_value='TEST') + @mock.patch('octavia.db.repositories.AmphoraRepository.get') + @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') + def test_get_amphora_network_configs_by_id( + self, mock_lb_get, mock_amp_get, + mock_get_session, mock_get_net_driver): + LB_ID = uuidutils.generate_uuid() + AMP_ID = uuidutils.generate_uuid() + mock_driver = mock.MagicMock() + mock_get_net_driver.return_value = mock_driver + mock_amp_get.return_value = 'mock amphora' + mock_lb_get.return_value = 'mock load balancer' + + net_task = network_tasks.GetAmphoraNetworkConfigsByID() + + net_task.execute(LB_ID, AMP_ID) + + mock_driver.get_network_configs.assert_called_once_with( + 'mock load balancer', amphora='mock amphora') + mock_amp_get.assert_called_once_with('TEST', id=AMP_ID) + mock_lb_get.assert_called_once_with('TEST', id=LB_ID) + @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') @mock.patch('octavia.db.api.get_session', return_value=_session_mock) def test_get_amphorae_network_configs(self, mock_session, mock_lb_get, @@ -854,49 +959,6 @@ class TestNetworkTasks(base.TestCase): mock_driver.plug_port.assert_any_call(self.db_amphora_mock, port1) mock_driver.plug_port.assert_any_call(self.db_amphora_mock, port2) - @mock.patch('octavia.db.repositories.AmphoraRepository.get') - @mock.patch('octavia.db.api.get_session', return_value=mock.MagicMock()) - def test_plug_vip_port(self, mock_session, mock_get, mock_get_net_driver): - mock_driver = mock.MagicMock() - mock_get.return_value = self.db_amphora_mock - mock_get_net_driver.return_value = mock_driver - vrrp_port = mock.MagicMock() - - amphorae_network_config = mock.MagicMock() - mock_driver.get_port.return_value = vrrp_port - - plugvipport = network_tasks.PlugVIPPort() - amp = {constants.ID: AMPHORA_ID, - constants.COMPUTE_ID: '1234'} - plugvipport.execute(amp, amphorae_network_config) - mock_driver.plug_port.assert_called_once_with(self.db_amphora_mock, - vrrp_port) - dict_amp_config = { - AMPHORA_ID: {constants.VRRP_PORT: {constants.ID: 5555}} - } - - # test revert - plugvipport.revert(None, amp, dict_amp_config) - mock_driver.unplug_port.assert_called_with(self.db_amphora_mock, - vrrp_port) - - @mock.patch('octavia.db.repositories.AmphoraRepository.get') - @mock.patch('octavia.db.api.get_session', return_value=mock.MagicMock()) - def test_wait_for_port_detach(self, mock_session, mock_get, - mock_get_net_driver): - mock_driver = mock.MagicMock() - mock_get.return_value = self.db_amphora_mock - mock_get_net_driver.return_value = mock_driver - - amphora = {constants.ID: AMPHORA_ID, - constants.LB_NETWORK_IP: IP_ADDRESS} - - waitforportdetach = network_tasks.WaitForPortDetach() - waitforportdetach.execute(amphora) - - mock_driver.wait_for_port_detach.assert_called_once_with( - self.db_amphora_mock) - @mock.patch('octavia.db.repositories.LoadBalancerRepository.get') @mock.patch('octavia.db.api.get_session', return_value=_session_mock) def test_update_vip_sg(self, mock_session, mock_lb_get, @@ -928,7 +990,7 @@ class TestNetworkTasks(base.TestCase): mock_lb_get.return_value = LB mock_get.return_value = self.db_amphora_mock mock_get_net_driver.return_value = mock_driver - net = network_tasks.PlugVIPAmpphora() + net = network_tasks.PlugVIPAmphora() mockSubnet = mock_driver.get_subnet() net.execute(self.load_balancer_mock, amphora, mockSubnet) mock_driver.plug_aap_port.assert_called_once_with( @@ -943,7 +1005,7 @@ class TestNetworkTasks(base.TestCase): mock_lb_get.return_value = LB mock_get.return_value = self.db_amphora_mock mock_get_net_driver.return_value = mock_driver - net = network_tasks.PlugVIPAmpphora() + net = network_tasks.PlugVIPAmphora() mockSubnet = mock.MagicMock() amphora = {constants.ID: AMPHORA_ID, constants.LB_NETWORK_IP: IP_ADDRESS} @@ -951,3 +1013,273 @@ class TestNetworkTasks(base.TestCase): amphora, mockSubnet) mock_driver.unplug_aap_port.assert_called_once_with( LB.vip, self.db_amphora_mock, mockSubnet) + + @mock.patch('octavia.controller.worker.v2.tasks.network_tasks.DeletePort.' + 'update_progress') + def test_delete_port(self, mock_update_progress, mock_get_net_driver): + PORT_ID = uuidutils.generate_uuid() + mock_driver = mock.MagicMock() + mock_get_net_driver.return_value = mock_driver + mock_driver.delete_port.side_effect = [ + mock.DEFAULT, exceptions.OctaviaException('boom'), mock.DEFAULT, + exceptions.OctaviaException('boom'), + exceptions.OctaviaException('boom'), + exceptions.OctaviaException('boom'), + exceptions.OctaviaException('boom'), + exceptions.OctaviaException('boom'), + exceptions.OctaviaException('boom')] + mock_driver.admin_down_port.side_effect = [ + mock.DEFAULT, exceptions.OctaviaException('boom')] + + net_task = network_tasks.DeletePort() + + # Limit the retry attempts for the test run to save time + net_task.execute.retry.stop = tenacity.stop_after_attempt(2) + + # Test port ID is None (no-op) + net_task.execute(None) + + mock_update_progress.assert_not_called() + mock_driver.delete_port.assert_not_called() + + # Test successful delete + mock_update_progress.reset_mock() + mock_driver.reset_mock() + + net_task.execute(PORT_ID) + + mock_update_progress.assert_called_once_with(0.5) + mock_driver.delete_port.assert_called_once_with(PORT_ID) + + # Test exception and successful retry + mock_update_progress.reset_mock() + mock_driver.reset_mock() + + net_task.execute(PORT_ID) + + mock_update_progress.assert_has_calls([mock.call(0.5), mock.call(1.0)]) + mock_driver.delete_port.assert_has_calls([mock.call(PORT_ID), + mock.call(PORT_ID)]) + + # Test passive failure + mock_update_progress.reset_mock() + mock_driver.reset_mock() + + net_task.execute(PORT_ID, passive_failure=True) + + mock_update_progress.assert_has_calls([mock.call(0.5), mock.call(1.0)]) + mock_driver.delete_port.assert_has_calls([mock.call(PORT_ID), + mock.call(PORT_ID)]) + mock_driver.admin_down_port.assert_called_once_with(PORT_ID) + + # Test passive failure admin down failure + mock_update_progress.reset_mock() + mock_driver.reset_mock() + mock_driver.admin_down_port.reset_mock() + + net_task.execute(PORT_ID, passive_failure=True) + + mock_update_progress.assert_has_calls([mock.call(0.5), mock.call(1.0)]) + mock_driver.delete_port.assert_has_calls([mock.call(PORT_ID), + mock.call(PORT_ID)]) + mock_driver.admin_down_port.assert_called_once_with(PORT_ID) + + # Test non-passive failure + mock_update_progress.reset_mock() + mock_driver.reset_mock() + mock_driver.admin_down_port.reset_mock() + + mock_driver.admin_down_port.side_effect = [ + exceptions.OctaviaException('boom')] + + self.assertRaises(exceptions.OctaviaException, net_task.execute, + PORT_ID) + + mock_update_progress.assert_has_calls([mock.call(0.5), mock.call(1.0)]) + mock_driver.delete_port.assert_has_calls([mock.call(PORT_ID), + mock.call(PORT_ID)]) + mock_driver.admin_down_port.assert_not_called() + + def test_create_vip_base_port(self, mock_get_net_driver): + AMP_ID = uuidutils.generate_uuid() + PORT_ID = uuidutils.generate_uuid() + VIP_NETWORK_ID = uuidutils.generate_uuid() + VIP_QOS_ID = uuidutils.generate_uuid() + VIP_SG_ID = uuidutils.generate_uuid() + VIP_SUBNET_ID = uuidutils.generate_uuid() + VIP_IP_ADDRESS = '203.0.113.81' + mock_driver = mock.MagicMock() + mock_get_net_driver.return_value = mock_driver + vip_dict = {constants.IP_ADDRESS: VIP_IP_ADDRESS, + constants.NETWORK_ID: VIP_NETWORK_ID, + constants.QOS_POLICY_ID: VIP_QOS_ID, + constants.SUBNET_ID: VIP_SUBNET_ID} + port_mock = mock.MagicMock() + port_mock.id = PORT_ID + + mock_driver.create_port.side_effect = [ + port_mock, exceptions.OctaviaException('boom'), + exceptions.OctaviaException('boom'), + exceptions.OctaviaException('boom')] + mock_driver.delete_port.side_effect = [mock.DEFAULT, Exception('boom')] + + net_task = network_tasks.CreateVIPBasePort() + + # Limit the retry attempts for the test run to save time + net_task.execute.retry.stop = tenacity.stop_after_attempt(2) + + # Test execute + result = net_task.execute(vip_dict, VIP_SG_ID, AMP_ID) + + self.assertEqual(port_mock.to_dict(), result) + mock_driver.create_port.assert_called_once_with( + VIP_NETWORK_ID, name=constants.AMP_BASE_PORT_PREFIX + AMP_ID, + fixed_ips=[{constants.SUBNET_ID: VIP_SUBNET_ID}], + secondary_ips=[VIP_IP_ADDRESS], security_group_ids=[VIP_SG_ID], + qos_policy_id=VIP_QOS_ID) + + # Test execute exception + mock_driver.reset_mock() + + self.assertRaises(exceptions.OctaviaException, net_task.execute, + vip_dict, None, AMP_ID) + + # Test revert when this task failed + mock_driver.reset_mock() + + net_task.revert(failure.Failure.from_exception(Exception('boom')), + vip_dict, VIP_SG_ID, AMP_ID) + + mock_driver.delete_port.assert_not_called() + + # Test revert + mock_driver.reset_mock() + + net_task.revert([port_mock], vip_dict, VIP_SG_ID, AMP_ID) + + mock_driver.delete_port.assert_called_once_with(PORT_ID) + + # Test revert exception + mock_driver.reset_mock() + + net_task.revert([port_mock], vip_dict, VIP_SG_ID, AMP_ID) + + mock_driver.delete_port.assert_called_once_with(PORT_ID) + + @mock.patch('time.sleep') + def test_admin_down_port(self, mock_sleep, mock_get_net_driver): + PORT_ID = uuidutils.generate_uuid() + mock_driver = mock.MagicMock() + mock_get_net_driver.return_value = mock_driver + port_down_mock = mock.MagicMock() + port_down_mock.status = constants.DOWN + port_up_mock = mock.MagicMock() + port_up_mock.status = constants.UP + mock_driver.set_port_admin_state_up.side_effect = [ + mock.DEFAULT, net_base.PortNotFound, mock.DEFAULT, mock.DEFAULT, + Exception('boom')] + mock_driver.get_port.side_effect = [port_down_mock, port_up_mock] + + net_task = network_tasks.AdminDownPort() + + # Test execute + net_task.execute(PORT_ID) + + mock_driver.set_port_admin_state_up.assert_called_once_with(PORT_ID, + False) + mock_driver.get_port.assert_called_once_with(PORT_ID) + + # Test passive fail on port not found + mock_driver.reset_mock() + + net_task.execute(PORT_ID) + + mock_driver.set_port_admin_state_up.assert_called_once_with(PORT_ID, + False) + mock_driver.get_port.assert_not_called() + + # Test passive fail on port stays up + mock_driver.reset_mock() + + net_task.execute(PORT_ID) + + mock_driver.set_port_admin_state_up.assert_called_once_with(PORT_ID, + False) + mock_driver.get_port.assert_called_once_with(PORT_ID) + + # Test revert when this task failed + mock_driver.reset_mock() + + net_task.revert(failure.Failure.from_exception(Exception('boom')), + PORT_ID) + + mock_driver.set_port_admin_state_up.assert_not_called() + + # Test revert + mock_driver.reset_mock() + + net_task.revert(None, PORT_ID) + + mock_driver.set_port_admin_state_up.assert_called_once_with(PORT_ID, + True) + + # Test revert exception passive failure + mock_driver.reset_mock() + + net_task.revert(None, PORT_ID) + + mock_driver.set_port_admin_state_up.assert_called_once_with(PORT_ID, + True) + + @mock.patch('octavia.common.utils.get_vip_security_group_name') + def test_get_vip_security_group_id(self, mock_get_sg_name, + mock_get_net_driver): + LB_ID = uuidutils.generate_uuid() + SG_ID = uuidutils.generate_uuid() + SG_NAME = 'fake_SG_name' + mock_driver = mock.MagicMock() + mock_get_net_driver.return_value = mock_driver + mock_get_sg_name.return_value = SG_NAME + sg_mock = mock.MagicMock() + sg_mock.id = SG_ID + mock_driver.get_security_group.side_effect = [ + sg_mock, None, net_base.SecurityGroupNotFound, + net_base.SecurityGroupNotFound] + + net_task = network_tasks.GetVIPSecurityGroupID() + + # Test execute + result = net_task.execute(LB_ID) + + mock_driver.get_security_group.assert_called_once_with(SG_NAME) + mock_get_sg_name.assert_called_once_with(LB_ID) + + # Test execute with empty get subnet response + mock_driver.reset_mock() + mock_get_sg_name.reset_mock() + + result = net_task.execute(LB_ID) + + self.assertIsNone(result) + mock_get_sg_name.assert_called_once_with(LB_ID) + + # Test execute no security group found, security groups enabled + mock_driver.reset_mock() + mock_get_sg_name.reset_mock() + mock_driver.sec_grp_enabled = True + + self.assertRaises(net_base.SecurityGroupNotFound, net_task.execute, + LB_ID) + mock_driver.get_security_group.assert_called_once_with(SG_NAME) + mock_get_sg_name.assert_called_once_with(LB_ID) + + # Test execute no security group found, security groups disabled + mock_driver.reset_mock() + mock_get_sg_name.reset_mock() + mock_driver.sec_grp_enabled = False + + result = net_task.execute(LB_ID) + + self.assertIsNone(result) + mock_driver.get_security_group.assert_called_once_with(SG_NAME) + mock_get_sg_name.assert_called_once_with(LB_ID) diff --git a/octavia/tests/unit/controller/worker/v2/tasks/test_retry_tasks.py b/octavia/tests/unit/controller/worker/v2/tasks/test_retry_tasks.py new file mode 100644 index 0000000000..d7ac3a7d65 --- /dev/null +++ b/octavia/tests/unit/controller/worker/v2/tasks/test_retry_tasks.py @@ -0,0 +1,47 @@ +# Copyright 2020 Red Hat, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from unittest import mock + +from taskflow import retry + +from octavia.controller.worker.v2.tasks import retry_tasks +import octavia.tests.unit.base as base + + +class TestRetryTasks(base.TestCase): + + def setUp(self): + super(TestRetryTasks, self).setUp() + + @mock.patch('time.sleep') + def test_sleeping_retry_times_controller(self, mock_sleep): + retry_ctrlr = retry_tasks.SleepingRetryTimesController( + attempts=2, name='test_retry') + + # Test on_failure that should RETRY + history = ['boom'] + + result = retry_ctrlr.on_failure(history) + + self.assertEqual(retry.RETRY, result) + + # Test on_failure retries exhausted, should REVERT + history = ['boom', 'bang', 'pow'] + + result = retry_ctrlr.on_failure(history) + + self.assertEqual(retry.REVERT, result) + + # Test revert - should not raise + retry_ctrlr.revert(history) diff --git a/octavia/tests/unit/controller/worker/v2/test_controller_worker.py b/octavia/tests/unit/controller/worker/v2/test_controller_worker.py index 68a7351d7e..e502142c09 100644 --- a/octavia/tests/unit/controller/worker/v2/test_controller_worker.py +++ b/octavia/tests/unit/controller/worker/v2/test_controller_worker.py @@ -21,6 +21,7 @@ from oslo_utils import uuidutils from octavia.api.drivers import utils as provider_utils from octavia.common import constants from octavia.common import data_models +from octavia.common import exceptions from octavia.controller.worker.v2 import controller_worker from octavia.controller.worker.v2.flows import flow_utils import octavia.tests.unit.base as base @@ -38,6 +39,9 @@ L7POLICY_ID = uuidutils.generate_uuid() L7RULE_ID = uuidutils.generate_uuid() PROJECT_ID = uuidutils.generate_uuid() LISTENER_ID = uuidutils.generate_uuid() +FLAVOR_ID = uuidutils.generate_uuid() +SERVER_GROUP_ID = uuidutils.generate_uuid() +AZ_ID = uuidutils.generate_uuid() HEALTH_UPDATE_DICT = {'delay': 1, 'timeout': 2} LISTENER_UPDATE_DICT = {'name': 'test', 'description': 'test2'} MEMBER_UPDATE_DICT = {'weight': 1, 'ip_address': '10.0.0.0'} @@ -63,7 +67,11 @@ _vip_mock = mock.MagicMock() _listener_mock = mock.MagicMock() _db_load_balancer_mock = mock.MagicMock() _load_balancer_mock = { - constants.LOADBALANCER_ID: LB_ID + constants.LOADBALANCER_ID: LB_ID, + constants.TOPOLOGY: constants.TOPOLOGY_SINGLE, + constants.FLAVOR_ID: None, + constants.AVAILABILITY_ZONE: None, + constants.SERVER_GROUP_ID: None } _member_mock = mock.MagicMock() @@ -124,13 +132,19 @@ class TestControllerWorker(base.TestCase): _db_load_balancer_mock.amphorae = _db_amphora_mock _db_load_balancer_mock.vip = _vip_mock _db_load_balancer_mock.id = LB_ID + _db_load_balancer_mock.flavor_id = None + _db_load_balancer_mock.availability_zone = None + _db_load_balancer_mock.server_group_id = None _db_load_balancer_mock.project_id = PROJECT_ID + _db_load_balancer_mock.topology = constants.TOPOLOGY_SINGLE _listener_mock.load_balancer = _db_load_balancer_mock _listener_mock.id = LISTENER_ID _listener_mock.to_dict.return_value = { - constants.ID: LISTENER_ID, constants.LOAD_BALANCER_ID: LB_ID} + constants.ID: LISTENER_ID, constants.LOAD_BALANCER_ID: LB_ID, + constants.PROJECT_ID: PROJECT_ID} self.ref_listener_dict = {constants.LISTENER_ID: LISTENER_ID, - constants.LOADBALANCER_ID: LB_ID} + constants.LOADBALANCER_ID: LB_ID, + constants.PROJECT_ID: PROJECT_ID} _member_mock.pool = _db_pool_mock _l7policy_mock.listener = _listener_mock @@ -180,6 +194,7 @@ class TestControllerWorker(base.TestCase): store={constants.BUILD_TYPE_PRIORITY: constants.LB_CREATE_SPARES_POOL_PRIORITY, constants.FLAVOR: None, + constants.SERVER_GROUP_ID: None, constants.AVAILABILITY_ZONE: None})) @mock.patch('octavia.db.repositories.AvailabilityZoneRepository.' @@ -216,40 +231,9 @@ class TestControllerWorker(base.TestCase): store={constants.BUILD_TYPE_PRIORITY: constants.LB_CREATE_SPARES_POOL_PRIORITY, constants.FLAVOR: None, + constants.SERVER_GROUP_ID: None, constants.AVAILABILITY_ZONE: az_data})) - @mock.patch('octavia.controller.worker.v2.flows.' - 'amphora_flows.AmphoraFlows.get_delete_amphora_flow', - return_value='TEST') - def test_delete_amphora(self, - mock_get_delete_amp_flow, - mock_api_get_session, - mock_dyn_log_listener, - mock_taskflow_load, - mock_pool_repo_get, - mock_member_repo_get, - mock_l7rule_repo_get, - mock_l7policy_repo_get, - mock_listener_repo_get, - mock_lb_repo_get, - mock_health_mon_repo_get, - mock_amp_repo_get): - - _flow_mock.reset_mock() - - cw = controller_worker.ControllerWorker() - cw.delete_amphora(AMP_ID) - - mock_amp_repo_get.assert_called_once_with( - _db_session, - id=AMP_ID) - mock_amp_repo_get.return_value = _db_amphora_mock - - (cw.services_controller.run_poster. - assert_called_once_with( - flow_utils.get_delete_amphora_flow, - store={constants.AMPHORA: _db_amphora_mock.to_dict()})) - @mock.patch('octavia.controller.worker.v2.flows.' 'health_monitor_flows.HealthMonitorFlows.' 'get_create_health_monitor_flow', @@ -379,7 +363,8 @@ class TestControllerWorker(base.TestCase): cw = controller_worker.ControllerWorker() listener_dict = {constants.LISTENER_ID: LISTENER_ID, - constants.LOADBALANCER_ID: LB_ID} + constants.LOADBALANCER_ID: LB_ID, + constants.PROJECT_ID: PROJECT_ID} cw.create_listener(listener_dict) provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( _db_load_balancer_mock).to_dict() @@ -407,7 +392,8 @@ class TestControllerWorker(base.TestCase): _flow_mock.reset_mock() listener_dict = {constants.LISTENER_ID: LISTENER_ID, - constants.LOADBALANCER_ID: LB_ID} + constants.LOADBALANCER_ID: LB_ID, + constants.PROJECT_ID: PROJECT_ID} cw = controller_worker.ControllerWorker() cw.delete_listener(listener_dict) @@ -448,7 +434,37 @@ class TestControllerWorker(base.TestCase): constants.LISTENERS: [listener_dict]})) - def test_create_load_balancer_single( + def test_create_load_balancer_single_no_anti_affinity( + self, mock_api_get_session, + mock_dyn_log_listener, mock_taskflow_load, mock_pool_repo_get, + mock_member_repo_get, mock_l7rule_repo_get, mock_l7policy_repo_get, + mock_listener_repo_get, mock_lb_repo_get, + mock_health_mon_repo_get, mock_amp_repo_get): + # Test the code path with Nova anti-affinity disabled + self.conf.config(group="nova", enable_anti_affinity=False) + self._test_create_load_balancer_single( + mock_api_get_session, + mock_dyn_log_listener, mock_taskflow_load, mock_pool_repo_get, + mock_member_repo_get, mock_l7rule_repo_get, + mock_l7policy_repo_get, mock_listener_repo_get, + mock_lb_repo_get, mock_health_mon_repo_get, mock_amp_repo_get) + + def test_create_load_balancer_single_anti_affinity( + self, mock_api_get_session, + mock_dyn_log_listener, mock_taskflow_load, mock_pool_repo_get, + mock_member_repo_get, mock_l7rule_repo_get, mock_l7policy_repo_get, + mock_listener_repo_get, mock_lb_repo_get, + mock_health_mon_repo_get, mock_amp_repo_get): + # Test the code path with Nova anti-affinity enabled + self.conf.config(group="nova", enable_anti_affinity=True) + self._test_create_load_balancer_single( + mock_api_get_session, + mock_dyn_log_listener, mock_taskflow_load, mock_pool_repo_get, + mock_member_repo_get, mock_l7rule_repo_get, + mock_l7policy_repo_get, mock_listener_repo_get, + mock_lb_repo_get, mock_health_mon_repo_get, mock_amp_repo_get) + + def _test_create_load_balancer_single( self, mock_api_get_session, mock_dyn_log_listener, @@ -472,6 +488,7 @@ class TestControllerWorker(base.TestCase): 'update_dict': {'topology': constants.TOPOLOGY_SINGLE}, constants.BUILD_TYPE_PRIORITY: constants.LB_CREATE_NORMAL_PRIORITY, constants.FLAVOR: None, + constants.SERVER_GROUP_ID: None, constants.AVAILABILITY_ZONE: None, } lb_mock = mock.MagicMock() @@ -511,6 +528,7 @@ class TestControllerWorker(base.TestCase): 'update_dict': {'topology': constants.TOPOLOGY_ACTIVE_STANDBY}, constants.BUILD_TYPE_PRIORITY: constants.LB_CREATE_NORMAL_PRIORITY, constants.FLAVOR: None, + constants.SERVER_GROUP_ID: None, constants.AVAILABILITY_ZONE: None, } setattr(mock_lb_repo_get.return_value, 'topology', @@ -555,6 +573,7 @@ class TestControllerWorker(base.TestCase): 'update_dict': {'topology': constants.TOPOLOGY_SINGLE}, constants.BUILD_TYPE_PRIORITY: constants.LB_CREATE_NORMAL_PRIORITY, constants.FLAVOR: None, + constants.SERVER_GROUP_ID: None, constants.AVAILABILITY_ZONE: None, } @@ -600,6 +619,7 @@ class TestControllerWorker(base.TestCase): 'update_dict': {'topology': constants.TOPOLOGY_ACTIVE_STANDBY}, constants.BUILD_TYPE_PRIORITY: constants.LB_CREATE_NORMAL_PRIORITY, constants.FLAVOR: None, + constants.SERVER_GROUP_ID: None, constants.AVAILABILITY_ZONE: None, } @@ -638,6 +658,7 @@ class TestControllerWorker(base.TestCase): flow_utils.get_delete_load_balancer_flow, _load_balancer_mock, store={constants.LOADBALANCER: _load_balancer_mock, + constants.LOADBALANCER_ID: LB_ID, constants.SERVER_GROUP_ID: _db_load_balancer_mock.server_group_id, constants.PROJECT_ID: _db_load_balancer_mock.project_id, @@ -665,14 +686,16 @@ class TestControllerWorker(base.TestCase): mock_lb_repo_get.assert_called_once_with( _db_session, id=LB_ID) - list_name = 'listener_%s' % _listener_mock.id + + listener_list = [{constants.LISTENER_ID: LISTENER_ID, + constants.LOADBALANCER_ID: LB_ID, + constants.PROJECT_ID: PROJECT_ID}] (cw.services_controller.run_poster. assert_called_once_with( flow_utils.get_cascade_delete_load_balancer_flow, - _load_balancer_mock, + _load_balancer_mock, listener_list, [], store={constants.LOADBALANCER: _load_balancer_mock, - list_name: self.ref_listener_dict, constants.LOADBALANCER_ID: LB_ID, constants.SERVER_GROUP_ID: _db_load_balancer_mock.server_group_id, @@ -1179,55 +1202,340 @@ class TestControllerWorker(base.TestCase): constants.UPDATE_DICT: L7RULE_UPDATE_DICT})) + @mock.patch('octavia.api.drivers.utils.' + 'db_loadbalancer_to_provider_loadbalancer') + @mock.patch('octavia.db.repositories.LoadBalancerRepository.update') + def test_failover_amphora_lb_single(self, + mock_update, + mock_lb_db_to_provider, + mock_api_get_session, + mock_dyn_log_listener, + mock_taskflow_load, + mock_pool_repo_get, + mock_member_repo_get, + mock_l7rule_repo_get, + mock_l7policy_repo_get, + mock_listener_repo_get, + mock_lb_repo_get, + mock_health_mon_repo_get, + mock_amp_repo_get): + mock_lb = mock.MagicMock() + mock_lb.id = LB_ID + mock_lb.topology = constants.TOPOLOGY_SINGLE + mock_lb.flavor_id = None + mock_lb.availability_zone = None + mock_lb.server_group_id = None + mock_lb_repo_get.return_value = mock_lb + mock_provider_lb = mock.MagicMock() + mock_lb_db_to_provider.return_value = mock_provider_lb + mock_amphora = mock.MagicMock() + mock_amphora.load_balancer_id = None + mock_amphora.id = AMP_ID + mock_amphora.load_balancer_id = LB_ID + mock_amphora.status = constants.AMPHORA_READY + mock_amp_repo_get.return_value = mock_amphora + expected_stored_params = { + constants.AVAILABILITY_ZONE: {}, + constants.BUILD_TYPE_PRIORITY: + constants.LB_CREATE_FAILOVER_PRIORITY, + constants.FLAVOR: {constants.LOADBALANCER_TOPOLOGY: + constants.TOPOLOGY_SINGLE}, + constants.LOADBALANCER: mock_provider_lb.to_dict(), + constants.LOADBALANCER_ID: LB_ID, + constants.SERVER_GROUP_ID: None, + constants.VIP: mock_lb.vip.to_dict()} + + cw = controller_worker.ControllerWorker() + cw.services_controller.reset_mock() + cw.failover_amphora(AMP_ID) + + cw.services_controller.run_poster.assert_called_once_with( + flow_utils.get_failover_amphora_flow, + mock_amphora.to_dict(), 1, store=expected_stored_params, + wait=True) + @mock.patch('octavia.db.repositories.AvailabilityZoneRepository.' 'get_availability_zone_metadata_dict', return_value={}) + @mock.patch('octavia.api.drivers.utils.' + 'db_loadbalancer_to_provider_loadbalancer') + @mock.patch('octavia.db.repositories.LoadBalancerRepository.update') + def test_failover_amphora_lb_act_stdby(self, + mock_update, + mock_lb_db_to_provider, + mock_get_az_meta, + mock_api_get_session, + mock_dyn_log_listener, + mock_taskflow_load, + mock_pool_repo_get, + mock_member_repo_get, + mock_l7rule_repo_get, + mock_l7policy_repo_get, + mock_listener_repo_get, + mock_lb_repo_get, + mock_health_mon_repo_get, + mock_amp_repo_get): + mock_lb = mock.MagicMock() + mock_lb.id = LB_ID + mock_lb.topology = constants.TOPOLOGY_ACTIVE_STANDBY + mock_lb.flavor_id = None + mock_lb.availability_zone = None + mock_lb.server_group_id = None + mock_lb_repo_get.return_value = mock_lb + mock_provider_lb = mock.MagicMock() + mock_lb_db_to_provider.return_value = mock_provider_lb + mock_amphora = mock.MagicMock() + mock_amphora.load_balancer_id = None + mock_amphora.id = AMP_ID + mock_amphora.load_balancer_id = LB_ID + mock_amphora.status = constants.AMPHORA_READY + mock_amp_repo_get.return_value = mock_amphora + expected_stored_params = { + constants.AVAILABILITY_ZONE: {}, + constants.BUILD_TYPE_PRIORITY: + constants.LB_CREATE_FAILOVER_PRIORITY, + constants.FLAVOR: {constants.LOADBALANCER_TOPOLOGY: + constants.TOPOLOGY_ACTIVE_STANDBY}, + constants.LOADBALANCER: mock_provider_lb.to_dict(), + constants.LOADBALANCER_ID: LB_ID, + constants.SERVER_GROUP_ID: None, + constants.VIP: mock_lb.vip.to_dict()} + + cw = controller_worker.ControllerWorker() + cw.services_controller.reset_mock() + cw.failover_amphora(AMP_ID) + + cw.services_controller.run_poster.assert_called_once_with( + flow_utils.get_failover_amphora_flow, + mock_amphora.to_dict(), 2, store=expected_stored_params, + wait=True) + + @mock.patch('octavia.db.repositories.AvailabilityZoneRepository.' + 'get_availability_zone_metadata_dict', return_value={}) + @mock.patch('octavia.api.drivers.utils.' + 'db_loadbalancer_to_provider_loadbalancer') + @mock.patch('octavia.db.repositories.LoadBalancerRepository.update') + def test_failover_amphora_anti_affinity(self, + mock_update, + mock_lb_db_to_provider, + mock_get_az_meta, + mock_api_get_session, + mock_dyn_log_listener, + mock_taskflow_load, + mock_pool_repo_get, + mock_member_repo_get, + mock_l7rule_repo_get, + mock_l7policy_repo_get, + mock_listener_repo_get, + mock_lb_repo_get, + mock_health_mon_repo_get, + mock_amp_repo_get): + mock_lb = mock.MagicMock() + mock_lb.id = LB_ID + mock_lb.topology = constants.TOPOLOGY_ACTIVE_STANDBY + mock_lb.flavor_id = None + mock_lb.availability_zone = None + mock_lb.server_group_id = SERVER_GROUP_ID + mock_lb_repo_get.return_value = mock_lb + mock_provider_lb = mock.MagicMock() + mock_lb_db_to_provider.return_value = mock_provider_lb + mock_amphora = mock.MagicMock() + mock_amphora.load_balancer_id = None + mock_amphora.id = AMP_ID + mock_amphora.load_balancer_id = LB_ID + mock_amphora.status = constants.AMPHORA_READY + mock_amp_repo_get.return_value = mock_amphora + expected_stored_params = { + constants.AVAILABILITY_ZONE: {}, + constants.BUILD_TYPE_PRIORITY: + constants.LB_CREATE_FAILOVER_PRIORITY, + constants.FLAVOR: {constants.LOADBALANCER_TOPOLOGY: + constants.TOPOLOGY_ACTIVE_STANDBY}, + constants.LOADBALANCER: mock_provider_lb.to_dict(), + constants.LOADBALANCER_ID: LB_ID, + constants.SERVER_GROUP_ID: SERVER_GROUP_ID, + constants.VIP: mock_lb.vip.to_dict()} + + cw = controller_worker.ControllerWorker() + cw.services_controller.reset_mock() + cw.failover_amphora(AMP_ID) + + cw.services_controller.run_poster.assert_called_once_with( + flow_utils.get_failover_amphora_flow, + mock_amphora.to_dict(), 2, store=expected_stored_params, + wait=True) + + @mock.patch('octavia.api.drivers.utils.' + 'db_loadbalancer_to_provider_loadbalancer') + @mock.patch('octavia.db.repositories.LoadBalancerRepository.update') + def test_failover_amphora_unknown_topology(self, + mock_update, + mock_lb_db_to_provider, + mock_api_get_session, + mock_dyn_log_listener, + mock_taskflow_load, + mock_pool_repo_get, + mock_member_repo_get, + mock_l7rule_repo_get, + mock_l7policy_repo_get, + mock_listener_repo_get, + mock_lb_repo_get, + mock_health_mon_repo_get, + mock_amp_repo_get): + mock_lb = mock.MagicMock() + mock_lb.id = LB_ID + mock_lb.topology = 'bogus' + mock_lb.flavor_id = None + mock_lb.availability_zone = None + mock_lb.server_group_id = SERVER_GROUP_ID + mock_lb_repo_get.return_value = mock_lb + mock_provider_lb = mock.MagicMock() + mock_lb_db_to_provider.return_value = mock_provider_lb + mock_amphora = mock.MagicMock() + mock_amphora.load_balancer_id = None + mock_amphora.id = AMP_ID + mock_amphora.load_balancer_id = LB_ID + mock_amphora.status = constants.AMPHORA_READY + mock_amp_repo_get.return_value = mock_amphora + expected_stored_params = { + constants.AVAILABILITY_ZONE: {}, + constants.BUILD_TYPE_PRIORITY: + constants.LB_CREATE_FAILOVER_PRIORITY, + constants.FLAVOR: {constants.LOADBALANCER_TOPOLOGY: + mock_lb.topology}, + constants.LOADBALANCER: mock_provider_lb.to_dict(), + constants.LOADBALANCER_ID: LB_ID, + constants.SERVER_GROUP_ID: SERVER_GROUP_ID, + constants.VIP: mock_lb.vip.to_dict()} + + cw = controller_worker.ControllerWorker() + cw.services_controller.reset_mock() + cw.failover_amphora(AMP_ID) + + cw.services_controller.run_poster.assert_called_once_with( + flow_utils.get_failover_amphora_flow, + mock_amphora.to_dict(), None, store=expected_stored_params, + wait=True) + @mock.patch('octavia.db.repositories.FlavorRepository.' 'get_flavor_metadata_dict', return_value={}) + @mock.patch('octavia.api.drivers.utils.' + 'db_loadbalancer_to_provider_loadbalancer') @mock.patch('octavia.db.repositories.LoadBalancerRepository.update') - def test_failover_amphora(self, - mock_update, - mock_get_flavor_meta, - mock_get_az_meta, - mock_api_get_session, - mock_dyn_log_listener, - mock_taskflow_load, - mock_pool_repo_get, - mock_member_repo_get, - mock_l7rule_repo_get, - mock_l7policy_repo_get, - mock_listener_repo_get, - mock_lb_repo_get, - mock_health_mon_repo_get, - mock_amp_repo_get): + def test_failover_amphora_with_flavor(self, + mock_update, + mock_lb_db_to_provider, + mock_get_flavor_meta, + mock_api_get_session, + mock_dyn_log_listener, + mock_taskflow_load, + mock_pool_repo_get, + mock_member_repo_get, + mock_l7rule_repo_get, + mock_l7policy_repo_get, + mock_listener_repo_get, + mock_lb_repo_get, + mock_health_mon_repo_get, + mock_amp_repo_get): + mock_lb = mock.MagicMock() + mock_lb.id = LB_ID + mock_lb.topology = constants.TOPOLOGY_SINGLE + mock_lb.flavor_id = FLAVOR_ID + mock_lb.availability_zone = None + mock_lb.server_group_id = None + mock_lb_repo_get.return_value = mock_lb + mock_provider_lb = mock.MagicMock() + mock_lb_db_to_provider.return_value = mock_provider_lb + mock_amphora = mock.MagicMock() + mock_amphora.load_balancer_id = None + mock_amphora.id = AMP_ID + mock_amphora.load_balancer_id = LB_ID + mock_amphora.status = constants.AMPHORA_READY + mock_amp_repo_get.return_value = mock_amphora + expected_stored_params = { + constants.AVAILABILITY_ZONE: {}, + constants.BUILD_TYPE_PRIORITY: + constants.LB_CREATE_FAILOVER_PRIORITY, + constants.FLAVOR: {constants.LOADBALANCER_TOPOLOGY: + constants.TOPOLOGY_SINGLE, + 'taste': 'spicy'}, + constants.LOADBALANCER: mock_provider_lb.to_dict(), + constants.LOADBALANCER_ID: LB_ID, + constants.SERVER_GROUP_ID: None, + constants.VIP: mock_lb.vip.to_dict()} + mock_get_flavor_meta.return_value = {'taste': 'spicy'} - _flow_mock.reset_mock() - _db_amphora_mock.reset_mock() - mock_amp_repo_get.return_value = _db_amphora_mock cw = controller_worker.ControllerWorker() + cw.services_controller.reset_mock() cw.failover_amphora(AMP_ID) - mock_lb_repo_get.return_value = _db_load_balancer_mock - (cw.services_controller.run_poster. - assert_called_once_with( - flow_utils.get_failover_flow, - role=_db_amphora_mock.role, - load_balancer={}, - wait=True, - store={constants.FAILED_AMPHORA: _db_amphora_mock.to_dict(), - constants.LOADBALANCER_ID: - _db_amphora_mock.load_balancer_id, - constants.BUILD_TYPE_PRIORITY: - constants.LB_CREATE_FAILOVER_PRIORITY, - constants.FLAVOR: {}, - constants.AVAILABILITY_ZONE: {} - })) - mock_update.assert_called_with(_db_session, LB_ID, - provisioning_status=constants.ACTIVE) + cw.services_controller.run_poster.assert_called_once_with( + flow_utils.get_failover_amphora_flow, + mock_amphora.to_dict(), 1, store=expected_stored_params, + wait=True) - @mock.patch('octavia.controller.worker.v2.controller_worker.' - 'ControllerWorker._perform_amphora_failover') + @mock.patch('octavia.db.repositories.AvailabilityZoneRepository.' + 'get_availability_zone_metadata_dict', return_value={}) + @mock.patch('octavia.api.drivers.utils.' + 'db_loadbalancer_to_provider_loadbalancer') + @mock.patch('octavia.db.repositories.LoadBalancerRepository.update') + def test_failover_amphora_with_az(self, + mock_update, + mock_lb_db_to_provider, + mock_get_az_meta, + mock_api_get_session, + mock_dyn_log_listener, + mock_taskflow_load, + mock_pool_repo_get, + mock_member_repo_get, + mock_l7rule_repo_get, + mock_l7policy_repo_get, + mock_listener_repo_get, + mock_lb_repo_get, + mock_health_mon_repo_get, + mock_amp_repo_get): + mock_lb = mock.MagicMock() + mock_lb.id = LB_ID + mock_lb.topology = constants.TOPOLOGY_SINGLE + mock_lb.flavor_id = None + mock_lb.availability_zone = AZ_ID + mock_lb.server_group_id = None + mock_lb_repo_get.return_value = mock_lb + mock_provider_lb = mock.MagicMock() + mock_lb_db_to_provider.return_value = mock_provider_lb + mock_amphora = mock.MagicMock() + mock_amphora.load_balancer_id = None + mock_amphora.id = AMP_ID + mock_amphora.load_balancer_id = LB_ID + mock_amphora.status = constants.AMPHORA_READY + mock_amp_repo_get.return_value = mock_amphora + expected_stored_params = { + constants.AVAILABILITY_ZONE: {'planet': 'jupiter'}, + constants.BUILD_TYPE_PRIORITY: + constants.LB_CREATE_FAILOVER_PRIORITY, + constants.FLAVOR: {constants.LOADBALANCER_TOPOLOGY: + constants.TOPOLOGY_SINGLE}, + constants.LOADBALANCER: mock_provider_lb.to_dict(), + constants.LOADBALANCER_ID: LB_ID, + constants.SERVER_GROUP_ID: None, + constants.VIP: mock_lb.vip.to_dict()} + mock_get_az_meta.return_value = {'planet': 'jupiter'} + + cw = controller_worker.ControllerWorker() + cw.services_controller.reset_mock() + cw.failover_amphora(AMP_ID) + + print(cw, flush=True) + print(cw.services_controller, flush=True) + print(cw.services_controller.run_poster, flush=True) + cw.services_controller.run_poster.assert_called_once_with( + flow_utils.get_failover_amphora_flow, + mock_amphora.to_dict(), 1, store=expected_stored_params, + wait=True) + + @mock.patch('octavia.controller.worker.v1.flows.amphora_flows.' + 'AmphoraFlows.get_failover_amphora_flow') def test_failover_amp_missing_amp(self, - mock_perform_amp_failover, + mock_get_amp_failover, mock_api_get_session, mock_dyn_log_listener, mock_taskflow_load, @@ -1245,14 +1553,11 @@ class TestControllerWorker(base.TestCase): cw = controller_worker.ControllerWorker() cw.failover_amphora(AMP_ID) - mock_perform_amp_failover.assert_not_called() + mock_get_amp_failover.assert_not_called() - @mock.patch('octavia.controller.worker.v2.controller_worker.' - 'ControllerWorker._perform_amphora_failover') @mock.patch('octavia.db.repositories.LoadBalancerRepository.update') def test_failover_amp_flow_exception(self, mock_update, - mock_perform_amp_failover, mock_api_get_session, mock_dyn_log_listener, mock_taskflow_load, @@ -1264,19 +1569,21 @@ class TestControllerWorker(base.TestCase): mock_lb_repo_get, mock_health_mon_repo_get, mock_amp_repo_get): + mock_amphora = mock.MagicMock() + mock_amphora.id = AMP_ID + mock_amphora.load_balancer_id = LB_ID + mock_amp_repo_get.return_value = mock_amphora - mock_perform_amp_failover.side_effect = TestException('boom') + mock_lb_repo_get.side_effect = TestException('boom') cw = controller_worker.ControllerWorker() - self.assertRaises(TestException, cw.failover_amphora, AMP_ID) + cw.failover_amphora(AMP_ID) mock_update.assert_called_with(_db_session, LB_ID, provisioning_status=constants.ERROR) - @mock.patch('octavia.controller.worker.v2.controller_worker.' - 'ControllerWorker._perform_amphora_failover') - @mock.patch('octavia.db.repositories.LoadBalancerRepository.update') + @mock.patch('octavia.controller.worker.v1.flows.amphora_flows.' + 'AmphoraFlows.get_failover_amphora_flow') def test_failover_amp_no_lb(self, - mock_lb_update, - mock_perform_amp_failover, + mock_get_failover_amp_flow, mock_api_get_session, mock_dyn_log_listener, mock_taskflow_load, @@ -1288,60 +1595,29 @@ class TestControllerWorker(base.TestCase): mock_lb_repo_get, mock_health_mon_repo_get, mock_amp_repo_get): - - amphora = mock.MagicMock() - amphora.load_balancer_id = None - mock_amp_repo_get.return_value = amphora - - cw = controller_worker.ControllerWorker() - cw.failover_amphora(AMP_ID) - - mock_lb_update.assert_not_called() - mock_perform_amp_failover.assert_called_once_with( - amphora, constants.LB_CREATE_FAILOVER_PRIORITY) - - @mock.patch( - 'octavia.db.repositories.AmphoraRepository.get_lb_for_amphora', - return_value=None) - def test_failover_spare_amphora(self, - mock_get_lb_for_amphora, - mock_api_get_session, - mock_dyn_log_listener, - mock_taskflow_load, - mock_pool_repo_get, - mock_member_repo_get, - mock_l7rule_repo_get, - mock_l7policy_repo_get, - mock_listener_repo_get, - mock_lb_repo_get, - mock_health_mon_repo_get, - mock_amp_repo_get): - - _flow_mock.reset_mock() - - # simulate a spare amphora (amphora not attached to any load_balancer) + _flow_mock.run.reset_mock() mock_amphora = mock.MagicMock() + mock_amphora.load_balancer_id = None mock_amphora.id = AMP_ID mock_amphora.status = constants.AMPHORA_READY - mock_amphora.load_balancer_id = None - mock_amphora.role = constants.ROLE_STANDALONE + mock_amp_repo_get.return_value = mock_amphora + expected_stored_params = {constants.AVAILABILITY_ZONE: {}, + constants.BUILD_TYPE_PRIORITY: + constants.LB_CREATE_FAILOVER_PRIORITY, + constants.FLAVOR: {}, + constants.LOADBALANCER: None, + constants.LOADBALANCER_ID: None, + constants.SERVER_GROUP_ID: None, + constants.VIP: {}} cw = controller_worker.ControllerWorker() - cw._perform_amphora_failover(mock_amphora, - constants.LB_CREATE_FAILOVER_PRIORITY) + cw.services_controller.reset_mock() + cw.failover_amphora(AMP_ID) - (cw.services_controller.run_poster. - assert_called_once_with( - flow_utils.get_failover_flow, - role=constants.ROLE_STANDALONE, load_balancer=None, - wait=True, - store={constants.FAILED_AMPHORA: mock_amphora.to_dict(), - constants.LOADBALANCER_ID: None, - constants.BUILD_TYPE_PRIORITY: - constants.LB_CREATE_FAILOVER_PRIORITY, - constants.FLAVOR: {}, - constants.AVAILABILITY_ZONE: {} - })) + cw.services_controller.run_poster.assert_called_once_with( + flow_utils.get_failover_amphora_flow, + mock_amphora.to_dict(), + None, store=expected_stored_params, wait=True) @mock.patch('octavia.db.repositories.AmphoraHealthRepository.delete') def test_failover_deleted_amphora(self, @@ -1362,112 +1638,391 @@ class TestControllerWorker(base.TestCase): mock_amphora = mock.MagicMock() mock_amphora.id = AMP_ID mock_amphora.status = constants.DELETED + mock_amp_repo_get.return_value = mock_amphora cw = controller_worker.ControllerWorker() - cw._perform_amphora_failover(mock_amphora, 10) + cw.failover_amphora(AMP_ID) mock_delete.assert_called_with(_db_session, amphora_id=AMP_ID) mock_taskflow_load.assert_not_called() - @mock.patch('octavia.controller.worker.v2.' - 'controller_worker.ControllerWorker._perform_amphora_failover') - @mock.patch('octavia.db.repositories.LoadBalancerRepository.update') - def test_failover_loadbalancer(self, - mock_update, - mock_perform, - mock_api_get_session, - mock_dyn_log_listener, - mock_taskflow_load, - mock_pool_repo_get, - mock_member_repo_get, - mock_l7rule_repo_get, - mock_l7policy_repo_get, - mock_listener_repo_get, - mock_lb_repo_get, - mock_health_mon_repo_get, - mock_amp_repo_get): - _amphora_mock2 = mock.MagicMock() - _amphora_mock3 = mock.MagicMock() - _amphora_mock3.status = constants.DELETED - _db_load_balancer_mock.amphorae = [ - _db_amphora_mock, _amphora_mock2, _amphora_mock3] + def test_get_amphorae_for_failover_single(self, + mock_api_get_session, + mock_dyn_log_listener, + mock_taskflow_load, + mock_pool_repo_get, + mock_member_repo_get, + mock_l7rule_repo_get, + mock_l7policy_repo_get, + mock_listener_repo_get, + mock_lb_repo_get, + mock_health_mon_repo_get, + mock_amp_repo_get): + amphora1_mock = mock.MagicMock() + amphora1_mock.status = constants.AMPHORA_ALLOCATED + amphora2_mock = mock.MagicMock() + amphora2_mock.status = constants.DELETED + + load_balancer_mock = mock.MagicMock() + load_balancer_mock.topology = constants.TOPOLOGY_SINGLE + load_balancer_mock.amphorae = [amphora1_mock, amphora2_mock] + cw = controller_worker.ControllerWorker() - cw.failover_loadbalancer('123') - mock_perform.assert_called_with( - _amphora_mock2, - constants.LB_CREATE_ADMIN_FAILOVER_PRIORITY) - mock_update.assert_called_with(_db_session, '123', - provisioning_status=constants.ACTIVE) + result = cw._get_amphorae_for_failover(load_balancer_mock) - mock_perform.reset_mock() - _db_load_balancer_mock.amphorae = [ - _db_amphora_mock, _amphora_mock2, _amphora_mock3] - _amphora_mock2.role = constants.ROLE_BACKUP - cw.failover_loadbalancer('123') - # because mock2 gets failed over earlier now _amphora_mock - # is the last one - mock_perform.assert_called_with( - _db_amphora_mock, constants.LB_CREATE_ADMIN_FAILOVER_PRIORITY) - mock_update.assert_called_with(_db_session, '123', - provisioning_status=constants.ACTIVE) + self.assertEqual([amphora1_mock.to_dict()], result) - mock_perform.side_effect = OverflowError() - self.assertRaises(OverflowError, cw.failover_loadbalancer, 123) - mock_update.assert_called_with(_db_session, 123, - provisioning_status=constants.ERROR) + @mock.patch('octavia.common.utils.get_amphora_driver') + def test_get_amphorae_for_failover_act_stdby(self, + mock_get_amp_driver, + mock_api_get_session, + mock_dyn_log_listener, + mock_taskflow_load, + mock_pool_repo_get, + mock_member_repo_get, + mock_l7rule_repo_get, + mock_l7policy_repo_get, + mock_listener_repo_get, + mock_lb_repo_get, + mock_health_mon_repo_get, + mock_amp_repo_get): + # Note: This test uses three amphora even though we only have + # two per load balancer to properly test the ordering from + # this method. + amp_driver_mock = mock.MagicMock() + amp_driver_mock.get_interface_from_ip.side_effect = [ + 'fake0', None, 'fake1'] + mock_get_amp_driver.return_value = amp_driver_mock + backup_amphora_mock = mock.MagicMock() + backup_amphora_mock.status = constants.AMPHORA_ALLOCATED + deleted_amphora_mock = mock.MagicMock() + deleted_amphora_mock.status = constants.DELETED + master_amphora_mock = mock.MagicMock() + master_amphora_mock.status = constants.AMPHORA_ALLOCATED + bogus_amphora_mock = mock.MagicMock() + bogus_amphora_mock.status = constants.AMPHORA_ALLOCATED - @mock.patch('octavia.db.repositories.AvailabilityZoneRepository.' - 'get_availability_zone_metadata_dict', return_value={}) - @mock.patch('octavia.db.repositories.FlavorRepository.' - 'get_flavor_metadata_dict', return_value={}) - @mock.patch( - 'octavia.db.repositories.AmphoraRepository.get_lb_for_amphora', - return_value=_db_load_balancer_mock) - @mock.patch('octavia.db.repositories.LoadBalancerRepository.update') - def test_failover_amphora_anti_affinity(self, - mock_update, - mock_get_lb_for_amphora, - mock_get_flavor_meta, - mock_get_az_meta, - mock_api_get_session, - mock_dyn_log_listener, - mock_taskflow_load, - mock_pool_repo_get, - mock_member_repo_get, - mock_l7rule_repo_get, - mock_l7policy_repo_get, - mock_listener_repo_get, - mock_lb_repo_get, - mock_health_mon_repo_get, - mock_amp_repo_get): + load_balancer_mock = mock.MagicMock() + load_balancer_mock.topology = constants.TOPOLOGY_ACTIVE_STANDBY + load_balancer_mock.amphorae = [ + master_amphora_mock, deleted_amphora_mock, backup_amphora_mock, + bogus_amphora_mock] - self.conf.config(group="nova", enable_anti_affinity=True) + cw = controller_worker.ControllerWorker() + result = cw._get_amphorae_for_failover(load_balancer_mock) + + self.assertEqual([master_amphora_mock.to_dict(), + bogus_amphora_mock.to_dict(), + backup_amphora_mock.to_dict()], result) + + @mock.patch('octavia.common.utils.get_amphora_driver') + def test_get_amphorae_for_failover_act_stdby_net_split( + self, mock_get_amp_driver, mock_api_get_session, + mock_dyn_log_listener, mock_taskflow_load, mock_pool_repo_get, + mock_member_repo_get, mock_l7rule_repo_get, mock_l7policy_repo_get, + mock_listener_repo_get, mock_lb_repo_get, mock_health_mon_repo_get, + mock_amp_repo_get): + # Case where the amps can't see eachother and somehow end up with + # two amphora with an interface. This is highly unlikely as the + # higher priority amphora should get the IP in a net split, but + # let's test the code for this odd case. + # Note: This test uses three amphora even though we only have + # two per load balancer to properly test the ordering from + # this method. + amp_driver_mock = mock.MagicMock() + amp_driver_mock.get_interface_from_ip.side_effect = [ + 'fake0', 'fake1'] + mock_get_amp_driver.return_value = amp_driver_mock + backup_amphora_mock = mock.MagicMock() + backup_amphora_mock.status = constants.AMPHORA_ALLOCATED + deleted_amphora_mock = mock.MagicMock() + deleted_amphora_mock.status = constants.DELETED + master_amphora_mock = mock.MagicMock() + master_amphora_mock.status = constants.AMPHORA_ALLOCATED + + load_balancer_mock = mock.MagicMock() + load_balancer_mock.topology = constants.TOPOLOGY_ACTIVE_STANDBY + load_balancer_mock.amphorae = [ + backup_amphora_mock, deleted_amphora_mock, master_amphora_mock] + + cw = controller_worker.ControllerWorker() + result = cw._get_amphorae_for_failover(load_balancer_mock) + + self.assertEqual([backup_amphora_mock.to_dict(), + master_amphora_mock.to_dict()], result) + + def test_get_amphorae_for_failover_bogus_topology(self, + mock_api_get_session, + mock_dyn_log_listener, + mock_taskflow_load, + mock_pool_repo_get, + mock_member_repo_get, + mock_l7rule_repo_get, + mock_l7policy_repo_get, + mock_listener_repo_get, + mock_lb_repo_get, + mock_health_mon_repo_get, + mock_amp_repo_get): + load_balancer_mock = mock.MagicMock() + load_balancer_mock.topology = 'bogus' + + cw = controller_worker.ControllerWorker() + self.assertRaises(exceptions.InvalidTopology, + cw._get_amphorae_for_failover, + load_balancer_mock) + + @mock.patch('octavia.controller.worker.v2.controller_worker.' + 'ControllerWorker._get_amphorae_for_failover') + def test_failover_loadbalancer_single(self, + mock_get_amps_for_failover, + mock_api_get_session, + mock_dyn_log_listener, + mock_taskflow_load, + mock_pool_repo_get, + mock_member_repo_get, + mock_l7rule_repo_get, + mock_l7policy_repo_get, + mock_listener_repo_get, + mock_lb_repo_get, + mock_health_mon_repo_get, + mock_amp_repo_get): _flow_mock.reset_mock() - _db_load_balancer_mock.server_group_id = "123" - - cw = controller_worker.ControllerWorker() - cw.failover_amphora(AMP_ID) + mock_lb_repo_get.return_value = _db_load_balancer_mock + mock_get_amps_for_failover.return_value = [_amphora_mock] provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( _db_load_balancer_mock).to_dict() - (cw.services_controller.run_poster. - assert_called_once_with( - flow_utils.get_failover_flow, - role=_db_amphora_mock.role, - load_balancer=provider_lb, - wait=True, - store={constants.FAILED_AMPHORA: _db_amphora_mock.to_dict(), - constants.LOADBALANCER_ID: - _db_amphora_mock.load_balancer_id, - constants.BUILD_TYPE_PRIORITY: - constants.LB_CREATE_FAILOVER_PRIORITY, - constants.SERVER_GROUP_ID: "123", - constants.FLAVOR: {}, - constants.AVAILABILITY_ZONE: {} - })) + expected_flavor = {constants.LOADBALANCER_TOPOLOGY: + _load_balancer_mock[constants.TOPOLOGY]} + provider_lb[constants.FLAVOR] = expected_flavor + expected_flow_store = {constants.LOADBALANCER: provider_lb, + constants.BUILD_TYPE_PRIORITY: + constants.LB_CREATE_FAILOVER_PRIORITY, + constants.LOADBALANCER_ID: + _load_balancer_mock[ + constants.LOADBALANCER_ID], + constants.SERVER_GROUP_ID: + _load_balancer_mock[ + constants.SERVER_GROUP_ID], + constants.FLAVOR: expected_flavor, + constants.AVAILABILITY_ZONE: {}} - mock_update.assert_called_with(_db_session, LB_ID, - provisioning_status=constants.ACTIVE) + cw = controller_worker.ControllerWorker() + cw.failover_loadbalancer(LB_ID) + + mock_lb_repo_get.assert_called_once_with(_db_session, id=LB_ID) + mock_get_amps_for_failover.assert_called_once_with( + _db_load_balancer_mock) + + cw.services_controller.run_poster.assert_called_once_with( + flow_utils.get_failover_LB_flow, [_amphora_mock], provider_lb, + wait=True, store=expected_flow_store) + + @mock.patch('octavia.controller.worker.v2.controller_worker.' + 'ControllerWorker._get_amphorae_for_failover') + def test_failover_loadbalancer_act_stdby(self, + mock_get_amps_for_failover, + mock_api_get_session, + mock_dyn_log_listener, + mock_taskflow_load, + mock_pool_repo_get, + mock_member_repo_get, + mock_l7rule_repo_get, + mock_l7policy_repo_get, + mock_listener_repo_get, + mock_lb_repo_get, + mock_health_mon_repo_get, + mock_amp_repo_get): + _flow_mock.reset_mock() + load_balancer_mock = mock.MagicMock() + load_balancer_mock.listeners = [_listener_mock] + load_balancer_mock.topology = constants.TOPOLOGY_ACTIVE_STANDBY + load_balancer_mock.flavor_id = None + load_balancer_mock.availability_zone = None + load_balancer_mock.vip = _vip_mock + mock_lb_repo_get.return_value = load_balancer_mock + mock_get_amps_for_failover.return_value = [_amphora_mock, + _amphora_mock] + provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( + load_balancer_mock).to_dict() + + expected_flavor = {constants.LOADBALANCER_TOPOLOGY: + load_balancer_mock.topology} + provider_lb[constants.FLAVOR] = expected_flavor + expected_flow_store = {constants.LOADBALANCER: provider_lb, + constants.BUILD_TYPE_PRIORITY: + constants.LB_CREATE_FAILOVER_PRIORITY, + constants.LOADBALANCER_ID: + load_balancer_mock.id, + constants.SERVER_GROUP_ID: + load_balancer_mock.server_group_id, + constants.FLAVOR: expected_flavor, + constants.AVAILABILITY_ZONE: {}} + + cw = controller_worker.ControllerWorker() + cw.failover_loadbalancer(LB_ID) + + mock_lb_repo_get.assert_called_once_with(_db_session, id=LB_ID) + mock_get_amps_for_failover.assert_called_once_with(load_balancer_mock) + + cw.services_controller.run_poster.assert_called_once_with( + flow_utils.get_failover_LB_flow, [_amphora_mock, _amphora_mock], + provider_lb, wait=True, store=expected_flow_store) + + @mock.patch('octavia.db.repositories.LoadBalancerRepository.update') + def test_failover_loadbalancer_no_lb(self, + mock_lb_repo_update, + mock_api_get_session, + mock_dyn_log_listener, + mock_taskflow_load, + mock_pool_repo_get, + mock_member_repo_get, + mock_l7rule_repo_get, + mock_l7policy_repo_get, + mock_listener_repo_get, + mock_lb_repo_get, + mock_health_mon_repo_get, + mock_amp_repo_get): + mock_lb_repo_get.return_value = None + + cw = controller_worker.ControllerWorker() + cw.failover_loadbalancer(LB_ID) + + mock_lb_repo_update.assert_called_once_with( + _db_session, LB_ID, provisioning_status=constants.ERROR) + + @mock.patch('octavia.db.repositories.LoadBalancerRepository.update') + @mock.patch('octavia.controller.worker.v2.controller_worker.' + 'ControllerWorker._get_amphorae_for_failover') + def test_failover_loadbalancer_with_bogus_topology( + self, mock_get_amps_for_failover, mock_lb_repo_update, + mock_api_get_session, mock_dyn_log_listener, mock_taskflow_load, + mock_pool_repo_get, mock_member_repo_get, mock_l7rule_repo_get, + mock_l7policy_repo_get, mock_listener_repo_get, mock_lb_repo_get, + mock_health_mon_repo_get, mock_amp_repo_get): + _flow_mock.reset_mock() + load_balancer_mock = mock.MagicMock() + load_balancer_mock.topology = 'bogus' + mock_lb_repo_get.return_value = load_balancer_mock + mock_get_amps_for_failover.return_value = [_amphora_mock] + + cw = controller_worker.ControllerWorker() + result = cw.failover_loadbalancer(LB_ID) + + self.assertIsNone(result) + mock_lb_repo_update.assert_called_once_with( + _db_session, LB_ID, provisioning_status=constants.ERROR) + mock_lb_repo_get.assert_called_once_with(_db_session, id=LB_ID) + mock_get_amps_for_failover.assert_called_once_with(load_balancer_mock) + + @mock.patch('octavia.db.repositories.AvailabilityZoneRepository.' + 'get_availability_zone_metadata_dict', return_value={}) + @mock.patch('octavia.controller.worker.v2.controller_worker.' + 'ControllerWorker._get_amphorae_for_failover') + def test_failover_loadbalancer_with_az(self, + mock_get_amps_for_failover, + mock_get_az_meta, + mock_api_get_session, + mock_dyn_log_listener, + mock_taskflow_load, + mock_pool_repo_get, + mock_member_repo_get, + mock_l7rule_repo_get, + mock_l7policy_repo_get, + mock_listener_repo_get, + mock_lb_repo_get, + mock_health_mon_repo_get, + mock_amp_repo_get): + _flow_mock.reset_mock() + load_balancer_mock = mock.MagicMock() + load_balancer_mock.listeners = [_listener_mock] + load_balancer_mock.topology = constants.TOPOLOGY_ACTIVE_STANDBY + load_balancer_mock.flavor_id = None + load_balancer_mock.availability_zone = uuidutils.generate_uuid() + load_balancer_mock.vip = _vip_mock + mock_lb_repo_get.return_value = load_balancer_mock + mock_get_amps_for_failover.return_value = [_amphora_mock] + mock_get_az_meta.return_value = {'planet': 'jupiter'} + provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( + load_balancer_mock).to_dict() + + expected_flavor = {constants.LOADBALANCER_TOPOLOGY: + load_balancer_mock.topology} + provider_lb[constants.FLAVOR] = expected_flavor + expected_flow_store = {constants.LOADBALANCER: provider_lb, + constants.BUILD_TYPE_PRIORITY: + constants.LB_CREATE_FAILOVER_PRIORITY, + constants.LOADBALANCER_ID: + load_balancer_mock.id, + constants.FLAVOR: expected_flavor, + constants.SERVER_GROUP_ID: + load_balancer_mock.server_group_id, + constants.AVAILABILITY_ZONE: { + 'planet': 'jupiter'}} + + cw = controller_worker.ControllerWorker() + cw.failover_loadbalancer(LB_ID) + + mock_lb_repo_get.assert_called_once_with(_db_session, id=LB_ID) + mock_get_amps_for_failover.assert_called_once_with(load_balancer_mock) + + cw.services_controller.run_poster.assert_called_once_with( + flow_utils.get_failover_LB_flow, [_amphora_mock], provider_lb, + wait=True, store=expected_flow_store) + + @mock.patch('octavia.db.repositories.FlavorRepository.' + 'get_flavor_metadata_dict', return_value={'taste': 'spicy'}) + @mock.patch('octavia.controller.worker.v2.controller_worker.' + 'ControllerWorker._get_amphorae_for_failover') + def test_failover_loadbalancer_with_flavor(self, + mock_get_amps_for_failover, + mock_get_flavor_meta, + mock_api_get_session, + mock_dyn_log_listener, + mock_taskflow_load, + mock_pool_repo_get, + mock_member_repo_get, + mock_l7rule_repo_get, + mock_l7policy_repo_get, + mock_listener_repo_get, + mock_lb_repo_get, + mock_health_mon_repo_get, + mock_amp_repo_get): + _flow_mock.reset_mock() + load_balancer_mock = mock.MagicMock() + load_balancer_mock.listeners = [_listener_mock] + load_balancer_mock.topology = constants.TOPOLOGY_SINGLE + load_balancer_mock.flavor_id = uuidutils.generate_uuid() + load_balancer_mock.availability_zone = None + load_balancer_mock.vip = _vip_mock + mock_lb_repo_get.return_value = load_balancer_mock + mock_get_amps_for_failover.return_value = [_amphora_mock, + _amphora_mock] + provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer( + load_balancer_mock).to_dict() + + expected_flavor = {'taste': 'spicy', constants.LOADBALANCER_TOPOLOGY: + load_balancer_mock.topology} + provider_lb[constants.FLAVOR] = expected_flavor + expected_flow_store = {constants.LOADBALANCER: provider_lb, + constants.BUILD_TYPE_PRIORITY: + constants.LB_CREATE_FAILOVER_PRIORITY, + constants.LOADBALANCER_ID: + load_balancer_mock.id, + constants.FLAVOR: expected_flavor, + constants.SERVER_GROUP_ID: + load_balancer_mock.server_group_id, + constants.AVAILABILITY_ZONE: {}} + + cw = controller_worker.ControllerWorker() + cw.failover_loadbalancer(LB_ID) + + mock_lb_repo_get.assert_called_once_with(_db_session, id=LB_ID) + mock_get_amps_for_failover.assert_called_once_with(load_balancer_mock) + + cw.services_controller.run_poster.assert_called_once_with( + flow_utils.get_failover_LB_flow, [_amphora_mock, _amphora_mock], + provider_lb, wait=True, store=expected_flow_store) def test_amphora_cert_rotation(self, mock_api_get_session, @@ -1483,6 +2038,7 @@ class TestControllerWorker(base.TestCase): mock_amp_repo_get): _flow_mock.reset_mock() cw = controller_worker.ControllerWorker() + cw.services_controller.reset_mock() cw.amphora_cert_rotation(AMP_ID) mock_amp_repo_get.return_value = _db_amphora_mock (cw.services_controller.run_poster. @@ -1515,6 +2071,7 @@ class TestControllerWorker(base.TestCase): mock_get_lb_for_amp.return_value = mock_lb mock_flavor_meta.return_value = {'test': 'dict'} cw = controller_worker.ControllerWorker() + cw.services_controller.reset_mock() cw.update_amphora_agent_config(AMP_ID) mock_amp_repo_get.assert_called_once_with(_db_session, id=AMP_ID) diff --git a/tools/create_flow_docs.py b/tools/create_flow_docs.py index 4038e7374f..68ac1362f1 100755 --- a/tools/create_flow_docs.py +++ b/tools/create_flow_docs.py @@ -21,6 +21,7 @@ import os import graphviz from taskflow import engines +from octavia.api.drivers import utils from octavia.common import constants from octavia.tests.common import data_model_helpers as dmh @@ -56,6 +57,9 @@ def generate(flow_list, output_directory): amp1 = dmh.generate_amphora() amp2 = dmh.generate_amphora() lb = dmh.generate_load_balancer(amphorae=[amp1, amp2]) + if 'v2' in current_tuple[0]: + lb = utils.lb_dict_to_provider_dict(lb.to_dict()) + amp1 = amp1.to_dict() current_engine = engines.load( get_flow_method(amp1, 2)) elif (current_tuple[1] == 'LoadBalancerFlows' and @@ -66,12 +70,28 @@ def generate(flow_list, output_directory): elif (current_tuple[1] == 'LoadBalancerFlows' and current_tuple[2] == 'get_delete_load_balancer_flow'): lb = dmh.generate_load_balancer() - delete_flow, store = get_flow_method(lb) + if 'v2' in current_tuple[0]: + lb = utils.lb_dict_to_provider_dict(lb.to_dict()) + delete_flow = get_flow_method(lb) + else: + delete_flow, store = get_flow_method(lb) current_engine = engines.load(delete_flow) elif (current_tuple[1] == 'LoadBalancerFlows' and current_tuple[2] == 'get_cascade_delete_load_balancer_flow'): + listeners = [{constants.LISTENER_ID: + '368dffc7-7440-4ee0-aca5-11052d001b05'}, + {constants.LISTENER_ID: + 'd9c45ec4-9dbe-491b-9f21-6886562348bf'}] + pools = [{constants.POOL_ID: + '6886a40b-1f2a-41a3-9ece-5c51845a7ac4'}, + {constants.POOL_ID: + '08ada7a2-3eff-42c6-bdd8-b6f2ecd73358'}] lb = dmh.generate_load_balancer() - delete_flow, store = get_flow_method(lb) + if 'v2' in current_tuple[0]: + lb = utils.lb_dict_to_provider_dict(lb.to_dict()) + delete_flow = get_flow_method(lb, listeners, pools) + else: + delete_flow, store = get_flow_method(lb) current_engine = engines.load(delete_flow) elif (current_tuple[1] == 'LoadBalancerFlows' and current_tuple[2] == 'get_failover_LB_flow'): @@ -80,6 +100,13 @@ def generate(flow_list, output_directory): lb = dmh.generate_load_balancer( amphorae=[amp1, amp2], topology=constants.TOPOLOGY_ACTIVE_STANDBY) + if 'v2' in current_tuple[0]: + lb = utils.lb_dict_to_provider_dict(lb.to_dict()) + flavor = {constants.LOADBALANCER_TOPOLOGY: + constants.TOPOLOGY_ACTIVE_STANDBY} + lb[constants.FLAVOR] = flavor + amp1 = amp1.to_dict() + amp2 = amp2.to_dict() current_engine = engines.load( get_flow_method([amp1, amp2], lb)) elif (current_tuple[1] == 'MemberFlows' and diff --git a/tools/flow-list-v2.txt b/tools/flow-list-v2.txt new file mode 100644 index 0000000000..626d7055c8 --- /dev/null +++ b/tools/flow-list-v2.txt @@ -0,0 +1,32 @@ +# List of TaskFlow flows that should be documented +# Some flows are used by other flows, so just list the primary flows here +# Format: +# module class flow +octavia.controller.worker.v2.flows.amphora_flows AmphoraFlows get_create_amphora_flow +octavia.controller.worker.v2.flows.amphora_flows AmphoraFlows get_failover_amphora_flow +octavia.controller.worker.v2.flows.amphora_flows AmphoraFlows cert_rotate_amphora_flow +octavia.controller.worker.v2.flows.load_balancer_flows LoadBalancerFlows get_create_load_balancer_flow +octavia.controller.worker.v2.flows.load_balancer_flows LoadBalancerFlows get_delete_load_balancer_flow +octavia.controller.worker.v2.flows.load_balancer_flows LoadBalancerFlows get_cascade_delete_load_balancer_flow +octavia.controller.worker.v2.flows.load_balancer_flows LoadBalancerFlows get_update_load_balancer_flow +octavia.controller.worker.v2.flows.load_balancer_flows LoadBalancerFlows get_failover_LB_flow +octavia.controller.worker.v2.flows.listener_flows ListenerFlows get_create_listener_flow +octavia.controller.worker.v2.flows.listener_flows ListenerFlows get_create_all_listeners_flow +octavia.controller.worker.v2.flows.listener_flows ListenerFlows get_delete_listener_flow +octavia.controller.worker.v2.flows.listener_flows ListenerFlows get_update_listener_flow +octavia.controller.worker.v2.flows.pool_flows PoolFlows get_create_pool_flow +octavia.controller.worker.v2.flows.pool_flows PoolFlows get_delete_pool_flow +octavia.controller.worker.v2.flows.pool_flows PoolFlows get_update_pool_flow +octavia.controller.worker.v2.flows.member_flows MemberFlows get_create_member_flow +octavia.controller.worker.v2.flows.member_flows MemberFlows get_delete_member_flow +octavia.controller.worker.v2.flows.member_flows MemberFlows get_update_member_flow +octavia.controller.worker.v2.flows.member_flows MemberFlows get_batch_update_members_flow +octavia.controller.worker.v2.flows.health_monitor_flows HealthMonitorFlows get_create_health_monitor_flow +octavia.controller.worker.v2.flows.health_monitor_flows HealthMonitorFlows get_delete_health_monitor_flow +octavia.controller.worker.v2.flows.health_monitor_flows HealthMonitorFlows get_update_health_monitor_flow +octavia.controller.worker.v2.flows.l7policy_flows L7PolicyFlows get_create_l7policy_flow +octavia.controller.worker.v2.flows.l7policy_flows L7PolicyFlows get_delete_l7policy_flow +octavia.controller.worker.v2.flows.l7policy_flows L7PolicyFlows get_update_l7policy_flow +octavia.controller.worker.v2.flows.l7rule_flows L7RuleFlows get_create_l7rule_flow +octavia.controller.worker.v2.flows.l7rule_flows L7RuleFlows get_delete_l7rule_flow +octavia.controller.worker.v2.flows.l7rule_flows L7RuleFlows get_update_l7rule_flow