openstack/octavia
Code Issues Proposed changes

Merge "Update amphora v2 for the failover refactor"

Browse Source
This commit is contained in:
Zuul 2020-09-04 14:15:37 +00:00 committed by Gerrit Code Review
commit 46de66b240
32 changed files with 3742 additions and 1386 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -10,6 +10,7 @@ doc/build
doc/source/configuration/_static/octavia.policy.yaml.sample
doc/source/contributor/devref/erd.svg
doc/source/contributor/devref/flow_diagrams/
doc/source/contributor/devref/flow_diagrams_v2/
doc/source/contributor/modules
api-ref/build
.idea/*

2
doc/source/conf.py
View File

@ -29,6 +29,8 @@ from tools import create_flow_docs
# Generate our flow diagrams
create_flow_docs.generate(
'tools/flow-list.txt', 'doc/source/contributor/devref/flow_diagrams')
create_flow_docs.generate(
'tools/flow-list-v2.txt', 'doc/source/contributor/devref/flow_diagrams_v2')
# Generate entity relationship diagram
desc = sadisplay.describe(

14
doc/source/contributor/devref/flows.rst
View File

@ -19,3 +19,17 @@ Octavia controller.
flow_diagrams/LoadBalancerFlows.rst
flow_diagrams/MemberFlows.rst
flow_diagrams/PoolFlows.rst
The following are flow diagrams for the **amphora V2** driver.
.. toctree::
:maxdepth: 1
flow_diagrams_v2/AmphoraFlows.rst
flow_diagrams_v2/HealthMonitorFlows.rst
flow_diagrams_v2/L7PolicyFlows.rst
flow_diagrams_v2/L7RuleFlows.rst
flow_diagrams_v2/ListenerFlows.rst
flow_diagrams_v2/LoadBalancerFlows.rst
flow_diagrams_v2/MemberFlows.rst
flow_diagrams_v2/PoolFlows.rst

7
octavia/api/drivers/amphora_driver/v2/driver.py
View File

@ -87,8 +87,11 @@ class AmphoraProviderDriver(driver_base.ProviderDriver):
try:
vip = network_driver.allocate_vip(lb_obj)
except network_base.AllocateVIPException as e:
raise exceptions.DriverError(user_fault_string=e.orig_msg,
operator_fault_string=e.orig_msg)
message = str(e)
if getattr(e, 'orig_msg', None) is not None:
message = e.orig_msg
raise exceptions.DriverError(user_fault_string=message,
operator_fault_string=message)
LOG.info('Amphora provider created VIP port %s for load balancer %s.',
vip.port_id, loadbalancer_id)

1
octavia/common/constants.py
View File

@ -300,6 +300,7 @@ ALLOWED_ADDRESS_PAIRS = 'allowed_address_pairs'
AMP_DATA = 'amp_data'
AMP_VRRP_INT = 'amp_vrrp_int'
AMPHORA = 'amphora'
AMPHORA_DICT = 'amphora_dict'
AMPHORA_ID = 'amphora_id'
AMPHORA_INDEX = 'amphora_index'
AMPHORA_NETWORK_CONFIG = 'amphora_network_config'

13
octavia/controller/worker/v1/tasks/amphora_driver_tasks.py
View File

@ -122,7 +122,7 @@ class ListenersStart(BaseAmphoraTask):
class AmphoraIndexListenersReload(BaseAmphoraTask):
"""Task to reload all listeners on an amphora."""
def execute(self, loadbalancer, amphorae, amphora_index,
def execute(self, loadbalancer, amphora_index, amphorae,
timeout_dict=None):
"""Execute listener reload routines for listeners on an amphora."""
if loadbalancer.listeners:
@ -304,7 +304,7 @@ class AmphoraUpdateVRRPInterface(BaseAmphoraTask):
class AmphoraIndexUpdateVRRPInterface(BaseAmphoraTask):
"""Task to get and update the VRRP interface device name from amphora."""
def execute(self, amphorae, amphora_index, timeout_dict=None):
def execute(self, amphora_index, amphorae, timeout_dict=None):
amphora_id = amphorae[amphora_index].id
try:
interface = self.amphora_driver.get_interface_from_ip(
@ -376,15 +376,6 @@ class AmphoraIndexVRRPUpdate(BaseAmphoraTask):
LOG.debug("Uploaded VRRP configuration of amphora %s.", amphora_id)
class AmphoraVRRPStop(BaseAmphoraTask):
"""Task to stop keepalived of all amphorae of a LB."""
def execute(self, loadbalancer):
self.amphora_driver.stop_vrrp_service(loadbalancer)
LOG.debug("Stopped VRRP of loadbalancer %s amphorae",
loadbalancer.id)
class AmphoraVRRPStart(BaseAmphoraTask):
"""Task to start keepalived on an amphora.

426
octavia/controller/worker/v2/controller_worker.py
View File

@ -13,6 +13,7 @@
# under the License.
#
from octavia_lib.common import constants as lib_consts
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import excutils
@ -21,10 +22,12 @@ from stevedore import driver as stevedore_driver
from taskflow.listeners import logging as tf_logging
import tenacity
from octavia.amphorae.driver_exceptions import exceptions
from octavia.amphorae.driver_exceptions import exceptions as driver_exc
from octavia.api.drivers import utils as provider_utils
from octavia.common import base_taskflow
from octavia.common import constants
from octavia.common import exceptions
from octavia.common import utils
from octavia.controller.worker.v2.flows import flow_utils
from octavia.controller.worker.v2 import taskflow_jobboard_driver as tsk_driver
from octavia.db import api as db_apis
@ -33,17 +36,12 @@ from octavia.db import repositories as repo
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
RETRY_ATTEMPTS = 15
RETRY_INITIAL_DELAY = 1
RETRY_BACKOFF = 1
RETRY_MAX = 5
# We do not need to log retry exception information. Warning "Could not connect
# to instance" will be logged as usual.
def retryMaskFilter(record):
if record.exc_info is not None and isinstance(
record.exc_info[1], exceptions.AmpConnectionRetry):
record.exc_info[1], driver_exc.AmpConnectionRetry):
return False
return True
@ -87,8 +85,11 @@ class ControllerWorker(object):
tenacity.retry_if_result(_is_provisioning_status_pending_update) |
tenacity.retry_if_exception_type()),
wait=tenacity.wait_incrementing(
RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
CONF.haproxy_amphora.api_db_commit_retry_initial_delay,
CONF.haproxy_amphora.api_db_commit_retry_backoff,
CONF.haproxy_amphora.api_db_commit_retry_max),
stop=tenacity.stop_after_attempt(
CONF.haproxy_amphora.api_db_commit_retry_attempts))
def _get_db_obj_until_pending_update(self, repo, id):
return repo.get(db_apis.get_session(), id=id)
@ -117,6 +118,7 @@ class ControllerWorker(object):
store = {constants.BUILD_TYPE_PRIORITY:
constants.LB_CREATE_SPARES_POOL_PRIORITY,
constants.FLAVOR: None,
constants.SERVER_GROUP_ID: None,
constants.AVAILABILITY_ZONE: None}
if availability_zone:
store[constants.AVAILABILITY_ZONE] = (
@ -145,8 +147,11 @@ class ControllerWorker(object):
@tenacity.retry(
retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound),
wait=tenacity.wait_incrementing(
RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
CONF.haproxy_amphora.api_db_commit_retry_initial_delay,
CONF.haproxy_amphora.api_db_commit_retry_backoff,
CONF.haproxy_amphora.api_db_commit_retry_max),
stop=tenacity.stop_after_attempt(
CONF.haproxy_amphora.api_db_commit_retry_attempts))
def create_health_monitor(self, health_monitor):
"""Creates a health monitor.
@ -251,8 +256,11 @@ class ControllerWorker(object):
@tenacity.retry(
retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound),
wait=tenacity.wait_incrementing(
RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
CONF.haproxy_amphora.api_db_commit_retry_initial_delay,
CONF.haproxy_amphora.api_db_commit_retry_backoff,
CONF.haproxy_amphora.api_db_commit_retry_max),
stop=tenacity.stop_after_attempt(
CONF.haproxy_amphora.api_db_commit_retry_attempts))
def create_listener(self, listener):
"""Creates a listener.
@ -292,14 +300,10 @@ class ControllerWorker(object):
:returns: None
:raises ListenerNotFound: The referenced listener was not found
"""
# TODO(johnsom) Remove once the provider data model includes
# the project ID
lb = self._lb_repo.get(db_apis.get_session(),
id=listener[constants.LOADBALANCER_ID])
store = {constants.LISTENER: listener,
constants.LOADBALANCER_ID:
listener[constants.LOADBALANCER_ID],
constants.PROJECT_ID: lb.project_id}
constants.PROJECT_ID: listener[constants.PROJECT_ID]}
self.run_flow(
flow_utils.get_delete_listener_flow,
store=store)
@ -325,8 +329,11 @@ class ControllerWorker(object):
@tenacity.retry(
retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound),
wait=tenacity.wait_incrementing(
RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
CONF.haproxy_amphora.api_db_commit_retry_initial_delay,
CONF.haproxy_amphora.api_db_commit_retry_backoff,
CONF.haproxy_amphora.api_db_commit_retry_max),
stop=tenacity.stop_after_attempt(
CONF.haproxy_amphora.api_db_commit_retry_attempts))
def create_load_balancer(self, loadbalancer, flavor=None,
availability_zone=None):
"""Creates a load balancer by allocating Amphorae.
@ -347,16 +354,18 @@ class ControllerWorker(object):
loadbalancer[constants.LOADBALANCER_ID])
raise db_exceptions.NoResultFound
# TODO(johnsom) convert this to octavia_lib constant flavor
# once octavia is transitioned to use octavia_lib
store = {constants.LOADBALANCER_ID:
loadbalancer[constants.LOADBALANCER_ID],
store = {lib_consts.LOADBALANCER_ID:
loadbalancer[lib_consts.LOADBALANCER_ID],
constants.BUILD_TYPE_PRIORITY:
constants.LB_CREATE_NORMAL_PRIORITY,
constants.FLAVOR: flavor,
constants.AVAILABILITY_ZONE: availability_zone}
lib_consts.FLAVOR: flavor,
lib_consts.AVAILABILITY_ZONE: availability_zone}
topology = lb.topology
if (not CONF.nova.enable_anti_affinity or
topology == constants.TOPOLOGY_SINGLE):
store[constants.SERVER_GROUP_ID] = None
listeners_dicts = (
provider_utils.db_listeners_to_provider_dicts_list_of_dicts(
lb.listeners)
@ -377,17 +386,19 @@ class ControllerWorker(object):
:returns: None
:raises LBNotFound: The referenced load balancer was not found
"""
db_lb = self._lb_repo.get(db_apis.get_session(),
id=load_balancer[constants.LOADBALANCER_ID])
loadbalancer_id = load_balancer[constants.LOADBALANCER_ID]
db_lb = self._lb_repo.get(db_apis.get_session(), id=loadbalancer_id)
store = {constants.LOADBALANCER: load_balancer,
constants.LOADBALANCER_ID: loadbalancer_id,
constants.SERVER_GROUP_ID: db_lb.server_group_id,
constants.PROJECT_ID: db_lb.project_id}
if cascade:
store.update(flow_utils.get_delete_pools_store(db_lb))
store.update(flow_utils.get_delete_listeners_store(db_lb))
listeners = flow_utils.get_listeners_on_lb(db_lb)
pools = flow_utils.get_pools_on_lb(db_lb)
self.run_flow(
flow_utils.get_cascade_delete_load_balancer_flow,
load_balancer, store=store)
load_balancer, listeners, pools, store=store)
else:
self.run_flow(
flow_utils.get_delete_load_balancer_flow,
@ -548,7 +559,6 @@ class ControllerWorker(object):
listeners_dicts = (
provider_utils.db_listeners_to_provider_dicts_list_of_dicts(
pool.listeners))
store = {
constants.MEMBER: member,
constants.LISTENERS: listeners_dicts,
@ -570,8 +580,11 @@ class ControllerWorker(object):
@tenacity.retry(
retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound),
wait=tenacity.wait_incrementing(
RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
CONF.haproxy_amphora.api_db_commit_retry_initial_delay,
CONF.haproxy_amphora.api_db_commit_retry_backoff,
CONF.haproxy_amphora.api_db_commit_retry_max),
stop=tenacity.stop_after_attempt(
CONF.haproxy_amphora.api_db_commit_retry_attempts))
def create_pool(self, pool):
"""Creates a node pool.
@ -816,152 +829,249 @@ class ControllerWorker(object):
flow_utils.get_update_l7rule_flow,
store=store)
def _perform_amphora_failover(self, amp, priority):
"""Internal method to perform failover operations for an amphora.
def failover_amphora(self, amphora_id):
"""Perform failover operations for an amphora.
:param amp: The amphora to failover
:param priority: The create priority
Note: This expects the load balancer to already be in
provisioning_status=PENDING_UPDATE state.
:param amphora_id: ID for amphora to failover
:returns: None
:raises octavia.common.exceptions.NotFound: The referenced amphora was
not found
"""
stored_params = {constants.FAILED_AMPHORA: amp.to_dict(),
constants.LOADBALANCER_ID: amp.load_balancer_id,
constants.BUILD_TYPE_PRIORITY: priority, }
amphora = None
try:
amphora = self._amphora_repo.get(db_apis.get_session(),
id=amphora_id)
if amphora is None:
LOG.error('Amphora failover for amphora %s failed because '
'there is no record of this amphora in the '
'database. Check that the [house_keeping] '
'amphora_expiry_age configuration setting is not '
'too short. Skipping failover.', amphora_id)
raise exceptions.NotFound(resource=constants.AMPHORA,
id=amphora_id)
if amp.role in (constants.ROLE_MASTER, constants.ROLE_BACKUP):
amp_role = 'master_or_backup'
elif amp.role == constants.ROLE_STANDALONE:
amp_role = 'standalone'
elif amp.role is None:
amp_role = 'spare'
else:
amp_role = 'undefined'
LOG.info("Perform failover for an amphora: %s",
{"id": amp.id,
"load_balancer_id": amp.load_balancer_id,
"lb_network_ip": amp.lb_network_ip,
"compute_id": amp.compute_id,
"role": amp_role})
if amp.status == constants.DELETED:
if amphora.status == constants.DELETED:
LOG.warning('Amphora %s is marked DELETED in the database but '
'was submitted for failover. Deleting it from the '
'amphora health table to exclude it from health '
'checks and skipping the failover.', amp.id)
'checks and skipping the failover.', amphora.id)
self._amphora_health_repo.delete(db_apis.get_session(),
amphora_id=amp.id)
amphora_id=amphora.id)
return
if (CONF.house_keeping.spare_amphora_pool_size == 0) and (
CONF.nova.enable_anti_affinity is False):
LOG.warning("Failing over amphora with no spares pool may "
"cause delays in failover times while a new "
"amphora instance boots.")
loadbalancer = None
if amphora.load_balancer_id:
loadbalancer = self._lb_repo.get(db_apis.get_session(),
id=amphora.load_balancer_id)
lb_amp_count = None
if loadbalancer:
if loadbalancer.topology == constants.TOPOLOGY_ACTIVE_STANDBY:
lb_amp_count = 2
elif loadbalancer.topology == constants.TOPOLOGY_SINGLE:
lb_amp_count = 1
# if we run with anti-affinity we need to set the server group
# as well
lb = self._amphora_repo.get_lb_for_amphora(
db_apis.get_session(), amp.id)
provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer(
lb).to_dict() if lb else lb
if CONF.nova.enable_anti_affinity and lb:
stored_params[constants.SERVER_GROUP_ID] = lb.server_group_id
if lb is not None and lb.flavor_id:
stored_params[constants.FLAVOR] = (
self._flavor_repo.get_flavor_metadata_dict(
db_apis.get_session(), lb.flavor_id))
az_metadata = {}
flavor_dict = {}
lb_id = None
vip_dict = {}
server_group_id = None
if loadbalancer:
lb_id = loadbalancer.id
# Even if the LB doesn't have a flavor, create one and
# pass through the topology.
if loadbalancer.flavor_id:
flavor_dict = self._flavor_repo.get_flavor_metadata_dict(
db_apis.get_session(), loadbalancer.flavor_id)
flavor_dict[constants.LOADBALANCER_TOPOLOGY] = (
loadbalancer.topology)
else:
stored_params[constants.FLAVOR] = {}
if lb and lb.availability_zone:
flavor_dict = {constants.LOADBALANCER_TOPOLOGY:
loadbalancer.topology}
if loadbalancer.availability_zone:
az_metadata = (
self._az_repo.get_availability_zone_metadata_dict(
db_apis.get_session(),
loadbalancer.availability_zone))
vip_dict = loadbalancer.vip.to_dict()
server_group_id = loadbalancer.server_group_id
provider_lb_dict = (provider_utils.
db_loadbalancer_to_provider_loadbalancer)(
loadbalancer).to_dict() if loadbalancer else loadbalancer
stored_params = {constants.AVAILABILITY_ZONE: az_metadata,
constants.BUILD_TYPE_PRIORITY:
constants.LB_CREATE_FAILOVER_PRIORITY,
constants.FLAVOR: flavor_dict,
constants.LOADBALANCER: provider_lb_dict,
constants.SERVER_GROUP_ID: server_group_id,
constants.LOADBALANCER_ID: lb_id,
constants.VIP: vip_dict}
self.services_controller.run_poster(
flow_utils.get_failover_amphora_flow,
amphora.to_dict(), lb_amp_count,
store=stored_params, wait=True)
LOG.info("Successfully completed the failover for an amphora: %s",
{"id": amphora_id,
"load_balancer_id": lb_id,
"lb_network_ip": amphora.lb_network_ip,
"compute_id": amphora.compute_id,
"role": amphora.role})
except Exception as e:
with excutils.save_and_reraise_exception(reraise=False):
LOG.exception("Amphora %s failover exception: %s",
amphora_id, str(e))
self._amphora_repo.update(db_apis.get_session(),
amphora_id, status=constants.ERROR)
if amphora and amphora.load_balancer_id:
self._lb_repo.update(
db_apis.get_session(), amphora.load_balancer_id,
provisioning_status=constants.ERROR)
@staticmethod
def _get_amphorae_for_failover(load_balancer):
"""Returns an ordered list of amphora to failover.
:param load_balancer: The load balancer being failed over.
:returns: An ordered list of amphora to failover,
first amp to failover is last in the list
:raises octavia.common.exceptions.InvalidTopology: LB has an unknown
topology.
"""
if load_balancer.topology == constants.TOPOLOGY_SINGLE:
# In SINGLE topology, amp failover order does not matter
return [a.to_dict() for a in load_balancer.amphorae
if a.status != constants.DELETED]
if load_balancer.topology == constants.TOPOLOGY_ACTIVE_STANDBY:
# In Active/Standby we should preference the standby amp
# for failover first in case the Active is still able to pass
# traffic.
# Note: The active amp can switch at any time and in less than a
# second, so this is "best effort".
amphora_driver = utils.get_amphora_driver()
timeout_dict = {
constants.CONN_MAX_RETRIES:
CONF.haproxy_amphora.failover_connection_max_retries,
constants.CONN_RETRY_INTERVAL:
CONF.haproxy_amphora.failover_connection_retry_interval}
amps = []
selected_amp = None
for amp in load_balancer.amphorae:
if amp.status == constants.DELETED:
continue
if selected_amp is None:
try:
if amphora_driver.get_interface_from_ip(
amp, load_balancer.vip.ip_address,
timeout_dict):
# This is a potential ACTIVE, add it to the list
amps.append(amp.to_dict())
else:
# This one doesn't have the VIP IP, so start
# failovers here.
selected_amp = amp
LOG.debug("Selected amphora %s as the initial "
"failover amphora.", amp.id)
except Exception:
# This amphora is broken, so start failovers here.
selected_amp = amp
else:
# We have already found a STANDBY, so add the rest to the
# list without querying them.
amps.append(amp.to_dict())
# Put the selected amphora at the end of the list so it is
# first to failover.
if selected_amp:
amps.append(selected_amp.to_dict())
return amps
LOG.error('Unknown load balancer topology found: %s, aborting '
'failover.', load_balancer.topology)
raise exceptions.InvalidTopology(topology=load_balancer.topology)
def failover_loadbalancer(self, load_balancer_id):
"""Perform failover operations for a load balancer.
Note: This expects the load balancer to already be in
provisioning_status=PENDING_UPDATE state.
:param load_balancer_id: ID for load balancer to failover
:returns: None
:raises octavia.commom.exceptions.NotFound: The load balancer was not
found.
"""
try:
lb = self._lb_repo.get(db_apis.get_session(),
id=load_balancer_id)
if lb is None:
raise exceptions.NotFound(resource=constants.LOADBALANCER,
id=load_balancer_id)
# Get the ordered list of amphorae to failover for this LB.
amps = self._get_amphorae_for_failover(lb)
if lb.topology == constants.TOPOLOGY_SINGLE:
if len(amps) != 1:
LOG.warning('%d amphorae found on load balancer %s where '
'one should exist. Repairing.', len(amps),
load_balancer_id)
elif lb.topology == constants.TOPOLOGY_ACTIVE_STANDBY:
if len(amps) != 2:
LOG.warning('%d amphorae found on load balancer %s where '
'two should exist. Repairing.', len(amps),
load_balancer_id)
else:
LOG.error('Unknown load balancer topology found: %s, aborting '
'failover!', lb.topology)
raise exceptions.InvalidTopology(topology=lb.topology)
# We must provide a topology in the flavor definition
# here for the amphora to be created with the correct
# configuration.
if lb.flavor_id:
flavor = self._flavor_repo.get_flavor_metadata_dict(
db_apis.get_session(), lb.flavor_id)
flavor[constants.LOADBALANCER_TOPOLOGY] = lb.topology
else:
flavor = {constants.LOADBALANCER_TOPOLOGY: lb.topology}
provider_lb_dict = (
provider_utils.db_loadbalancer_to_provider_loadbalancer(
lb).to_dict() if lb else lb)
provider_lb_dict[constants.FLAVOR] = flavor
stored_params = {constants.LOADBALANCER: provider_lb_dict,
constants.BUILD_TYPE_PRIORITY:
constants.LB_CREATE_FAILOVER_PRIORITY,
constants.SERVER_GROUP_ID: lb.server_group_id,
constants.LOADBALANCER_ID: lb.id,
constants.FLAVOR: flavor}
if lb.availability_zone:
stored_params[constants.AVAILABILITY_ZONE] = (
self._az_repo.get_availability_zone_metadata_dict(
db_apis.get_session(), lb.availability_zone))
else:
stored_params[constants.AVAILABILITY_ZONE] = {}
self.run_flow(
flow_utils.get_failover_flow,
role=amp.role, load_balancer=provider_lb,
self.services_controller.run_poster(
flow_utils.get_failover_LB_flow, amps, provider_lb_dict,
store=stored_params, wait=True)
LOG.info("Successfully completed the failover for an amphora: %s",
{"id": amp.id,
"load_balancer_id": amp.load_balancer_id,
"lb_network_ip": amp.lb_network_ip,
"compute_id": amp.compute_id,
"role": amp_role})
def failover_amphora(self, amphora_id):
"""Perform failover operations for an amphora.
:param amphora_id: ID for amphora to failover
:returns: None
:raises AmphoraNotFound: The referenced amphora was not found
"""
try:
amp = self._amphora_repo.get(db_apis.get_session(),
id=amphora_id)
if not amp:
LOG.warning("Could not fetch Amphora %s from DB, ignoring "
"failover request.", amphora_id)
return
self._perform_amphora_failover(
amp, constants.LB_CREATE_FAILOVER_PRIORITY)
if amp.load_balancer_id:
LOG.info("Mark ACTIVE in DB for load balancer id: %s",
amp.load_balancer_id)
self._lb_repo.update(
db_apis.get_session(), amp.load_balancer_id,
provisioning_status=constants.ACTIVE)
except Exception as e:
try:
self._lb_repo.update(
db_apis.get_session(), amp.load_balancer_id,
provisioning_status=constants.ERROR)
except Exception:
LOG.error("Unable to revert LB status to ERROR.")
with excutils.save_and_reraise_exception():
LOG.error("Amphora %(id)s failover exception: %(exc)s",
{'id': amphora_id, 'exc': e})
def failover_loadbalancer(self, load_balancer_id):
"""Perform failover operations for a load balancer.
:param load_balancer_id: ID for load balancer to failover
:returns: None
:raises LBNotFound: The referenced load balancer was not found
"""
# Note: This expects that the load balancer is already in
# provisioning_status=PENDING_UPDATE state
try:
lb = self._lb_repo.get(db_apis.get_session(),
id=load_balancer_id)
# Exclude amphora already deleted
amps = [a for a in lb.amphorae if a.status != constants.DELETED]
for amp in amps:
# failover amphora in backup role
# Note: this amp may not currently be the backup
# TODO(johnsom) Change this to query the amp state
# once the amp API supports it.
if amp.role == constants.ROLE_BACKUP:
self._perform_amphora_failover(
amp, constants.LB_CREATE_ADMIN_FAILOVER_PRIORITY)
for amp in amps:
# failover everyhting else
if amp.role != constants.ROLE_BACKUP:
self._perform_amphora_failover(
amp, constants.LB_CREATE_ADMIN_FAILOVER_PRIORITY)
self._lb_repo.update(
db_apis.get_session(), load_balancer_id,
provisioning_status=constants.ACTIVE)
LOG.info('Failover of load balancer %s completed successfully.',
lb.id)
except Exception as e:
with excutils.save_and_reraise_exception():
LOG.error("LB %(lbid)s failover exception: %(exc)s",
with excutils.save_and_reraise_exception(reraise=False):
LOG.exception("LB %(lbid)s failover exception: %(exc)s",
{'lbid': load_balancer_id, 'exc': e})
self._lb_repo.update(
db_apis.get_session(), load_balancer_id,

742
octavia/controller/worker/v2/flows/amphora_flows.py
View File

@ -1,4 +1,5 @@
# Copyright 2015 Hewlett-Packard Development Company, L.P.
# Copyright 2020 Red Hat, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -14,28 +15,27 @@
#
from oslo_config import cfg
from oslo_log import log as logging
from taskflow.patterns import graph_flow
from taskflow.patterns import linear_flow
from taskflow.patterns import unordered_flow
from octavia.common import constants
from octavia.common import utils
from octavia.controller.worker.v2.tasks import amphora_driver_tasks
from octavia.controller.worker.v2.tasks import cert_task
from octavia.controller.worker.v2.tasks import compute_tasks
from octavia.controller.worker.v2.tasks import database_tasks
from octavia.controller.worker.v2.tasks import lifecycle_tasks
from octavia.controller.worker.v2.tasks import network_tasks
from octavia.db import api as db_apis
from octavia.db import repositories as repo
from octavia.controller.worker.v2.tasks import retry_tasks
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class AmphoraFlows(object):
def __init__(self):
self.lb_repo = repo.LoadBalancerRepository()
def get_create_amphora_flow(self):
"""Creates a flow to create an amphora.
@ -46,24 +46,16 @@ class AmphoraFlows(object):
provides=constants.AMPHORA_ID))
create_amphora_flow.add(lifecycle_tasks.AmphoraIDToErrorOnRevertTask(
requires=constants.AMPHORA_ID))
if (CONF.controller_worker.amphora_driver ==
'amphora_haproxy_rest_driver'):
create_amphora_flow.add(cert_task.GenerateServerPEMTask(
provides=constants.SERVER_PEM))
create_amphora_flow.add(
database_tasks.UpdateAmphoraDBCertExpiration(
requires=(constants.AMPHORA_ID, constants.SERVER_PEM)))
create_amphora_flow.add(compute_tasks.CertComputeCreate(
requires=(constants.AMPHORA_ID, constants.SERVER_PEM,
constants.SERVER_GROUP_ID,
constants.BUILD_TYPE_PRIORITY, constants.FLAVOR),
provides=constants.COMPUTE_ID))
else:
create_amphora_flow.add(compute_tasks.ComputeCreate(
requires=(constants.AMPHORA_ID, constants.BUILD_TYPE_PRIORITY,
constants.FLAVOR),
provides=constants.COMPUTE_ID))
create_amphora_flow.add(database_tasks.MarkAmphoraBootingInDB(
requires=(constants.AMPHORA_ID, constants.COMPUTE_ID)))
create_amphora_flow.add(compute_tasks.ComputeActiveWait(
@ -97,11 +89,6 @@ class AmphoraFlows(object):
post_map_amp_to_lb = linear_flow.Flow(
sf_name)
post_map_amp_to_lb.add(database_tasks.ReloadAmphora(
name=sf_name + '-' + constants.RELOAD_AMPHORA,
requires=constants.AMPHORA,
provides=constants.AMPHORA))
post_map_amp_to_lb.add(amphora_driver_tasks.AmphoraConfigUpdate(
name=sf_name + '-' + constants.AMPHORA_CONFIG_UPDATE_TASK,
requires=(constants.AMPHORA, constants.FLAVOR)))
@ -121,7 +108,7 @@ class AmphoraFlows(object):
return post_map_amp_to_lb
def _get_create_amp_for_lb_subflow(self, prefix, role):
def _get_create_amp_for_lb_subflow(self, prefix, role, is_spare=False):
"""Create a new amphora for lb."""
sf_name = prefix + '-' + constants.CREATE_AMP_FOR_LB_SUBFLOW
@ -131,12 +118,6 @@ class AmphoraFlows(object):
requires=constants.LOADBALANCER_ID,
provides=constants.AMPHORA_ID))
require_server_group_id_condition = (
role in (constants.ROLE_BACKUP, constants.ROLE_MASTER) and
CONF.nova.enable_anti_affinity)
if (CONF.controller_worker.amphora_driver ==
'amphora_haproxy_rest_driver'):
create_amp_for_lb_subflow.add(cert_task.GenerateServerPEMTask(
name=sf_name + '-' + constants.GENERATE_SERVER_PEM,
provides=constants.SERVER_PEM))
@ -146,52 +127,13 @@ class AmphoraFlows(object):
name=sf_name + '-' + constants.UPDATE_CERT_EXPIRATION,
requires=(constants.AMPHORA_ID, constants.SERVER_PEM)))
if require_server_group_id_condition:
create_amp_for_lb_subflow.add(compute_tasks.CertComputeCreate(
name=sf_name + '-' + constants.CERT_COMPUTE_CREATE,
requires=(
constants.AMPHORA_ID,
constants.SERVER_PEM,
requires=(constants.AMPHORA_ID, constants.SERVER_PEM,
constants.BUILD_TYPE_PRIORITY,
constants.SERVER_GROUP_ID,
constants.FLAVOR,
constants.AVAILABILITY_ZONE,
),
constants.FLAVOR, constants.AVAILABILITY_ZONE),
provides=constants.COMPUTE_ID))
else:
create_amp_for_lb_subflow.add(compute_tasks.CertComputeCreate(
name=sf_name + '-' + constants.CERT_COMPUTE_CREATE,
requires=(
constants.AMPHORA_ID,
constants.SERVER_PEM,
constants.BUILD_TYPE_PRIORITY,
constants.FLAVOR,
constants.AVAILABILITY_ZONE,
),
provides=constants.COMPUTE_ID))
else:
if require_server_group_id_condition:
create_amp_for_lb_subflow.add(compute_tasks.ComputeCreate(
name=sf_name + '-' + constants.COMPUTE_CREATE,
requires=(
constants.AMPHORA_ID,
constants.BUILD_TYPE_PRIORITY,
constants.SERVER_GROUP_ID,
constants.FLAVOR,
constants.AVAILABILITY_ZONE,
),
provides=constants.COMPUTE_ID))
else:
create_amp_for_lb_subflow.add(compute_tasks.ComputeCreate(
name=sf_name + '-' + constants.COMPUTE_CREATE,
requires=(
constants.AMPHORA_ID,
constants.BUILD_TYPE_PRIORITY,
constants.FLAVOR,
constants.AVAILABILITY_ZONE,
),
provides=constants.COMPUTE_ID))
create_amp_for_lb_subflow.add(database_tasks.UpdateAmphoraComputeId(
name=sf_name + '-' + constants.UPDATE_AMPHORA_COMPUTEID,
requires=(constants.AMPHORA_ID, constants.COMPUTE_ID)))
@ -207,6 +149,33 @@ class AmphoraFlows(object):
name=sf_name + '-' + constants.UPDATE_AMPHORA_INFO,
requires=(constants.AMPHORA_ID, constants.COMPUTE_OBJ),
provides=constants.AMPHORA))
create_amp_for_lb_subflow.add(self._retry_flow(sf_name))
create_amp_for_lb_subflow.add(amphora_driver_tasks.AmphoraFinalize(
name=sf_name + '-' + constants.AMPHORA_FINALIZE,
requires=constants.AMPHORA))
if is_spare:
create_amp_for_lb_subflow.add(
database_tasks.MarkAmphoraReadyInDB(
name=sf_name + '-' + constants.MARK_AMPHORA_READY_INDB,
requires=constants.AMPHORA))
else:
create_amp_for_lb_subflow.add(
database_tasks.MarkAmphoraAllocatedInDB(
name=sf_name + '-' + constants.MARK_AMPHORA_ALLOCATED_INDB,
requires=(constants.AMPHORA, constants.LOADBALANCER_ID)))
if role == constants.ROLE_MASTER:
create_amp_for_lb_subflow.add(database_tasks.MarkAmphoraMasterInDB(
name=sf_name + '-' + constants.MARK_AMP_MASTER_INDB,
requires=constants.AMPHORA))
elif role == constants.ROLE_BACKUP:
create_amp_for_lb_subflow.add(database_tasks.MarkAmphoraBackupInDB(
name=sf_name + '-' + constants.MARK_AMP_BACKUP_INDB,
requires=constants.AMPHORA))
elif role == constants.ROLE_STANDALONE:
create_amp_for_lb_subflow.add(
database_tasks.MarkAmphoraStandAloneInDB(
name=sf_name + '-' + constants.MARK_AMP_STANDALONE_INDB,
requires=constants.AMPHORA))
return create_amp_for_lb_subflow
@ -268,7 +237,7 @@ class AmphoraFlows(object):
return create_amp_for_lb_subflow
def get_amphora_for_lb_subflow(
self, prefix, role=constants.ROLE_STANDALONE):
self, prefix, role=constants.ROLE_STANDALONE, is_spare=False):
"""Tries to allocate a spare amphora to a loadbalancer if none
exists, create a new amphora.
@ -276,6 +245,14 @@ class AmphoraFlows(object):
sf_name = prefix + '-' + constants.GET_AMPHORA_FOR_LB_SUBFLOW
# Don't replace a spare with another spare, just build a fresh one.
if is_spare:
get_spare_amp_flow = linear_flow.Flow(sf_name)
get_spare_amp_flow.add(self._get_create_amp_for_lb_subflow(
prefix, role, is_spare=is_spare))
return get_spare_amp_flow
# We need a graph flow here for a conditional flow
amp_for_lb_flow = graph_flow.Flow(sf_name)
@ -318,280 +295,136 @@ class AmphoraFlows(object):
decider=self._create_new_amp_for_lb_decider,
decider_depth='flow')
# Plug the network
# todo(xgerman): Rework failover flow
if prefix != constants.FAILOVER_AMPHORA_FLOW:
sf_name = prefix + '-' + constants.AMP_PLUG_NET_SUBFLOW
amp_for_lb_net_flow = linear_flow.Flow(sf_name)
amp_for_lb_net_flow.add(amp_for_lb_flow)
amp_for_lb_net_flow.add(*self._get_amp_net_subflow(sf_name))
return amp_for_lb_net_flow
return amp_for_lb_flow
def _get_amp_net_subflow(self, sf_name):
flows = []
flows.append(network_tasks.PlugVIPAmpphora(
name=sf_name + '-' + constants.PLUG_VIP_AMPHORA,
requires=(constants.LOADBALANCER, constants.AMPHORA,
constants.SUBNET),
provides=constants.AMP_DATA))
def get_delete_amphora_flow(
self, amphora,
retry_attempts=CONF.controller_worker.amphora_delete_retries,
retry_interval=(
CONF.controller_worker.amphora_delete_retry_interval)):
"""Creates a subflow to delete an amphora and it's port.
flows.append(network_tasks.ApplyQosAmphora(
name=sf_name + '-' + constants.APPLY_QOS_AMP,
requires=(constants.LOADBALANCER, constants.AMP_DATA,
constants.UPDATE_DICT)))
flows.append(database_tasks.UpdateAmphoraVIPData(
name=sf_name + '-' + constants.UPDATE_AMPHORA_VIP_DATA,
requires=constants.AMP_DATA))
flows.append(database_tasks.ReloadAmphora(
name=sf_name + '-' + constants.RELOAD_AMP_AFTER_PLUG_VIP,
requires=constants.AMPHORA,
provides=constants.AMPHORA))
flows.append(database_tasks.ReloadLoadBalancer(
name=sf_name + '-' + constants.RELOAD_LB_AFTER_PLUG_VIP,
requires=constants.LOADBALANCER_ID,
provides=constants.LOADBALANCER))
flows.append(network_tasks.GetAmphoraNetworkConfigs(
name=sf_name + '-' + constants.GET_AMP_NETWORK_CONFIG,
requires=(constants.LOADBALANCER, constants.AMPHORA),
provides=constants.AMPHORA_NETWORK_CONFIG))
flows.append(amphora_driver_tasks.AmphoraPostVIPPlug(
name=sf_name + '-' + constants.AMP_POST_VIP_PLUG,
rebind={constants.AMPHORAE_NETWORK_CONFIG:
constants.AMPHORA_NETWORK_CONFIG},
requires=(constants.LOADBALANCER,
constants.AMPHORAE_NETWORK_CONFIG)))
return flows
This flow is idempotent and safe to retry.
def get_delete_amphora_flow(self):
"""Creates a flow to delete an amphora.
This should be configurable in the config file
:returns: The flow for deleting the amphora
:raises AmphoraNotFound: The referenced Amphora was not found
:param amphora: An amphora dict object.
:param retry_attempts: The number of times the flow is retried.
:param retry_interval: The time to wait, in seconds, between retries.
:returns: The subflow for deleting the amphora.
:raises AmphoraNotFound: The referenced Amphora was not found.
"""
delete_amphora_flow = linear_flow.Flow(constants.DELETE_AMPHORA_FLOW)
amphora_id = amphora[constants.ID]
delete_amphora_flow = linear_flow.Flow(
name=constants.DELETE_AMPHORA_FLOW + '-' + amphora_id,
retry=retry_tasks.SleepingRetryTimesController(
name='retry-' + constants.DELETE_AMPHORA_FLOW + '-' +
amphora_id,
attempts=retry_attempts, interval=retry_interval))
delete_amphora_flow.add(lifecycle_tasks.AmphoraToErrorOnRevertTask(
requires=constants.AMPHORA))
delete_amphora_flow.add(database_tasks.
MarkAmphoraPendingDeleteInDB(
requires=constants.AMPHORA))
delete_amphora_flow.add(database_tasks.
MarkAmphoraHealthBusy(
requires=constants.AMPHORA))
name=constants.AMPHORA_TO_ERROR_ON_REVERT + '-' + amphora_id,
inject={constants.AMPHORA: amphora}))
delete_amphora_flow.add(
database_tasks.MarkAmphoraPendingDeleteInDB(
name=constants.MARK_AMPHORA_PENDING_DELETE + '-' + amphora_id,
inject={constants.AMPHORA: amphora}))
delete_amphora_flow.add(database_tasks.MarkAmphoraHealthBusy(
name=constants.MARK_AMPHORA_HEALTH_BUSY + '-' + amphora_id,
inject={constants.AMPHORA: amphora}))
delete_amphora_flow.add(compute_tasks.ComputeDelete(
requires=constants.AMPHORA))
delete_amphora_flow.add(database_tasks.
DisableAmphoraHealthMonitoring(
requires=constants.AMPHORA))
delete_amphora_flow.add(database_tasks.
MarkAmphoraDeletedInDB(
requires=constants.AMPHORA))
name=constants.DELETE_AMPHORA + '-' + amphora_id,
inject={constants.AMPHORA: amphora,
constants.PASSIVE_FAILURE: True}))
delete_amphora_flow.add(database_tasks.DisableAmphoraHealthMonitoring(
name=constants.DISABLE_AMP_HEALTH_MONITORING + '-' + amphora_id,
inject={constants.AMPHORA: amphora}))
delete_amphora_flow.add(database_tasks.MarkAmphoraDeletedInDB(
name=constants.MARK_AMPHORA_DELETED + '-' + amphora_id,
inject={constants.AMPHORA: amphora}))
if amphora.get(constants.VRRP_PORT_ID):
delete_amphora_flow.add(network_tasks.DeletePort(
name=(constants.DELETE_PORT + '-' + str(amphora_id) + '-' +
str(amphora[constants.VRRP_PORT_ID])),
inject={constants.PORT_ID: amphora[constants.VRRP_PORT_ID],
constants.PASSIVE_FAILURE: True}))
# TODO(johnsom) What about cleaning up any member ports?
# maybe we should get the list of attached ports prior to delete
# and call delete on them here. Fix this as part of
# https://storyboard.openstack.org/#!/story/2007077
return delete_amphora_flow
def get_failover_flow(self, role=constants.ROLE_STANDALONE,
load_balancer=None):
"""Creates a flow to failover a stale amphora
:returns: The flow for amphora failover
"""
failover_amphora_flow = linear_flow.Flow(
constants.FAILOVER_AMPHORA_FLOW)
failover_amphora_flow.add(lifecycle_tasks.AmphoraToErrorOnRevertTask(
rebind={constants.AMPHORA: constants.FAILED_AMPHORA},
requires=constants.AMPHORA))
failover_amphora_flow.add(network_tasks.FailoverPreparationForAmphora(
rebind={constants.AMPHORA: constants.FAILED_AMPHORA},
requires=constants.AMPHORA))
# Note: It seems intuitive to boot an amphora prior to deleting
# the old amphora, however this is a complicated issue.
# If the target host (due to anit-affinity) is resource
# constrained, this will fail where a post-delete will
# succeed. Since this is async with the API it would result
# in the LB ending in ERROR though the amps are still alive.
# Consider in the future making this a complicated
# try-on-failure-retry flow, or move upgrade failovers to be
# synchronous with the API. For now spares pool and act/stdby
# will mitigate most of this delay.
# Delete the old amphora
failover_amphora_flow.add(
database_tasks.MarkAmphoraPendingDeleteInDB(
rebind={constants.AMPHORA: constants.FAILED_AMPHORA},
requires=constants.AMPHORA))
failover_amphora_flow.add(
database_tasks.MarkAmphoraHealthBusy(
rebind={constants.AMPHORA: constants.FAILED_AMPHORA},
requires=constants.AMPHORA))
failover_amphora_flow.add(compute_tasks.ComputeDelete(
rebind={constants.AMPHORA: constants.FAILED_AMPHORA},
requires=constants.AMPHORA))
failover_amphora_flow.add(network_tasks.WaitForPortDetach(
rebind={constants.AMPHORA: constants.FAILED_AMPHORA},
requires=constants.AMPHORA))
failover_amphora_flow.add(database_tasks.MarkAmphoraDeletedInDB(
rebind={constants.AMPHORA: constants.FAILED_AMPHORA},
requires=constants.AMPHORA))
# If this is an unallocated amp (spares pool), we're done
if not load_balancer:
failover_amphora_flow.add(
database_tasks.DisableAmphoraHealthMonitoring(
rebind={constants.AMPHORA: constants.FAILED_AMPHORA},
requires=constants.AMPHORA))
return failover_amphora_flow
# Save failed amphora details for later
failover_amphora_flow.add(
database_tasks.GetAmphoraDetails(
rebind={constants.AMPHORA: constants.FAILED_AMPHORA},
requires=constants.AMPHORA,
provides=constants.AMP_DATA))
# Get a new amphora
# Note: Role doesn't matter here. We will update it later.
get_amp_subflow = self.get_amphora_for_lb_subflow(
prefix=constants.FAILOVER_AMPHORA_FLOW)
failover_amphora_flow.add(get_amp_subflow)
# Update the new amphora with the failed amphora details
failover_amphora_flow.add(database_tasks.UpdateAmpFailoverDetails(
requires=(constants.AMPHORA, constants.AMP_DATA)))
# Update the data stored in the flow from the database
failover_amphora_flow.add(database_tasks.ReloadLoadBalancer(
requires=constants.LOADBALANCER_ID,
provides=constants.LOADBALANCER))
failover_amphora_flow.add(database_tasks.ReloadAmphora(
requires=constants.AMPHORA,
provides=constants.AMPHORA))
# Prepare to reconnect the network interface(s)
failover_amphora_flow.add(network_tasks.GetAmphoraeNetworkConfigs(
requires=constants.LOADBALANCER,
provides=constants.AMPHORAE_NETWORK_CONFIG))
failover_amphora_flow.add(database_tasks.GetListenersFromLoadbalancer(
requires=constants.LOADBALANCER, provides=constants.LISTENERS))
failover_amphora_flow.add(database_tasks.GetAmphoraeFromLoadbalancer(
requires=constants.LOADBALANCER, provides=constants.AMPHORAE))
# Plug the VIP ports into the new amphora
# The reason for moving these steps here is the udp listeners want to
# do some kernel configuration before Listener update for forbidding
# failure during rebuild amphora.
failover_amphora_flow.add(network_tasks.PlugVIPPort(
requires=(constants.AMPHORA, constants.AMPHORAE_NETWORK_CONFIG)))
failover_amphora_flow.add(amphora_driver_tasks.AmphoraPostVIPPlug(
requires=(constants.AMPHORA, constants.LOADBALANCER,
constants.AMPHORAE_NETWORK_CONFIG)))
# Listeners update needs to be run on all amphora to update
# their peer configurations. So parallelize this with an
# unordered subflow.
update_amps_subflow = unordered_flow.Flow(
constants.UPDATE_AMPS_SUBFLOW)
timeout_dict = {
constants.CONN_MAX_RETRIES:
CONF.haproxy_amphora.active_connection_max_retries,
constants.CONN_RETRY_INTERVAL:
CONF.haproxy_amphora.active_connection_rety_interval}
# Setup parallel flows for each amp. We don't know the new amp
# details at flow creation time, so setup a subflow for each
# amp on the LB, they let the task index into a list of amps
# to find the amphora it should work on.
amp_index = 0
db_lb = self.lb_repo.get(db_apis.get_session(),
id=load_balancer[constants.LOADBALANCER_ID])
for amp in db_lb.amphorae:
if amp.status == constants.DELETED:
continue
update_amps_subflow.add(
amphora_driver_tasks.AmpListenersUpdate(
name=constants.AMP_LISTENER_UPDATE + '-' + str(amp_index),
requires=(constants.LOADBALANCER, constants.AMPHORAE),
inject={constants.AMPHORA_INDEX: amp_index,
constants.TIMEOUT_DICT: timeout_dict}))
amp_index += 1
failover_amphora_flow.add(update_amps_subflow)
# Plug the member networks into the new amphora
failover_amphora_flow.add(network_tasks.CalculateAmphoraDelta(
requires=(constants.LOADBALANCER, constants.AMPHORA,
constants.AVAILABILITY_ZONE),
provides=constants.DELTA))
failover_amphora_flow.add(network_tasks.HandleNetworkDelta(
requires=(constants.AMPHORA, constants.DELTA),
provides=constants.ADDED_PORTS))
failover_amphora_flow.add(amphora_driver_tasks.AmphoraePostNetworkPlug(
requires=(constants.LOADBALANCER, constants.ADDED_PORTS)))
failover_amphora_flow.add(database_tasks.ReloadLoadBalancer(
name='octavia-failover-LB-reload-2',
requires=constants.LOADBALANCER_ID,
provides=constants.LOADBALANCER))
# Handle the amphora role and VRRP if necessary
if role == constants.ROLE_MASTER:
failover_amphora_flow.add(database_tasks.MarkAmphoraMasterInDB(
name=constants.MARK_AMP_MASTER_INDB,
requires=constants.AMPHORA))
vrrp_subflow = self.get_vrrp_subflow(role)
failover_amphora_flow.add(vrrp_subflow)
elif role == constants.ROLE_BACKUP:
failover_amphora_flow.add(database_tasks.MarkAmphoraBackupInDB(
name=constants.MARK_AMP_BACKUP_INDB,
requires=constants.AMPHORA))
vrrp_subflow = self.get_vrrp_subflow(role)
failover_amphora_flow.add(vrrp_subflow)
elif role == constants.ROLE_STANDALONE:
failover_amphora_flow.add(
database_tasks.MarkAmphoraStandAloneInDB(
name=constants.MARK_AMP_STANDALONE_INDB,
requires=constants.AMPHORA))
failover_amphora_flow.add(amphora_driver_tasks.ListenersStart(
requires=(constants.LOADBALANCER, constants.AMPHORA)))
failover_amphora_flow.add(
database_tasks.DisableAmphoraHealthMonitoring(
rebind={constants.AMPHORA: constants.FAILED_AMPHORA},
requires=constants.AMPHORA))
return failover_amphora_flow
def get_vrrp_subflow(self, prefix):
def get_vrrp_subflow(self, prefix, timeout_dict=None,
create_vrrp_group=True):
sf_name = prefix + '-' + constants.GET_VRRP_SUBFLOW
vrrp_subflow = linear_flow.Flow(sf_name)
vrrp_subflow.add(network_tasks.GetAmphoraeNetworkConfigs(
name=sf_name + '-' + constants.GET_AMP_NETWORK_CONFIG,
requires=constants.LOADBALANCER,
provides=constants.AMPHORAE_NETWORK_CONFIG))
vrrp_subflow.add(amphora_driver_tasks.AmphoraUpdateVRRPInterface(
name=sf_name + '-' + constants.AMP_UPDATE_VRRP_INTF,
requires=constants.LOADBALANCER,
provides=constants.LOADBALANCER))
# Optimization for failover flow. No reason to call this
# when configuring the secondary amphora.
if create_vrrp_group:
vrrp_subflow.add(database_tasks.CreateVRRPGroupForLB(
name=sf_name + '-' + constants.CREATE_VRRP_GROUP_FOR_LB,
requires=constants.LOADBALANCER,
provides=constants.LOADBALANCER))
vrrp_subflow.add(amphora_driver_tasks.AmphoraVRRPUpdate(
name=sf_name + '-' + constants.AMP_VRRP_UPDATE,
requires=(constants.LOADBALANCER,
constants.AMPHORAE_NETWORK_CONFIG)))
vrrp_subflow.add(amphora_driver_tasks.AmphoraVRRPStart(
name=sf_name + '-' + constants.AMP_VRRP_START,
requires=constants.LOADBALANCER))
requires=constants.LOADBALANCER_ID))
vrrp_subflow.add(network_tasks.GetAmphoraeNetworkConfigs(
name=sf_name + '-' + constants.GET_AMP_NETWORK_CONFIG,
requires=constants.LOADBALANCER_ID,
provides=constants.AMPHORAE_NETWORK_CONFIG))
# VRRP update needs to be run on all amphora to update
# their peer configurations. So parallelize this with an
# unordered subflow.
update_amps_subflow = unordered_flow.Flow('VRRP-update-subflow')
# We have three tasks to run in order, per amphora
amp_0_subflow = linear_flow.Flow('VRRP-amp-0-update-subflow')
amp_0_subflow.add(amphora_driver_tasks.AmphoraIndexUpdateVRRPInterface(
name=sf_name + '-0-' + constants.AMP_UPDATE_VRRP_INTF,
requires=constants.AMPHORAE,
inject={constants.AMPHORA_INDEX: 0,
constants.TIMEOUT_DICT: timeout_dict},
provides=constants.AMP_VRRP_INT))
amp_0_subflow.add(amphora_driver_tasks.AmphoraIndexVRRPUpdate(
name=sf_name + '-0-' + constants.AMP_VRRP_UPDATE,
requires=(constants.LOADBALANCER_ID,
constants.AMPHORAE_NETWORK_CONFIG, constants.AMPHORAE,
constants.AMP_VRRP_INT),
inject={constants.AMPHORA_INDEX: 0,
constants.TIMEOUT_DICT: timeout_dict}))
amp_0_subflow.add(amphora_driver_tasks.AmphoraIndexVRRPStart(
name=sf_name + '-0-' + constants.AMP_VRRP_START,
requires=constants.AMPHORAE,
inject={constants.AMPHORA_INDEX: 0,
constants.TIMEOUT_DICT: timeout_dict}))
amp_1_subflow = linear_flow.Flow('VRRP-amp-1-update-subflow')
amp_1_subflow.add(amphora_driver_tasks.AmphoraIndexUpdateVRRPInterface(
name=sf_name + '-1-' + constants.AMP_UPDATE_VRRP_INTF,
requires=constants.AMPHORAE,
inject={constants.AMPHORA_INDEX: 1,
constants.TIMEOUT_DICT: timeout_dict},
provides=constants.AMP_VRRP_INT))
amp_1_subflow.add(amphora_driver_tasks.AmphoraIndexVRRPUpdate(
name=sf_name + '-1-' + constants.AMP_VRRP_UPDATE,
requires=(constants.LOADBALANCER_ID,
constants.AMPHORAE_NETWORK_CONFIG, constants.AMPHORAE,
constants.AMP_VRRP_INT),
inject={constants.AMPHORA_INDEX: 1,
constants.TIMEOUT_DICT: timeout_dict}))
amp_1_subflow.add(amphora_driver_tasks.AmphoraIndexVRRPStart(
name=sf_name + '-1-' + constants.AMP_VRRP_START,
requires=constants.AMPHORAE,
inject={constants.AMPHORA_INDEX: 1,
constants.TIMEOUT_DICT: timeout_dict}))
update_amps_subflow.add(amp_0_subflow)
update_amps_subflow.add(amp_1_subflow)
vrrp_subflow.add(update_amps_subflow)
return vrrp_subflow
def cert_rotate_amphora_flow(self):
@ -644,3 +477,254 @@ class AmphoraFlows(object):
requires=(constants.AMPHORA, constants.FLAVOR)))
return update_amphora_flow
def get_amphora_for_lb_failover_subflow(
self, prefix, role=constants.ROLE_STANDALONE,
failed_amp_vrrp_port_id=None, is_vrrp_ipv6=False, is_spare=False):
"""Creates a new amphora that will be used in a failover flow.
:requires: loadbalancer_id, flavor, vip, vip_sg_id, loadbalancer
:provides: amphora_id, amphora
:param prefix: The flow name prefix to use on the flow and tasks.
:param role: The role this amphora will have in the topology.
:param failed_amp_vrrp_port_id: The base port ID of the failed amp.
:param is_vrrp_ipv6: True if the base port IP is IPv6.
:param is_spare: True if we are getting a spare amphroa.
:return: A Taskflow sub-flow that will create the amphora.
"""
sf_name = prefix + '-' + constants.CREATE_AMP_FOR_FAILOVER_SUBFLOW
amp_for_failover_flow = linear_flow.Flow(sf_name)
# Try to allocate or boot an amphora instance (unconfigured)
amp_for_failover_flow.add(self.get_amphora_for_lb_subflow(
prefix=prefix + '-' + constants.FAILOVER_LOADBALANCER_FLOW,
role=role, is_spare=is_spare))
# If we are getting a spare amphora, this is all we need to do.
if is_spare:
return amp_for_failover_flow
# Create the VIP base (aka VRRP) port for the amphora.
amp_for_failover_flow.add(network_tasks.CreateVIPBasePort(
name=prefix + '-' + constants.CREATE_VIP_BASE_PORT,
requires=(constants.VIP, constants.VIP_SG_ID,
constants.AMPHORA_ID),
provides=constants.BASE_PORT))
# Attach the VIP base (aka VRRP) port to the amphora.
amp_for_failover_flow.add(compute_tasks.AttachPort(
name=prefix + '-' + constants.ATTACH_PORT,
requires=(constants.AMPHORA, constants.PORT),
rebind={constants.PORT: constants.BASE_PORT}))
# Update the amphora database record with the VIP base port info.
amp_for_failover_flow.add(database_tasks.UpdateAmpFailoverDetails(
name=prefix + '-' + constants.UPDATE_AMP_FAILOVER_DETAILS,
requires=(constants.AMPHORA, constants.VIP, constants.BASE_PORT)))
# Update the amphora networking for the plugged VIP port
amp_for_failover_flow.add(network_tasks.GetAmphoraNetworkConfigsByID(
name=prefix + '-' + constants.GET_AMPHORA_NETWORK_CONFIGS_BY_ID,
requires=(constants.LOADBALANCER_ID, constants.AMPHORA_ID),
provides=constants.AMPHORAE_NETWORK_CONFIG))
# Disable the base (vrrp) port on the failed amphora
# This prevents a DAD failure when bringing up the new amphora.
# Keepalived will handle this for act/stdby.
if (role == constants.ROLE_STANDALONE and failed_amp_vrrp_port_id and
is_vrrp_ipv6):
amp_for_failover_flow.add(network_tasks.AdminDownPort(
name=prefix + '-' + constants.ADMIN_DOWN_PORT,
inject={constants.PORT_ID: failed_amp_vrrp_port_id}))
amp_for_failover_flow.add(amphora_driver_tasks.AmphoraPostVIPPlug(
name=prefix + '-' + constants.AMPHORA_POST_VIP_PLUG,
requires=(constants.AMPHORA, constants.LOADBALANCER,
constants.AMPHORAE_NETWORK_CONFIG)))
# Plug member ports
amp_for_failover_flow.add(network_tasks.CalculateAmphoraDelta(
name=prefix + '-' + constants.CALCULATE_AMPHORA_DELTA,
requires=(constants.LOADBALANCER, constants.AMPHORA,
constants.AVAILABILITY_ZONE, constants.VRRP_PORT),
rebind={constants.VRRP_PORT: constants.BASE_PORT},
provides=constants.DELTA))
amp_for_failover_flow.add(network_tasks.HandleNetworkDelta(
name=prefix + '-' + constants.HANDLE_NETWORK_DELTA,
requires=(constants.AMPHORA, constants.DELTA),
provides=constants.ADDED_PORTS))
amp_for_failover_flow.add(amphora_driver_tasks.AmphoraePostNetworkPlug(
name=prefix + '-' + constants.AMPHORAE_POST_NETWORK_PLUG,
requires=(constants.LOADBALANCER, constants.ADDED_PORTS)))
return amp_for_failover_flow
def get_failover_amphora_flow(self, failed_amphora, lb_amp_count):
"""Get a Taskflow flow to failover an amphora.
1. Build a replacement amphora.
2. Delete the old amphora.
3. Update the amphorae listener configurations.
4. Update the VRRP configurations if needed.
:param failed_amphora: The amphora dict to failover.
:param lb_amp_count: The number of amphora on this load balancer.
:returns: The flow that will provide the failover.
"""
failover_amp_flow = linear_flow.Flow(
constants.FAILOVER_AMPHORA_FLOW)
# Revert amphora to status ERROR if this flow goes wrong
failover_amp_flow.add(lifecycle_tasks.AmphoraToErrorOnRevertTask(
requires=constants.AMPHORA,
inject={constants.AMPHORA: failed_amphora}))
if failed_amphora[constants.ROLE] in (constants.ROLE_MASTER,
constants.ROLE_BACKUP):
amp_role = 'master_or_backup'
elif failed_amphora[constants.ROLE] == constants.ROLE_STANDALONE:
amp_role = 'standalone'
elif failed_amphora[constants.ROLE] is None:
amp_role = 'spare'
else:
amp_role = 'undefined'
LOG.info("Performing failover for amphora: %s",
{"id": failed_amphora[constants.ID],
"load_balancer_id": failed_amphora.get(
constants.LOAD_BALANCER_ID),
"lb_network_ip": failed_amphora.get(constants.LB_NETWORK_IP),
"compute_id": failed_amphora.get(constants.COMPUTE_ID),
"role": amp_role})
failover_amp_flow.add(database_tasks.MarkAmphoraPendingDeleteInDB(
requires=constants.AMPHORA,
inject={constants.AMPHORA: failed_amphora}))
failover_amp_flow.add(database_tasks.MarkAmphoraHealthBusy(
requires=constants.AMPHORA,
inject={constants.AMPHORA: failed_amphora}))
failover_amp_flow.add(network_tasks.GetVIPSecurityGroupID(
requires=constants.LOADBALANCER_ID,
provides=constants.VIP_SG_ID))
is_spare = True
is_vrrp_ipv6 = False
if failed_amphora.get(constants.LOAD_BALANCER_ID):
is_spare = False
if failed_amphora.get(constants.VRRP_IP):
is_vrrp_ipv6 = utils.is_ipv6(failed_amphora[constants.VRRP_IP])
# Get a replacement amphora and plug all of the networking.
#
# Do this early as the compute services have been observed to be
# unreliable. The community decided the chance that deleting first
# would open resources for an instance is less likely than the
# compute service failing to boot an instance for other reasons.
# TODO(johnsom) Move this back out to run for spares after
# delete amphora API is available.
failover_amp_flow.add(self.get_amphora_for_lb_failover_subflow(
prefix=constants.FAILOVER_LOADBALANCER_FLOW,
role=failed_amphora[constants.ROLE],
failed_amp_vrrp_port_id=failed_amphora.get(
constants.VRRP_PORT_ID),
is_vrrp_ipv6=is_vrrp_ipv6,
is_spare=is_spare))
failover_amp_flow.add(
self.get_delete_amphora_flow(
failed_amphora,
retry_attempts=CONF.controller_worker.amphora_delete_retries,
retry_interval=(
CONF.controller_worker.amphora_delete_retry_interval)))
failover_amp_flow.add(
database_tasks.DisableAmphoraHealthMonitoring(
requires=constants.AMPHORA,
inject={constants.AMPHORA: failed_amphora}))
if not failed_amphora.get(constants.LOAD_BALANCER_ID):
# This is an unallocated amphora (spares pool), we are done.
return failover_amp_flow
failover_amp_flow.add(database_tasks.GetLoadBalancer(
requires=constants.LOADBALANCER_ID,
inject={constants.LOADBALANCER_ID:
failed_amphora[constants.LOAD_BALANCER_ID]},
provides=constants.LOADBALANCER))
failover_amp_flow.add(database_tasks.GetAmphoraeFromLoadbalancer(
name=constants.GET_AMPHORAE_FROM_LB,
requires=constants.LOADBALANCER_ID,
inject={constants.LOADBALANCER_ID:
failed_amphora[constants.LOAD_BALANCER_ID]},
provides=constants.AMPHORAE))
# Setup timeouts for our requests to the amphorae
timeout_dict = {
constants.CONN_MAX_RETRIES:
CONF.haproxy_amphora.active_connection_max_retries,
constants.CONN_RETRY_INTERVAL:
CONF.haproxy_amphora.active_connection_rety_interval}
# Listeners update needs to be run on all amphora to update
# their peer configurations. So parallelize this with an
# unordered subflow.
update_amps_subflow = unordered_flow.Flow(
constants.UPDATE_AMPS_SUBFLOW)
for amp_index in range(0, lb_amp_count):
update_amps_subflow.add(
amphora_driver_tasks.AmphoraIndexListenerUpdate(
name=str(amp_index) + '-' + constants.AMP_LISTENER_UPDATE,
requires=(constants.LOADBALANCER, constants.AMPHORAE),
inject={constants.AMPHORA_INDEX: amp_index,
constants.TIMEOUT_DICT: timeout_dict}))
failover_amp_flow.add(update_amps_subflow)
# Configure and enable keepalived in the amphora
if lb_amp_count == 2:
failover_amp_flow.add(
self.get_vrrp_subflow(constants.GET_VRRP_SUBFLOW,
timeout_dict, create_vrrp_group=False))
# Reload the listener. This needs to be done here because
# it will create the required haproxy check scripts for
# the VRRP deployed above.
# A "U" or newer amphora-agent will remove the need for this
# task here.
# TODO(johnsom) Remove this in the "W" cycle
reload_listener_subflow = unordered_flow.Flow(
constants.AMPHORA_LISTENER_RELOAD_SUBFLOW)
for amp_index in range(0, lb_amp_count):
reload_listener_subflow.add(
amphora_driver_tasks.AmphoraIndexListenersReload(
name=(str(amp_index) + '-' +
constants.AMPHORA_RELOAD_LISTENER),
requires=(constants.LOADBALANCER, constants.AMPHORAE),
inject={constants.AMPHORA_INDEX: amp_index,
constants.TIMEOUT_DICT: timeout_dict}))
failover_amp_flow.add(reload_listener_subflow)
# Remove any extraneous ports
# Note: Nova sometimes fails to delete ports attached to an instance.
# For example, if you create an LB with a listener, then
# 'openstack server delete' the amphora, you will see the vrrp
# port attached to that instance will remain after the instance
# is deleted.
# TODO(johnsom) Fix this as part of
# https://storyboard.openstack.org/#!/story/2007077
# Mark LB ACTIVE
failover_amp_flow.add(
database_tasks.MarkLBActiveInDB(mark_subobjects=True,
requires=constants.LOADBALANCER))
return failover_amp_flow

47
octavia/controller/worker/v2/flows/flow_utils.py
View File

@ -11,7 +11,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from octavia.common import constants
from octavia.api.drivers import utils as provider_utils
from octavia.controller.worker.v2.flows import amphora_flows
from octavia.controller.worker.v2.flows import health_monitor_flows
from octavia.controller.worker.v2.flows import l7policy_flows
@ -41,16 +41,36 @@ def get_delete_load_balancer_flow(lb):
return LB_FLOWS.get_delete_load_balancer_flow(lb)
def get_delete_listeners_store(lb):
return LB_FLOWS.get_delete_listeners_store(lb)
def get_listeners_on_lb(db_lb):
"""Get a list of the listeners on a load balancer.
:param db_lb: A load balancer database model object.
:returns: A list of provider dict format listeners.
"""
listener_dicts = []
for listener in db_lb.listeners:
prov_listener = provider_utils.db_listener_to_provider_listener(
listener)
listener_dicts.append(prov_listener.to_dict())
return listener_dicts
def get_delete_pools_store(lb):
return LB_FLOWS.get_delete_pools_store(lb)
def get_pools_on_lb(db_lb):
"""Get a list of the pools on a load balancer.
:param db_lb: A load balancer database model object.
:returns: A list of provider dict format pools.
"""
pool_dicts = []
for pool in db_lb.pools:
prov_pool = provider_utils.db_pool_to_provider_pool(pool)
pool_dicts.append(prov_pool.to_dict())
return pool_dicts
def get_cascade_delete_load_balancer_flow(lb):
return LB_FLOWS.get_cascade_delete_load_balancer_flow(lb)
def get_cascade_delete_load_balancer_flow(lb, listeners=(), pools=()):
return LB_FLOWS.get_cascade_delete_load_balancer_flow(lb, listeners,
pools)
def get_update_load_balancer_flow():
@ -61,12 +81,17 @@ def get_create_amphora_flow():
return AMP_FLOWS.get_create_amphora_flow()
def get_delete_amphora_flow():
return AMP_FLOWS.get_delete_amphora_flow()
def get_delete_amphora_flow(amphora, retry_attempts=None, retry_interval=None):
return AMP_FLOWS.get_delete_amphora_flow(amphora, retry_attempts,
retry_interval)
def get_failover_flow(role=constants.ROLE_STANDALONE, load_balancer=None):
return AMP_FLOWS.get_failover_flow(role=role, load_balancer=load_balancer)
def get_failover_LB_flow(amps, lb):
return LB_FLOWS.get_failover_LB_flow(amps, lb)
def get_failover_amphora_flow(amphora_dict, lb_amp_count):
return AMP_FLOWS.get_failover_amphora_flow(amphora_dict, lb_amp_count)
def cert_rotate_amphora_flow():

14
octavia/controller/worker/v2/flows/listener_flows.py
View File

@ -83,24 +83,26 @@ class ListenerFlows(object):
return delete_listener_flow
def get_delete_listener_internal_flow(self, listener_name):
def get_delete_listener_internal_flow(self, listener):
"""Create a flow to delete a listener and l7policies internally
(will skip deletion on the amp and marking LB active)
:returns: The flow for deleting a listener
"""
delete_listener_flow = linear_flow.Flow(constants.DELETE_LISTENER_FLOW)
listener_id = listener[constants.LISTENER_ID]
delete_listener_flow = linear_flow.Flow(
constants.DELETE_LISTENER_FLOW + '-' + listener_id)
# Should cascade delete all L7 policies
delete_listener_flow.add(network_tasks.UpdateVIPForDelete(
name='delete_update_vip_' + listener_name,
name='delete_update_vip_' + listener_id,
requires=constants.LOADBALANCER_ID))
delete_listener_flow.add(database_tasks.DeleteListenerInDB(
name='delete_listener_in_db_' + listener_name,
name='delete_listener_in_db_' + listener_id,
requires=constants.LISTENER,
rebind={constants.LISTENER: listener_name}))
inject={constants.LISTENER: listener}))
delete_listener_flow.add(database_tasks.DecrementListenerQuota(
name='decrement_listener_quota_' + listener_name,
name='decrement_listener_quota_' + listener_id,
requires=constants.PROJECT_ID))
return delete_listener_flow

503
octavia/controller/worker/v2/flows/load_balancer_flows.py
View File

@ -1,4 +1,5 @@
# Copyright 2015 Hewlett-Packard Development Company, L.P.
# Copyright 2020 Red Hat, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -18,9 +19,9 @@ from oslo_log import log as logging
from taskflow.patterns import linear_flow
from taskflow.patterns import unordered_flow
from octavia.api.drivers import utils as provider_utils
from octavia.common import constants
from octavia.common import exceptions
from octavia.common import utils
from octavia.controller.worker.v2.flows import amphora_flows
from octavia.controller.worker.v2.flows import listener_flows
from octavia.controller.worker.v2.flows import member_flows
@ -30,7 +31,6 @@ from octavia.controller.worker.v2.tasks import compute_tasks
from octavia.controller.worker.v2.tasks import database_tasks
from octavia.controller.worker.v2.tasks import lifecycle_tasks
from octavia.controller.worker.v2.tasks import network_tasks
from octavia.db import api as db_apis
from octavia.db import repositories as repo
CONF = cfg.CONF
@ -72,7 +72,7 @@ class LoadBalancerFlows(object):
requires=(constants.LOADBALANCER_ID, constants.VIP),
provides=constants.LOADBALANCER))
lb_create_flow.add(network_tasks.UpdateVIPSecurityGroup(
requires=constants.LOADBALANCER))
requires=constants.LOADBALANCER_ID))
lb_create_flow.add(network_tasks.GetSubnetFromVIP(
requires=constants.LOADBALANCER,
provides=constants.SUBNET))
@ -97,9 +97,15 @@ class LoadBalancerFlows(object):
return lb_create_flow
def _create_single_topology(self):
return (self.amp_flows.get_amphora_for_lb_subflow(
sf_name = (constants.ROLE_STANDALONE + '-' +
constants.AMP_PLUG_NET_SUBFLOW)
amp_for_lb_net_flow = linear_flow.Flow(sf_name)
amp_for_lb_flow = self.amp_flows.get_amphora_for_lb_subflow(
prefix=constants.ROLE_STANDALONE,
role=constants.ROLE_STANDALONE), )
role=constants.ROLE_STANDALONE)
amp_for_lb_net_flow.add(amp_for_lb_flow)
amp_for_lb_net_flow.add(*self._get_amp_net_subflow(sf_name))
return amp_for_lb_net_flow
def _create_active_standby_topology(
self, lf_name=constants.CREATE_LOADBALANCER_FLOW):
@ -128,16 +134,52 @@ class LoadBalancerFlows(object):
f_name = constants.CREATE_LOADBALANCER_FLOW
amps_flow = unordered_flow.Flow(f_name)
master_amp_sf = self.amp_flows.get_amphora_for_lb_subflow(
prefix=constants.ROLE_MASTER, role=constants.ROLE_MASTER
)
backup_amp_sf = self.amp_flows.get_amphora_for_lb_subflow(
prefix=constants.ROLE_BACKUP, role=constants.ROLE_BACKUP)
master_sf_name = (constants.ROLE_MASTER + '-' +
constants.AMP_PLUG_NET_SUBFLOW)
master_amp_sf = linear_flow.Flow(master_sf_name)
master_amp_sf.add(self.amp_flows.get_amphora_for_lb_subflow(
prefix=constants.ROLE_MASTER, role=constants.ROLE_MASTER))
master_amp_sf.add(*self._get_amp_net_subflow(master_sf_name))
backup_sf_name = (constants.ROLE_BACKUP + '-' +
constants.AMP_PLUG_NET_SUBFLOW)
backup_amp_sf = linear_flow.Flow(backup_sf_name)
backup_amp_sf.add(self.amp_flows.get_amphora_for_lb_subflow(
prefix=constants.ROLE_BACKUP, role=constants.ROLE_BACKUP))
backup_amp_sf.add(*self._get_amp_net_subflow(backup_sf_name))
amps_flow.add(master_amp_sf, backup_amp_sf)
return flows + [amps_flow]
def _get_amp_net_subflow(self, sf_name):
flows = []
flows.append(network_tasks.PlugVIPAmphora(
name=sf_name + '-' + constants.PLUG_VIP_AMPHORA,
requires=(constants.LOADBALANCER, constants.AMPHORA,
constants.SUBNET),
provides=constants.AMP_DATA))
flows.append(network_tasks.ApplyQosAmphora(
name=sf_name + '-' + constants.APPLY_QOS_AMP,
requires=(constants.LOADBALANCER, constants.AMP_DATA,
constants.UPDATE_DICT)))
flows.append(database_tasks.UpdateAmphoraVIPData(
name=sf_name + '-' + constants.UPDATE_AMPHORA_VIP_DATA,
requires=constants.AMP_DATA))
flows.append(network_tasks.GetAmphoraNetworkConfigs(
name=sf_name + '-' + constants.GET_AMP_NETWORK_CONFIG,
requires=(constants.LOADBALANCER, constants.AMPHORA),
provides=constants.AMPHORA_NETWORK_CONFIG))
flows.append(amphora_driver_tasks.AmphoraPostVIPPlug(
name=sf_name + '-' + constants.AMP_POST_VIP_PLUG,
rebind={constants.AMPHORAE_NETWORK_CONFIG:
constants.AMPHORA_NETWORK_CONFIG},
requires=(constants.LOADBALANCER,
constants.AMPHORAE_NETWORK_CONFIG)))
return flows
def _create_listeners_flow(self):
flows = []
flows.append(
@ -181,13 +223,6 @@ class LoadBalancerFlows(object):
created/allocated amphorae.
:return: Post amphorae association subflow
"""
# Note: If any task in this flow failed, the created amphorae will be
# left ''incorrectly'' allocated to the loadbalancer. Likely,
# the get_new_LB_networking_subflow is the most prune to failure
# shall deallocate the amphora from its loadbalancer and put it in a
# READY state.
sf_name = prefix + '-' + constants.POST_LB_AMP_ASSOCIATION_SUBFLOW
post_create_LB_flow = linear_flow.Flow(sf_name)
post_create_LB_flow.add(
@ -197,6 +232,9 @@ class LoadBalancerFlows(object):
provides=constants.LOADBALANCER))
if topology == constants.TOPOLOGY_ACTIVE_STANDBY:
post_create_LB_flow.add(database_tasks.GetAmphoraeFromLoadbalancer(
requires=constants.LOADBALANCER_ID,
provides=constants.AMPHORAE))
vrrp_subflow = self.amp_flows.get_vrrp_subflow(prefix)
post_create_LB_flow.add(vrrp_subflow)
@ -208,36 +246,19 @@ class LoadBalancerFlows(object):
requires=constants.LOADBALANCER))
return post_create_LB_flow
def _get_delete_listeners_flow(self, lb):
def _get_delete_listeners_flow(self, listeners):
"""Sets up an internal delete flow
Because task flow doesn't support loops we store each listener
we want to delete in the store part and then rebind
:param lb: load balancer
:return: (flow, store) -- flow for the deletion and store with all
the listeners stored properly
:param listeners: A list of listener dicts
:return: The flow for the deletion
"""
listeners_delete_flow = unordered_flow.Flow('listener_delete_flow')
db_lb = self.lb_repo.get(db_apis.get_session(),
id=lb[constants.LOADBALANCER_ID])
for listener in db_lb.listeners:
listener_name = 'listener_' + listener.id
listeners_delete_flow = unordered_flow.Flow('listeners_delete_flow')
for listener in listeners:
listeners_delete_flow.add(
self.listener_flows.get_delete_listener_internal_flow(
listener_name))
listener))
return listeners_delete_flow
def get_delete_listeners_store(self, lb):
store = {}
for listener in lb.listeners:
listener_name = 'listener_' + listener.id
prov_listener = provider_utils.db_listener_to_provider_listener(
listener)
store[listener_name] = prov_listener.to_dict()
store.update({constants.LOADBALANCER_ID: lb.id,
constants.PROJECT_ID: lb.project_id})
return store
def get_delete_load_balancer_flow(self, lb):
"""Creates a flow to delete a load balancer.
@ -245,14 +266,7 @@ class LoadBalancerFlows(object):
"""
return self._get_delete_load_balancer_flow(lb, False)
def get_delete_pools_store(self, lb):
store = {}
for pool in lb.pools:
pool_name = 'pool' + pool.id
store[pool_name] = pool.id
return store
def _get_delete_pools_flow(self, lb):
def _get_delete_pools_flow(self, pools):
"""Sets up an internal delete flow
Because task flow doesn't support loops we store each pool
@ -262,16 +276,14 @@ class LoadBalancerFlows(object):
the listeners stored properly
"""
pools_delete_flow = unordered_flow.Flow('pool_delete_flow')
db_lb = self.lb_repo.get(db_apis.get_session(),
id=lb[constants.LOADBALANCER_ID])
for pool in db_lb.pools:
pool_name = 'pool' + pool.id
for pool in pools:
pools_delete_flow.add(
self.pool_flows.get_delete_pool_flow_internal(
pool_name))
pool[constants.POOL_ID]))
return pools_delete_flow
def _get_delete_load_balancer_flow(self, lb, cascade):
def _get_delete_load_balancer_flow(self, lb, cascade,
listeners=(), pools=()):
delete_LB_flow = linear_flow.Flow(constants.DELETE_LOADBALANCER_FLOW)
delete_LB_flow.add(lifecycle_tasks.LoadBalancerToErrorOnRevertTask(
requires=constants.LOADBALANCER))
@ -280,8 +292,8 @@ class LoadBalancerFlows(object):
delete_LB_flow.add(database_tasks.MarkLBAmphoraeHealthBusy(
requires=constants.LOADBALANCER))
if cascade:
listeners_delete = self._get_delete_listeners_flow(lb)
pools_delete = self._get_delete_pools_flow(lb)
listeners_delete = self._get_delete_listeners_flow(listeners)
pools_delete = self._get_delete_pools_flow(pools)
delete_LB_flow.add(pools_delete)
delete_LB_flow.add(listeners_delete)
delete_LB_flow.add(network_tasks.UnplugVIP(
@ -300,47 +312,14 @@ class LoadBalancerFlows(object):
requires=constants.PROJECT_ID))
return delete_LB_flow
def get_cascade_delete_load_balancer_flow(self, lb):
def get_cascade_delete_load_balancer_flow(self, lb, listeners, pools):
"""Creates a flow to delete a load balancer.
:returns: The flow for deleting a load balancer
"""
return self._get_delete_load_balancer_flow(lb, True)
def get_new_LB_networking_subflow(self):
"""Create a sub-flow to setup networking.
:returns: The flow to setup networking for a new amphora
"""
new_LB_net_subflow = linear_flow.Flow(constants.
LOADBALANCER_NETWORKING_SUBFLOW)
new_LB_net_subflow.add(network_tasks.AllocateVIP(
requires=constants.LOADBALANCER,
provides=constants.VIP))
new_LB_net_subflow.add(database_tasks.UpdateVIPAfterAllocation(
requires=(constants.LOADBALANCER_ID, constants.VIP),
provides=constants.LOADBALANCER))
new_LB_net_subflow.add(network_tasks.PlugVIP(
requires=constants.LOADBALANCER,
provides=constants.AMPS_DATA))
new_LB_net_subflow.add(network_tasks.ApplyQos(
requires=(constants.LOADBALANCER, constants.AMPS_DATA,
constants.UPDATE_DICT)))
new_LB_net_subflow.add(database_tasks.UpdateAmphoraeVIPData(
requires=constants.AMPS_DATA))
new_LB_net_subflow.add(database_tasks.ReloadLoadBalancer(
name=constants.RELOAD_LB_AFTER_PLUG_VIP,
requires=constants.LOADBALANCER_ID,
provides=constants.LOADBALANCER))
new_LB_net_subflow.add(network_tasks.GetAmphoraeNetworkConfigs(
requires=constants.LOADBALANCER,
provides=constants.AMPHORAE_NETWORK_CONFIG))
new_LB_net_subflow.add(amphora_driver_tasks.AmphoraePostVIPPlug(
requires=(constants.LOADBALANCER,
constants.AMPHORAE_NETWORK_CONFIG)))
return new_LB_net_subflow
return self._get_delete_load_balancer_flow(lb, True,
listeners=listeners,
pools=pools)
def get_update_load_balancer_flow(self):
"""Creates a flow to update a load balancer.
@ -360,3 +339,341 @@ class LoadBalancerFlows(object):
requires=constants.LOADBALANCER))
return update_LB_flow
def get_failover_LB_flow(self, amps, lb):
"""Failover a load balancer.
1. Validate the VIP port is correct and present.
2. Build a replacement amphora.
3. Delete the failed amphora.
4. Configure the replacement amphora listeners.
5. Configure VRRP for the listeners.
6. Build the second replacement amphora.
7. Delete the second failed amphora.
8. Delete any extraneous amphora.
9. Configure the listeners on the new amphorae.
10. Configure the VRRP on the new amphorae.
11. Reload the listener configurations to pick up VRRP changes.
12. Mark the load balancer back to ACTIVE.
:returns: The flow that will provide the failover.
"""
lb_topology = lb[constants.FLAVOR][constants.LOADBALANCER_TOPOLOGY]
# Pick one amphora to be failed over if any exist.
failed_amp = None
if amps:
failed_amp = amps.pop()
failover_LB_flow = linear_flow.Flow(
constants.FAILOVER_LOADBALANCER_FLOW)
# Revert LB to provisioning_status ERROR if this flow goes wrong
failover_LB_flow.add(lifecycle_tasks.LoadBalancerToErrorOnRevertTask(
requires=constants.LOADBALANCER))
# Setup timeouts for our requests to the amphorae
timeout_dict = {
constants.CONN_MAX_RETRIES:
CONF.haproxy_amphora.active_connection_max_retries,
constants.CONN_RETRY_INTERVAL:
CONF.haproxy_amphora.active_connection_rety_interval}
if failed_amp:
failed_amp_role = failed_amp.get(constants.ROLE)
if failed_amp_role in (constants.ROLE_MASTER,
constants.ROLE_BACKUP):
amp_role = 'master_or_backup'
elif failed_amp_role == constants.ROLE_STANDALONE:
amp_role = 'standalone'
elif failed_amp_role is None:
amp_role = 'spare'
else:
amp_role = 'undefined'
LOG.info("Performing failover for amphora: %s",
{"id": failed_amp.get(constants.ID),
"load_balancer_id": lb.get(constants.ID),
"lb_network_ip": failed_amp.get(constants.LB_NETWORK_IP),
"compute_id": failed_amp.get(constants.COMPUTE_ID),
"role": amp_role})
failover_LB_flow.add(database_tasks.MarkAmphoraPendingDeleteInDB(
requires=constants.AMPHORA,
inject={constants.AMPHORA: failed_amp}))
failover_LB_flow.add(database_tasks.MarkAmphoraHealthBusy(
requires=constants.AMPHORA,
inject={constants.AMPHORA: failed_amp}))
# Check that the VIP port exists and is ok
failover_LB_flow.add(
network_tasks.AllocateVIP(requires=constants.LOADBALANCER,
provides=constants.VIP))
# Update the database with the VIP information
failover_LB_flow.add(database_tasks.UpdateVIPAfterAllocation(
requires=(constants.LOADBALANCER_ID, constants.VIP),
provides=constants.LOADBALANCER))
# Make sure the SG has the correct rules and re-apply to the
# VIP port. It is not used on the VIP port, but will help lock
# the SG as in use.
failover_LB_flow.add(network_tasks.UpdateVIPSecurityGroup(
requires=constants.LOADBALANCER_ID, provides=constants.VIP_SG_ID))
new_amp_role = constants.ROLE_STANDALONE
if lb_topology == constants.TOPOLOGY_ACTIVE_STANDBY:
new_amp_role = constants.ROLE_BACKUP
# Get a replacement amphora and plug all of the networking.
#
# Do this early as the compute services have been observed to be
# unreliable. The community decided the chance that deleting first
# would open resources for an instance is less likely than the compute
# service failing to boot an instance for other reasons.
if failed_amp:
failed_vrrp_is_ipv6 = False
if failed_amp.get(constants.VRRP_IP):
failed_vrrp_is_ipv6 = utils.is_ipv6(
failed_amp[constants.VRRP_IP])
failover_LB_flow.add(
self.amp_flows.get_amphora_for_lb_failover_subflow(
prefix=constants.FAILOVER_LOADBALANCER_FLOW,
role=new_amp_role,
failed_amp_vrrp_port_id=failed_amp.get(
constants.VRRP_PORT_ID),
is_vrrp_ipv6=failed_vrrp_is_ipv6))
else:
failover_LB_flow.add(
self.amp_flows.get_amphora_for_lb_failover_subflow(
prefix=constants.FAILOVER_LOADBALANCER_FLOW,
role=new_amp_role))
if lb_topology == constants.TOPOLOGY_ACTIVE_STANDBY:
failover_LB_flow.add(database_tasks.MarkAmphoraBackupInDB(
name=constants.MARK_AMP_BACKUP_INDB,
requires=constants.AMPHORA))
# Delete the failed amp
if failed_amp:
failover_LB_flow.add(
self.amp_flows.get_delete_amphora_flow(failed_amp))
# Update the data stored in the flow from the database
failover_LB_flow.add(database_tasks.ReloadLoadBalancer(
requires=constants.LOADBALANCER_ID,
provides=constants.LOADBALANCER))
# Configure the listener(s)
# We will run update on this amphora again later if this is
# an active/standby load balancer because we want this amp
# functional as soon as possible. It must run again to update
# the configurations for the new peers.
failover_LB_flow.add(amphora_driver_tasks.AmpListenersUpdate(
name=constants.AMP_LISTENER_UPDATE,
requires=(constants.LOADBALANCER, constants.AMPHORA),
inject={constants.TIMEOUT_DICT: timeout_dict}))
# Bring up the new "backup" amphora VIP now to reduce the outage
# on the final failover. This dropped the outage from 8-9 seconds
# to less than one in my lab.
# This does mean some steps have to be repeated later to reconfigure
# for the second amphora as a peer.
if lb_topology == constants.TOPOLOGY_ACTIVE_STANDBY:
failover_LB_flow.add(database_tasks.CreateVRRPGroupForLB(
name=new_amp_role + '-' + constants.CREATE_VRRP_GROUP_FOR_LB,
requires=constants.LOADBALANCER_ID))
failover_LB_flow.add(network_tasks.GetAmphoraNetworkConfigsByID(
name=(new_amp_role + '-' +
constants.GET_AMPHORA_NETWORK_CONFIGS_BY_ID),
requires=(constants.LOADBALANCER_ID, constants.AMPHORA_ID),
provides=constants.FIRST_AMP_NETWORK_CONFIGS))
failover_LB_flow.add(
amphora_driver_tasks.AmphoraUpdateVRRPInterface(
name=new_amp_role + '-' + constants.AMP_UPDATE_VRRP_INTF,
requires=constants.AMPHORA,
inject={constants.TIMEOUT_DICT: timeout_dict},
provides=constants.FIRST_AMP_VRRP_INTERFACE))
failover_LB_flow.add(amphora_driver_tasks.AmphoraVRRPUpdate(
name=new_amp_role + '-' + constants.AMP_VRRP_UPDATE,
requires=(constants.LOADBALANCER_ID, constants.AMPHORA),
rebind={constants.AMPHORAE_NETWORK_CONFIG:
constants.FIRST_AMP_NETWORK_CONFIGS,
constants.AMP_VRRP_INT:
constants.FIRST_AMP_VRRP_INTERFACE},
inject={constants.TIMEOUT_DICT: timeout_dict}))
failover_LB_flow.add(amphora_driver_tasks.AmphoraVRRPStart(
name=new_amp_role + '-' + constants.AMP_VRRP_START,
requires=constants.AMPHORA,
inject={constants.TIMEOUT_DICT: timeout_dict}))
# Start the listener. This needs to be done here because
# it will create the required haproxy check scripts for
# the VRRP deployed above.
# A "V" or newer amphora-agent will remove the need for this
# task here.
# TODO(johnsom) Remove this in the "X" cycle
failover_LB_flow.add(amphora_driver_tasks.ListenersStart(
name=new_amp_role + '-' + constants.AMP_LISTENER_START,
requires=(constants.LOADBALANCER, constants.AMPHORA)))
# #### Work on standby amphora if needed #####
new_amp_role = constants.ROLE_MASTER
failed_amp = None
if amps:
failed_amp = amps.pop()
if failed_amp:
failed_amp_role = failed_amp.get(constants.ROLE)
if failed_amp_role in (constants.ROLE_MASTER,
constants.ROLE_BACKUP):
amp_role = 'master_or_backup'
elif failed_amp_role == constants.ROLE_STANDALONE:
amp_role = 'standalone'
elif failed_amp_role is None:
amp_role = 'spare'
else:
amp_role = 'undefined'
LOG.info("Performing failover for amphora: %s",
{"id": failed_amp.get(constants.ID),
"load_balancer_id": lb.get(constants.ID),
"lb_network_ip": failed_amp.get(
constants.LB_NETWORK_IP),
"compute_id": failed_amp.get(constants.COMPUTE_ID),
"role": amp_role})
failover_LB_flow.add(
database_tasks.MarkAmphoraPendingDeleteInDB(
name=(new_amp_role + '-' +
constants.MARK_AMPHORA_PENDING_DELETE),
requires=constants.AMPHORA,
inject={constants.AMPHORA: failed_amp}))
failover_LB_flow.add(database_tasks.MarkAmphoraHealthBusy(
name=(new_amp_role + '-' +
constants.MARK_AMPHORA_HEALTH_BUSY),
requires=constants.AMPHORA,
inject={constants.AMPHORA: failed_amp}))
# Get a replacement amphora and plug all of the networking.
#
# Do this early as the compute services have been observed to be
# unreliable. The community decided the chance that deleting first
# would open resources for an instance is less likely than the
# compute service failing to boot an instance for other reasons.
failover_LB_flow.add(
self.amp_flows.get_amphora_for_lb_failover_subflow(
prefix=(new_amp_role + '-' +
constants.FAILOVER_LOADBALANCER_FLOW),
role=new_amp_role))
failover_LB_flow.add(database_tasks.MarkAmphoraMasterInDB(
name=constants.MARK_AMP_MASTER_INDB,
requires=constants.AMPHORA))
# Delete the failed amp
if failed_amp:
failover_LB_flow.add(
self.amp_flows.get_delete_amphora_flow(
failed_amp))
failover_LB_flow.add(
database_tasks.DisableAmphoraHealthMonitoring(
name=(new_amp_role + '-' +
constants.DISABLE_AMP_HEALTH_MONITORING),
requires=constants.AMPHORA,
inject={constants.AMPHORA: failed_amp}))
# Remove any extraneous amphora
# Note: This runs in all topology situations.
# It should run before the act/stdby final listener update so
# that we don't bother attempting to update dead amphorae.
delete_extra_amps_flow = unordered_flow.Flow(
constants.DELETE_EXTRA_AMPHORAE_FLOW)
for amp in amps:
LOG.debug('Found extraneous amphora %s on load balancer %s. '
'Deleting.', amp.get(constants.ID), lb.get(id))
delete_extra_amps_flow.add(
self.amp_flows.get_delete_amphora_flow(amp))
failover_LB_flow.add(delete_extra_amps_flow)
if lb_topology == constants.TOPOLOGY_ACTIVE_STANDBY:
# Update the data stored in the flow from the database
failover_LB_flow.add(database_tasks.ReloadLoadBalancer(
name=new_amp_role + '-' + constants.RELOAD_LB_AFTER_AMP_ASSOC,
requires=constants.LOADBALANCER_ID,
provides=constants.LOADBALANCER))
failover_LB_flow.add(database_tasks.GetAmphoraeFromLoadbalancer(
name=new_amp_role + '-' + constants.GET_AMPHORAE_FROM_LB,
requires=constants.LOADBALANCER_ID,
provides=constants.AMPHORAE))
# Listeners update needs to be run on all amphora to update
# their peer configurations. So parallelize this with an
# unordered subflow.
update_amps_subflow = unordered_flow.Flow(
constants.UPDATE_AMPS_SUBFLOW)
# Setup parallel flows for each amp. We don't know the new amp
# details at flow creation time, so setup a subflow for each
# amp on the LB, they let the task index into a list of amps
# to find the amphora it should work on.
update_amps_subflow.add(
amphora_driver_tasks.AmphoraIndexListenerUpdate(
name=(constants.AMPHORA + '-0-' +
constants.AMP_LISTENER_UPDATE),
requires=(constants.LOADBALANCER, constants.AMPHORAE),
inject={constants.AMPHORA_INDEX: 0,
constants.TIMEOUT_DICT: timeout_dict}))
update_amps_subflow.add(
amphora_driver_tasks.AmphoraIndexListenerUpdate(
name=(constants.AMPHORA + '-1-' +
constants.AMP_LISTENER_UPDATE),
requires=(constants.LOADBALANCER, constants.AMPHORAE),
inject={constants.AMPHORA_INDEX: 1,
constants.TIMEOUT_DICT: timeout_dict}))
failover_LB_flow.add(update_amps_subflow)
# Configure and enable keepalived in the amphora
failover_LB_flow.add(self.amp_flows.get_vrrp_subflow(
new_amp_role + '-' + constants.GET_VRRP_SUBFLOW,
timeout_dict, create_vrrp_group=False))
# #### End of standby ####
# Reload the listener. This needs to be done here because
# it will create the required haproxy check scripts for
# the VRRP deployed above.
# A "V" or newer amphora-agent will remove the need for this
# task here.
# TODO(johnsom) Remove this in the "X" cycle
failover_LB_flow.add(
amphora_driver_tasks.AmphoraIndexListenersReload(
name=(new_amp_role + '-' +
constants.AMPHORA_RELOAD_LISTENER),
requires=(constants.LOADBALANCER, constants.AMPHORAE),
inject={constants.AMPHORA_INDEX: 1,
constants.TIMEOUT_DICT: timeout_dict}))
# Remove any extraneous ports
# Note: Nova sometimes fails to delete ports attached to an instance.
# For example, if you create an LB with a listener, then
# 'openstack server delete' the amphora, you will see the vrrp
# port attached to that instance will remain after the instance
# is deleted.
# TODO(johnsom) Fix this as part of
# https://storyboard.openstack.org/#!/story/2007077
# Mark LB ACTIVE
failover_LB_flow.add(
database_tasks.MarkLBActiveInDB(mark_subobjects=True,
requires=constants.LOADBALANCER))
return failover_LB_flow

9
octavia/controller/worker/v2/flows/pool_flows.py
View File

@ -74,22 +74,23 @@ class PoolFlows(object):
:returns: The flow for deleting a pool
"""
delete_pool_flow = linear_flow.Flow(constants.DELETE_POOL_FLOW)
delete_pool_flow = linear_flow.Flow(constants.DELETE_POOL_FLOW + '-' +
pool_id)
# health monitor should cascade
# members should cascade
delete_pool_flow.add(database_tasks.MarkPoolPendingDeleteInDB(
name='mark_pool_pending_delete_in_db_' + pool_id,
requires=constants.POOL_ID,
rebind={constants.POOL_ID: pool_id}))
inject={constants.POOL_ID: pool_id}))
delete_pool_flow.add(database_tasks.CountPoolChildrenForQuota(
name='count_pool_children_for_quota_' + pool_id,
requires=constants.POOL_ID,
provides=constants.POOL_CHILD_COUNT,
rebind={constants.POOL_ID: pool_id}))
inject={constants.POOL_ID: pool_id}))
delete_pool_flow.add(database_tasks.DeletePoolInDB(
name='delete_pool_in_db_' + pool_id,
requires=constants.POOL_ID,
rebind={constants.POOL_ID: pool_id}))
inject={constants.POOL_ID: pool_id}))
delete_pool_flow.add(database_tasks.DecrementPoolQuota(
name='decrement_pool_quota_' + pool_id,
requires=[constants.PROJECT_ID, constants.POOL_CHILD_COUNT]))

244
octavia/controller/worker/v2/tasks/amphora_driver_tasks.py
View File

@ -23,7 +23,6 @@ from taskflow.types import failure
from octavia.amphorae.backends.agent import agent_jinja_cfg
from octavia.amphorae.driver_exceptions import exceptions as driver_except
from octavia.api.drivers import utils as provider_utils
from octavia.common import constants
from octavia.common import utils
from octavia.controller.worker import task_utils as task_utilities
@ -74,21 +73,46 @@ class AmpRetry(retry.Times):
class AmpListenersUpdate(BaseAmphoraTask):
"""Task to update the listeners on one amphora."""
def execute(self, loadbalancer, amphora, timeout_dict=None):
# Note, we don't want this to cause a revert as it may be used
# in a failover flow with both amps failing. Skip it and let
# health manager fix it.
# TODO(johnsom) Optimize this to use the dicts and not need the
# DB lookups
db_amp = self.amphora_repo.get(db_apis.get_session(),
id=amphora[constants.ID])
try:
db_lb = self.loadbalancer_repo.get(
db_apis.get_session(),
id=loadbalancer[constants.LOADBALANCER_ID])
self.amphora_driver.update_amphora_listeners(
db_lb, db_amp, timeout_dict)
except Exception as e:
LOG.error('Failed to update listeners on amphora %s. Skipping '
'this amphora as it is failing to update due to: %s',
db_amp.id, str(e))
self.amphora_repo.update(db_apis.get_session(), db_amp.id,
status=constants.ERROR)
class AmphoraIndexListenerUpdate(BaseAmphoraTask):
"""Task to update the listeners on one amphora."""
def execute(self, loadbalancer, amphora_index, amphorae, timeout_dict=()):
# Note, we don't want this to cause a revert as it may be used
# in a failover flow with both amps failing. Skip it and let
# health manager fix it.
try:
db_amphorae = []
for amp in amphorae:
db_amp = self.amphora_repo.get(db_apis.get_session(),
id=amp[constants.ID])
db_amphorae.append(db_amp)
# TODO(johnsom) Optimize this to use the dicts and not need the
# DB lookups
db_amp = self.amphora_repo.get(
db_apis.get_session(),
id=amphorae[amphora_index][constants.ID])
db_lb = self.loadbalancer_repo.get(
db_apis.get_session(),
id=loadbalancer[constants.LOADBALANCER_ID])
self.amphora_driver.update_amphora_listeners(
db_lb, db_amphorae[amphora_index], timeout_dict)
db_lb, db_amp, timeout_dict)
except Exception as e:
amphora_id = amphorae[amphora_index].get(constants.ID)
LOG.error('Failed to update listeners on amphora %s. Skipping '
@ -148,6 +172,35 @@ class ListenersStart(BaseAmphoraTask):
self.task_utils.mark_listener_prov_status_error(listener.id)
class AmphoraIndexListenersReload(BaseAmphoraTask):
"""Task to reload all listeners on an amphora."""
def execute(self, loadbalancer, amphora_index, amphorae,
timeout_dict=None):
"""Execute listener reload routines for listeners on an amphora."""
if amphorae is None:
return
# TODO(johnsom) Optimize this to use the dicts and not need the
# DB lookups
db_amp = self.amphora_repo.get(
db_apis.get_session(), id=amphorae[amphora_index][constants.ID])
db_lb = self.loadbalancer_repo.get(
db_apis.get_session(),
id=loadbalancer[constants.LOADBALANCER_ID])
if db_lb.listeners:
self.amphora_driver.reload(db_lb, db_amp, timeout_dict)
def revert(self, loadbalancer, *args, **kwargs):
"""Handle failed listeners reloads."""
LOG.warning("Reverting listener reload.")
db_lb = self.loadbalancer_repo.get(
db_apis.get_session(),
id=loadbalancer[constants.LOADBALANCER_ID])
for listener in db_lb.listeners:
self.task_utils.mark_listener_prov_status_error(listener.id)
class ListenerDelete(BaseAmphoraTask):
"""Task to delete the listener on the vip."""
@ -332,96 +385,149 @@ class AmphoraCertUpload(BaseAmphoraTask):
db_amp, fer.decrypt(server_pem.encode('utf-8')))
# TODO(johnsom) REMOVE ME!
class AmphoraUpdateVRRPInterface(BaseAmphoraTask):
"""Task to get and update the VRRP interface device name from amphora."""
def execute(self, loadbalancer):
"""Execute post_vip_routine."""
amps = []
db_lb = self.loadbalancer_repo.get(
db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID])
timeout_dict = {
constants.CONN_MAX_RETRIES:
CONF.haproxy_amphora.active_connection_max_retries,
constants.CONN_RETRY_INTERVAL:
CONF.haproxy_amphora.active_connection_rety_interval}
for amp in filter(
lambda amp: amp.status == constants.AMPHORA_ALLOCATED,
db_lb.amphorae):
def execute(self, amphora, timeout_dict=None):
try:
# TODO(johnsom) Optimize this to use the dicts and not need the
# DB lookups
db_amp = self.amphora_repo.get(db_apis.get_session(),
id=amphora[constants.ID])
interface = self.amphora_driver.get_interface_from_ip(
amp, amp.vrrp_ip, timeout_dict=timeout_dict)
db_amp, db_amp.vrrp_ip, timeout_dict=timeout_dict)
except Exception as e:
# This can occur when an active/standby LB has no listener
LOG.error('Failed to get amphora VRRP interface on amphora '
'%s. Skipping this amphora as it is failing due to: '
'%s', amp.id, str(e))
self.amphora_repo.update(db_apis.get_session(), amp.id,
'%s', amphora.get(constants.ID), str(e))
self.amphora_repo.update(db_apis.get_session(),
amphora.get(constants.ID),
status=constants.ERROR)
continue
return None
self.amphora_repo.update(db_apis.get_session(), amp.id,
self.amphora_repo.update(db_apis.get_session(), amphora[constants.ID],
vrrp_interface=interface)
amps.append(self.amphora_repo.get(db_apis.get_session(),
id=amp.id))
db_lb.amphorae = amps
return provider_utils.db_loadbalancer_to_provider_loadbalancer(
db_lb).to_dict()
return interface
def revert(self, result, loadbalancer, *args, **kwargs):
"""Handle a failed amphora vip plug notification."""
if isinstance(result, failure.Failure):
return
LOG.warning("Reverting Get Amphora VRRP Interface.")
db_lb = self.loadbalancer_repo.get(
db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID])
for amp in filter(
lambda amp: amp.status == constants.AMPHORA_ALLOCATED,
db_lb.amphorae):
class AmphoraIndexUpdateVRRPInterface(BaseAmphoraTask):
"""Task to get and update the VRRP interface device name from amphora."""
def execute(self, amphora_index, amphorae, timeout_dict=None):
amphora_id = amphorae[amphora_index][constants.ID]
try:
self.amphora_repo.update(db_apis.get_session(), amp.id,
vrrp_interface=None)
# TODO(johnsom) Optimize this to use the dicts and not need the
# DB lookups
db_amp = self.amphora_repo.get(db_apis.get_session(),
id=amphora_id)
interface = self.amphora_driver.get_interface_from_ip(
db_amp, db_amp.vrrp_ip, timeout_dict=timeout_dict)
except Exception as e:
LOG.error("Failed to update amphora %(amp)s "
"VRRP interface to None due to: %(except)s",
{'amp': amp.id, 'except': e})
# This can occur when an active/standby LB has no listener
LOG.error('Failed to get amphora VRRP interface on amphora '
'%s. Skipping this amphora as it is failing due to: '
'%s', amphora_id, str(e))
self.amphora_repo.update(db_apis.get_session(), amphora_id,
status=constants.ERROR)
return None
self.amphora_repo.update(db_apis.get_session(), amphora_id,
vrrp_interface=interface)
return interface
class AmphoraVRRPUpdate(BaseAmphoraTask):
"""Task to update the VRRP configuration of the loadbalancer amphorae."""
"""Task to update the VRRP configuration of an amphora."""
def execute(self, loadbalancer, amphorae_network_config):
def execute(self, loadbalancer_id, amphorae_network_config, amphora,
amp_vrrp_int, timeout_dict=None):
"""Execute update_vrrp_conf."""
db_lb = self.loadbalancer_repo.get(
db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID])
self.amphora_driver.update_vrrp_conf(db_lb,
amphorae_network_config)
LOG.debug("Uploaded VRRP configuration of loadbalancer %s amphorae",
loadbalancer[constants.LOADBALANCER_ID])
# Note, we don't want this to cause a revert as it may be used
# in a failover flow with both amps failing. Skip it and let
# health manager fix it.
amphora_id = amphora[constants.ID]
try:
# TODO(johnsom) Optimize this to use the dicts and not need the
# DB lookups
db_amp = self.amphora_repo.get(db_apis.get_session(),
id=amphora_id)
loadbalancer = self.loadbalancer_repo.get(db_apis.get_session(),
id=loadbalancer_id)
db_amp.vrrp_interface = amp_vrrp_int
self.amphora_driver.update_vrrp_conf(
loadbalancer, amphorae_network_config, db_amp, timeout_dict)
except Exception as e:
LOG.error('Failed to update VRRP configuration amphora %s. '
'Skipping this amphora as it is failing to update due '
'to: %s', amphora_id, str(e))
self.amphora_repo.update(db_apis.get_session(), amphora_id,
status=constants.ERROR)
LOG.debug("Uploaded VRRP configuration of amphora %s.", amphora_id)
class AmphoraVRRPStop(BaseAmphoraTask):
"""Task to stop keepalived of all amphorae of a LB."""
class AmphoraIndexVRRPUpdate(BaseAmphoraTask):
"""Task to update the VRRP configuration of an amphora."""
def execute(self, loadbalancer):
db_lb = self.loadbalancer_repo.get(
db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID])
self.amphora_driver.stop_vrrp_service(db_lb)
LOG.debug("Stopped VRRP of loadbalancer %s amphorae",
loadbalancer[constants.LOADBALANCER_ID])
def execute(self, loadbalancer_id, amphorae_network_config, amphora_index,
amphorae, amp_vrrp_int, timeout_dict=None):
"""Execute update_vrrp_conf."""
# Note, we don't want this to cause a revert as it may be used
# in a failover flow with both amps failing. Skip it and let
# health manager fix it.
amphora_id = amphorae[amphora_index][constants.ID]
try:
# TODO(johnsom) Optimize this to use the dicts and not need the
# DB lookups
db_amp = self.amphora_repo.get(db_apis.get_session(),
id=amphora_id)
loadbalancer = self.loadbalancer_repo.get(db_apis.get_session(),
id=loadbalancer_id)
db_amp.vrrp_interface = amp_vrrp_int
self.amphora_driver.update_vrrp_conf(
loadbalancer, amphorae_network_config, db_amp, timeout_dict)
except Exception as e:
LOG.error('Failed to update VRRP configuration amphora %s. '
'Skipping this amphora as it is failing to update due '
'to: %s', amphora_id, str(e))
self.amphora_repo.update(db_apis.get_session(), amphora_id,
status=constants.ERROR)
LOG.debug("Uploaded VRRP configuration of amphora %s.", amphora_id)
class AmphoraVRRPStart(BaseAmphoraTask):
"""Task to start keepalived of all amphorae of a LB."""
"""Task to start keepalived on an amphora.
def execute(self, loadbalancer):
db_lb = self.loadbalancer_repo.get(
db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID])
self.amphora_driver.start_vrrp_service(db_lb)
LOG.debug("Started VRRP of loadbalancer %s amphorae",
loadbalancer[constants.LOADBALANCER_ID])
This will reload keepalived if it is already running.
"""
def execute(self, amphora, timeout_dict=None):
# TODO(johnsom) Optimize this to use the dicts and not need the
# DB lookups
db_amp = self.amphora_repo.get(
db_apis.get_session(), id=amphora[constants.ID])
self.amphora_driver.start_vrrp_service(db_amp, timeout_dict)
LOG.debug("Started VRRP on amphora %s.", amphora[constants.ID])
class AmphoraIndexVRRPStart(BaseAmphoraTask):
"""Task to start keepalived on an amphora.
This will reload keepalived if it is already running.
"""
def execute(self, amphora_index, amphorae, timeout_dict=None):
# TODO(johnsom) Optimize this to use the dicts and not need the
# DB lookups
db_amp = self.amphora_repo.get(
db_apis.get_session(), id=amphorae[amphora_index][constants.ID])
self.amphora_driver.start_vrrp_service(db_amp, timeout_dict)
LOG.debug("Started VRRP on amphora %s.",
amphorae[amphora_index][constants.ID])
class AmphoraComputeConnectivityWait(BaseAmphoraTask):

90
octavia/controller/worker/v2/tasks/compute_tasks.py
View File

@ -21,6 +21,7 @@ from oslo_log import log as logging
from stevedore import driver as stevedore_driver
from taskflow import task
from taskflow.types import failure
import tenacity
from octavia.amphorae.backends.agent import agent_jinja_cfg
from octavia.common import constants
@ -53,10 +54,9 @@ class BaseComputeTask(task.Task):
class ComputeCreate(BaseComputeTask):
"""Create the compute instance for a new amphora."""
def execute(self, amphora_id, config_drive_files=None,
def execute(self, amphora_id, server_group_id, config_drive_files=None,
build_type_priority=constants.LB_CREATE_NORMAL_PRIORITY,
server_group_id=None, ports=None, flavor=None,
availability_zone=None):
ports=None, flavor=None, availability_zone=None):
"""Create an amphora
:param availability_zone: availability zone metadata dictionary
@ -154,10 +154,9 @@ class ComputeCreate(BaseComputeTask):
class CertComputeCreate(ComputeCreate):
def execute(self, amphora_id, server_pem,
def execute(self, amphora_id, server_pem, server_group_id,
build_type_priority=constants.LB_CREATE_NORMAL_PRIORITY,
server_group_id=None, ports=None, flavor=None,
availability_zone=None):
ports=None, flavor=None, availability_zone=None):
"""Create an amphora
:param availability_zone: availability zone metadata dictionary
@ -202,15 +201,52 @@ class DeleteAmphoraeOnLoadBalancer(BaseComputeTask):
class ComputeDelete(BaseComputeTask):
def execute(self, amphora):
LOG.debug("Compute Delete execute for amphora with id %s",
amphora.get(constants.ID))
@tenacity.retry(retry=tenacity.retry_if_exception_type(),
stop=tenacity.stop_after_attempt(CONF.compute.max_retries),
wait=tenacity.wait_exponential(
multiplier=CONF.compute.retry_backoff,
min=CONF.compute.retry_interval,
max=CONF.compute.retry_max), reraise=True)
def execute(self, amphora, passive_failure=False):
amphora_id = amphora.get(constants.ID)
compute_id = amphora[constants.COMPUTE_ID]
if self.execute.retry.statistics.get(constants.ATTEMPT_NUMBER, 1) == 1:
LOG.debug('Compute delete execute for amphora with ID %s and '
'compute ID: %s', amphora_id, compute_id)
else:
LOG.warning('Retrying compute delete of %s attempt %s of %s.',
compute_id,
self.execute.retry.statistics[
constants.ATTEMPT_NUMBER],
self.execute.retry.stop.max_attempt_number)
# Let the Taskflow engine know we are working and alive
# Don't use get with a default for 'attempt_number', we need to fail
# if that number is missing.
self.update_progress(
self.execute.retry.statistics[constants.ATTEMPT_NUMBER] /
self.execute.retry.stop.max_attempt_number)
try:
self.compute.delete(amphora[constants.COMPUTE_ID])
self.compute.delete(compute_id)
except Exception:
LOG.exception("Compute delete for amphora id: %s failed",
amphora.get(constants.ID))
if (self.execute.retry.statistics[constants.ATTEMPT_NUMBER] !=
self.execute.retry.stop.max_attempt_number):
LOG.warning('Compute delete for amphora id: %s failed. '
'Retrying.', amphora_id)
raise
if passive_failure:
LOG.exception('Compute delete for compute ID: %s on amphora '
'ID: %s failed. This resource will be abandoned '
'and should manually be cleaned up once the '
'compute service is functional.',
compute_id, amphora_id)
else:
LOG.exception('Compute delete for compute ID: %s on amphora '
'ID: %s failed. The compute service has failed. '
'Aborting and reverting.', compute_id,
amphora_id)
raise
@ -284,3 +320,33 @@ class NovaServerGroupDelete(BaseComputeTask):
self.compute.delete_server_group(server_group_id)
else:
return
class AttachPort(BaseComputeTask):
def execute(self, amphora, port):
"""Attach a port to an amphora instance.
:param amphora: The amphora to attach the port to.
:param port: The port to attach to the amphora.
:returns: None
"""
LOG.debug('Attaching port: %s to compute: %s',
port[constants.ID], amphora[constants.COMPUTE_ID])
self.compute.attach_network_or_port(amphora[constants.COMPUTE_ID],
port_id=port[constants.ID])
def revert(self, amphora, port, *args, **kwargs):
"""Revert our port attach.
:param amphora: The amphora to detach the port from.
:param port: The port to attach to the amphora.
"""
LOG.warning('Reverting port: %s attach to compute: %s',
port[constants.ID], amphora[constants.COMPUTE_ID])
try:
self.compute.detach_port(amphora[constants.COMPUTE_ID],
port[constants.ID])
except Exception as e:
LOG.error('Failed to detach port %s from compute %s for revert '
'due to %s.', port[constants.ID],
amphora[constants.COMPUTE_ID], str(e))

122
octavia/controller/worker/v2/tasks/database_tasks.py
View File

@ -280,6 +280,7 @@ class DeleteListenerInDB(BaseDatabaseTask):
:returns: None
"""
# TODO(johnsom) Fix this, it doesn't revert anything
LOG.warning("Reverting mark listener delete in DB for listener id %s",
listener[constants.LISTENER_ID])
@ -490,22 +491,26 @@ class UpdateAmphoraVIPData(BaseDatabaseTask):
class UpdateAmpFailoverDetails(BaseDatabaseTask):
"""Update amphora failover details in the database."""
def execute(self, amphora, amp_data):
def execute(self, amphora, vip, base_port):
"""Update amphora failover details in the database.
:param amphora: The amphora to update
:param amp_data: data_models.Amphora object with update data
:param vip: The VIP object associated with this amphora.
:param base_port: The base port object associated with the amphora.
:returns: None
"""
# role and vrrp_priority will be updated later.
self.repos.amphora.update(
db_apis.get_session(),
amphora.get(constants.ID),
vrrp_ip=amp_data[constants.VRRP_IP],
ha_ip=amp_data[constants.HA_IP],
vrrp_port_id=amp_data[constants.VRRP_PORT_ID],
ha_port_id=amp_data[constants.HA_PORT_ID],
vrrp_id=amp_data[constants.VRRP_ID])
# TODO(johnsom) We should do a better job getting the fixed_ip
# as this could be a problem with dual stack.
# Fix this during the multi-vip patch.
vrrp_ip=base_port[constants.FIXED_IPS][0][constants.IP_ADDRESS],
ha_ip=vip[constants.IP_ADDRESS],
vrrp_port_id=base_port[constants.ID],
ha_port_id=vip[constants.PORT_ID],
vrrp_id=1)
class AssociateFailoverAmphoraWithLBID(BaseDatabaseTask):
@ -595,26 +600,24 @@ class MapLoadbalancerToAmphora(BaseDatabaseTask):
class _MarkAmphoraRoleAndPriorityInDB(BaseDatabaseTask):
"""Alter the amphora role and priority in DB."""
def _execute(self, amphora, amp_role, vrrp_priority):
def _execute(self, amphora_id, amp_role, vrrp_priority):
"""Alter the amphora role and priority in DB.
:param amphora: Amphora to update.
:param amphora_id: Amphora ID to update.
:param amp_role: Amphora role to be set.
:param vrrp_priority: VRRP priority to set.
:returns: None
"""
LOG.debug("Mark %(role)s in DB for amphora: %(amp)s",
{constants.ROLE: amp_role, 'amp': amphora[constants.ID]})
self.amphora_repo.update(db_apis.get_session(),
amphora[constants.ID],
role=amp_role,
vrrp_priority=vrrp_priority)
{constants.ROLE: amp_role, 'amp': amphora_id})
self.amphora_repo.update(db_apis.get_session(), amphora_id,
role=amp_role, vrrp_priority=vrrp_priority)
def _revert(self, result, amphora, *args, **kwargs):
def _revert(self, result, amphora_id, *args, **kwargs):
"""Removes role and vrrp_priority association.
:param result: Result of the association.
:param amphora: Amphora which role/vrrp_priority association
:param amphora_id: Amphora ID which role/vrrp_priority association
failed.
:returns: None
"""
@ -623,17 +626,14 @@ class _MarkAmphoraRoleAndPriorityInDB(BaseDatabaseTask):
return
LOG.warning("Reverting amphora role in DB for amp id %(amp)s",
{'amp': amphora[constants.ID]})
{'amp': amphora_id})
try:
self.amphora_repo.update(db_apis.get_session(),
amphora[constants.ID],
role=None,
vrrp_priority=None)
self.amphora_repo.update(db_apis.get_session(), amphora_id,
role=None, vrrp_priority=None)
except Exception as e:
LOG.error("Failed to update amphora %(amp)s "
"role and vrrp_priority to None due to: "
"%(except)s", {'amp': amphora[constants.ID],
'except': e})
"%(except)s", {'amp': amphora_id, 'except': e})
class MarkAmphoraMasterInDB(_MarkAmphoraRoleAndPriorityInDB):
@ -646,7 +646,8 @@ class MarkAmphoraMasterInDB(_MarkAmphoraRoleAndPriorityInDB):
:returns: None
"""
amp_role = constants.ROLE_MASTER
self._execute(amphora, amp_role, constants.ROLE_MASTER_PRIORITY)
self._execute(amphora[constants.ID], amp_role,
constants.ROLE_MASTER_PRIORITY)
def revert(self, result, amphora, *args, **kwargs):
"""Removes amphora role association.
@ -654,7 +655,7 @@ class MarkAmphoraMasterInDB(_MarkAmphoraRoleAndPriorityInDB):
:param amphora: Amphora to update role.
:returns: None
"""
self._revert(result, amphora, *args, **kwargs)
self._revert(result, amphora[constants.ID], *args, **kwargs)
class MarkAmphoraBackupInDB(_MarkAmphoraRoleAndPriorityInDB):
@ -667,7 +668,8 @@ class MarkAmphoraBackupInDB(_MarkAmphoraRoleAndPriorityInDB):
:returns: None
"""
amp_role = constants.ROLE_BACKUP
self._execute(amphora, amp_role, constants.ROLE_BACKUP_PRIORITY)
self._execute(amphora[constants.ID], amp_role,
constants.ROLE_BACKUP_PRIORITY)
def revert(self, result, amphora, *args, **kwargs):
"""Removes amphora role association.
@ -675,7 +677,7 @@ class MarkAmphoraBackupInDB(_MarkAmphoraRoleAndPriorityInDB):
:param amphora: Amphora to update role.
:returns: None
"""
self._revert(result, amphora, *args, **kwargs)
self._revert(result, amphora[constants.ID], *args, **kwargs)
class MarkAmphoraStandAloneInDB(_MarkAmphoraRoleAndPriorityInDB):
@ -688,7 +690,7 @@ class MarkAmphoraStandAloneInDB(_MarkAmphoraRoleAndPriorityInDB):
:returns: None
"""
amp_role = constants.ROLE_STANDALONE
self._execute(amphora, amp_role, None)
self._execute(amphora[constants.ID], amp_role, None)
def revert(self, result, amphora, *args, **kwargs):
"""Removes amphora role association.
@ -696,7 +698,7 @@ class MarkAmphoraStandAloneInDB(_MarkAmphoraRoleAndPriorityInDB):
:param amphora: Amphora to update role.
:returns: None
"""
self._revert(result, amphora, *args, **kwargs)
self._revert(result, amphora[constants.ID], *args, **kwargs)
class MarkAmphoraAllocatedInDB(BaseDatabaseTask):
@ -809,10 +811,10 @@ class MarkAmphoraDeletedInDB(BaseDatabaseTask):
LOG.debug("Mark DELETED in DB for amphora: %(amp)s with "
"compute id %(comp)s",
{'amp': amphora.get(constants.ID),
{'amp': amphora[constants.ID],
'comp': amphora[constants.COMPUTE_ID]})
self.amphora_repo.update(db_apis.get_session(),
amphora.get(constants.ID),
amphora[constants.ID],
status=constants.DELETED)
def revert(self, amphora, *args, **kwargs):
@ -824,10 +826,10 @@ class MarkAmphoraDeletedInDB(BaseDatabaseTask):
LOG.warning("Reverting mark amphora deleted in DB "
"for amp id %(amp)s and compute id %(comp)s",
{'amp': amphora.get(constants.ID),
{'amp': amphora[constants.ID],
'comp': amphora[constants.COMPUTE_ID]})
self.task_utils.mark_amphora_status_error(amphora.get(constants.ID))
self.task_utils.mark_amphora_status_error(amphora[constants.ID])
class MarkAmphoraPendingDeleteInDB(BaseDatabaseTask):
@ -845,10 +847,10 @@ class MarkAmphoraPendingDeleteInDB(BaseDatabaseTask):
LOG.debug("Mark PENDING DELETE in DB for amphora: %(amp)s "
"with compute id %(id)s",
{'amp': amphora.get(constants.ID),
{'amp': amphora[constants.ID],
'id': amphora[constants.COMPUTE_ID]})
self.amphora_repo.update(db_apis.get_session(),
amphora.get(constants.ID),
amphora[constants.ID],
status=constants.PENDING_DELETE)
def revert(self, amphora, *args, **kwargs):
@ -860,9 +862,9 @@ class MarkAmphoraPendingDeleteInDB(BaseDatabaseTask):
LOG.warning("Reverting mark amphora pending delete in DB "
"for amp id %(amp)s and compute id %(comp)s",
{'amp': amphora.get(constants.ID),
{'amp': amphora[constants.ID],
'comp': amphora[constants.COMPUTE_ID]})
self.task_utils.mark_amphora_status_error(amphora.get(constants.ID))
self.task_utils.mark_amphora_status_error(amphora[constants.ID])
class MarkAmphoraPendingUpdateInDB(BaseDatabaseTask):
@ -1111,7 +1113,7 @@ class MarkLBActiveInDB(BaseDatabaseTask):
def _mark_member_status(self, member, status):
self.member_repo.update(
db_apis.get_session(), member[constants.MEMBER_ID],
db_apis.get_session(), member.id,
provisioning_status=status)
def revert(self, loadbalancer, *args, **kwargs):
@ -1691,17 +1693,17 @@ class GetAmphoraDetails(BaseDatabaseTask):
class GetAmphoraeFromLoadbalancer(BaseDatabaseTask):
"""Task to pull the listeners from a loadbalancer."""
"""Task to pull the amphorae from a loadbalancer."""
def execute(self, loadbalancer):
def execute(self, loadbalancer_id):
"""Pull the amphorae from a loadbalancer.
:param loadbalancer: Load balancer which listeners are required
:param loadbalancer_id: Load balancer ID to get amphorae from
:returns: A list of Listener objects
"""
amphorae = []
db_lb = self.repos.load_balancer.get(
db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID])
db_lb = self.repos.load_balancer.get(db_apis.get_session(),
id=loadbalancer_id)
for amp in db_lb.amphorae:
a = self.amphora_repo.get(db_apis.get_session(), id=amp.id,
show_deleted=False)
@ -1746,29 +1748,45 @@ class GetVipFromLoadbalancer(BaseDatabaseTask):
return db_lb.vip.to_dict(recurse=True)
class GetLoadBalancer(BaseDatabaseTask):
"""Get an load balancer object from the database."""
def execute(self, loadbalancer_id, *args, **kwargs):
"""Get an load balancer object from the database.
:param loadbalancer_id: The load balancer ID to lookup
:returns: The load balancer object
"""
LOG.debug("Get load balancer from DB for load balancer id: %s",
loadbalancer_id)
db_lb = self.loadbalancer_repo.get(db_apis.get_session(),
id=loadbalancer_id)
provider_lb = provider_utils.db_loadbalancer_to_provider_loadbalancer(
db_lb)
return provider_lb.to_dict()
class CreateVRRPGroupForLB(BaseDatabaseTask):
"""Create a VRRP group for a load balancer."""
def execute(self, loadbalancer):
def execute(self, loadbalancer_id):
"""Create a VRRP group for a load balancer.
:param loadbalancer: Load balancer for which a VRRP group
:param loadbalancer_id: Load balancer ID for which a VRRP group
should be created
:returns: Updated load balancer
"""
try:
self.repos.vrrpgroup.create(
db_apis.get_session(),
load_balancer_id=loadbalancer[constants.LOADBALANCER_ID],
vrrp_group_name=str(
loadbalancer[constants.LOADBALANCER_ID]).replace('-', ''),
load_balancer_id=loadbalancer_id,
vrrp_group_name=str(loadbalancer_id).replace('-', ''),
vrrp_auth_type=constants.VRRP_AUTH_DEFAULT,
vrrp_auth_pass=uuidutils.generate_uuid().replace('-', '')[0:7],
advert_int=CONF.keepalived_vrrp.vrrp_advert_int)
except odb_exceptions.DBDuplicateEntry:
LOG.debug('VRRP_GROUP entry already exists for load balancer, '
'skipping create.')
return loadbalancer
class DisableAmphoraHealthMonitoring(BaseDatabaseTask):
@ -1784,7 +1802,7 @@ class DisableAmphoraHealthMonitoring(BaseDatabaseTask):
:param amphora: The amphora to disable health monitoring for
:returns: None
"""
self._delete_from_amp_health(amphora.get(constants.ID))
self._delete_from_amp_health(amphora[constants.ID])
class DisableLBAmphoraeHealthMonitoring(BaseDatabaseTask):
@ -1819,7 +1837,7 @@ class MarkAmphoraHealthBusy(BaseDatabaseTask):
:param amphora: The amphora to mark amphora health busy
:returns: None
"""
self._mark_amp_health_busy(amphora.get(constants.ID))
self._mark_amp_health_busy(amphora[constants.ID])
class MarkLBAmphoraeHealthBusy(BaseDatabaseTask):

254
octavia/controller/worker/v2/tasks/network_tasks.py
View File

@ -12,11 +12,14 @@
# License for the specific language governing permissions and limitations
# under the License.
#
import time
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import excutils
from taskflow import task
from taskflow.types import failure
import tenacity
from octavia.common import constants
from octavia.common import data_models
@ -52,21 +55,27 @@ class CalculateAmphoraDelta(BaseNetworkTask):
default_provides = constants.DELTA
def execute(self, loadbalancer, amphora, availability_zone):
def execute(self, loadbalancer, amphora, availability_zone,
vrrp_port=None):
LOG.debug("Calculating network delta for amphora id: %s",
amphora.get(constants.ID))
# Figure out what networks we want
# seed with lb network(s)
if vrrp_port is None:
vrrp_port = self.network_driver.get_port(
amphora[constants.VRRP_PORT_ID])
vrrp_port_network_id = vrrp_port.network_id
else:
vrrp_port_network_id = vrrp_port[constants.NETWORK_ID]
# Figure out what networks we want
# seed with lb network(s)
if availability_zone:
management_nets = (
[availability_zone.get(constants.MANAGEMENT_NETWORK)] or
CONF.controller_worker.amp_boot_network_list)
else:
management_nets = CONF.controller_worker.amp_boot_network_list
desired_network_ids = {vrrp_port.network_id}.union(management_nets)
desired_network_ids = {vrrp_port_network_id}.union(management_nets)
db_lb = self.loadbalancer_repo.get(
db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID])
for pool in db_lb.pools:
@ -84,7 +93,7 @@ class CalculateAmphoraDelta(BaseNetworkTask):
del_ids = set(actual_network_nics) - desired_network_ids
delete_nics = list(
actual_network_nics[net_id] for net_id in del_ids)
n_data_models.Interface(network_id=net_id) for net_id in del_ids)
add_ids = desired_network_ids - set(actual_network_nics)
add_nics = list(n_data_models.Interface(
@ -353,7 +362,8 @@ class PlugVIP(BaseNetworkTask):
LOG.debug("Plumbing VIP for loadbalancer id: %s",
loadbalancer[constants.LOADBALANCER_ID])
db_lb = self.loadbalancer_repo.get(
db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID])
db_apis.get_session(),
id=loadbalancer[constants.LOADBALANCER_ID])
amps_data = self.network_driver.plug_vip(db_lb,
db_lb.vip)
return [amp.to_dict() for amp in amps_data]
@ -367,7 +377,8 @@ class PlugVIP(BaseNetworkTask):
loadbalancer[constants.LOADBALANCER_ID])
db_lb = self.loadbalancer_repo.get(
db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID])
db_apis.get_session(),
id=loadbalancer[constants.LOADBALANCER_ID])
try:
# Make sure we have the current port IDs for cleanup
for amp_data in result:
@ -388,13 +399,12 @@ class PlugVIP(BaseNetworkTask):
class UpdateVIPSecurityGroup(BaseNetworkTask):
"""Task to setup SG for LB."""
def execute(self, loadbalancer):
def execute(self, loadbalancer_id):
"""Task to setup SG for LB."""
LOG.debug("Setup SG for loadbalancer id: %s",
loadbalancer[constants.LOADBALANCER_ID])
LOG.debug("Setup SG for loadbalancer id: %s", loadbalancer_id)
db_lb = self.loadbalancer_repo.get(
db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID])
db_apis.get_session(), id=loadbalancer_id)
self.network_driver.update_vip_sg(db_lb, db_lb.vip)
@ -411,7 +421,7 @@ class GetSubnetFromVIP(BaseNetworkTask):
loadbalancer['vip_subnet_id']).to_dict()
class PlugVIPAmpphora(BaseNetworkTask):
class PlugVIPAmphora(BaseNetworkTask):
"""Task to plumb a VIP."""
def execute(self, loadbalancer, amphora, subnet):
@ -561,13 +571,29 @@ class GetAmphoraNetworkConfigs(BaseNetworkTask):
return provider_dict
class GetAmphoraNetworkConfigsByID(BaseNetworkTask):
"""Task to retrieve amphora network details."""
def execute(self, loadbalancer_id, amphora_id=None):
LOG.debug("Retrieving vip network details.")
loadbalancer = self.loadbalancer_repo.get(db_apis.get_session(),
id=loadbalancer_id)
amphora = self.amphora_repo.get(db_apis.get_session(), id=amphora_id)
db_configs = self.network_driver.get_network_configs(loadbalancer,
amphora=amphora)
provider_dict = {}
for amp_id, amp_conf in db_configs.items():
provider_dict[amp_id] = amp_conf.to_dict(recurse=True)
return provider_dict
class GetAmphoraeNetworkConfigs(BaseNetworkTask):
"""Task to retrieve amphorae network details."""
def execute(self, loadbalancer):
def execute(self, loadbalancer_id):
LOG.debug("Retrieving vip network details.")
db_lb = self.loadbalancer_repo.get(
db_apis.get_session(), id=loadbalancer[constants.LOADBALANCER_ID])
db_apis.get_session(), id=loadbalancer_id)
db_configs = self.network_driver.get_network_configs(db_lb)
provider_dict = {}
for amp_id, amp_conf in db_configs.items():
@ -625,56 +651,13 @@ class PlugPorts(BaseNetworkTask):
self.network_driver.plug_port(db_amp, port)
class PlugVIPPort(BaseNetworkTask):
"""Task to plug a VIP into a compute instance."""
def execute(self, amphora, amphorae_network_config):
vrrp_port = amphorae_network_config.get(
amphora.get(constants.ID))[constants.VRRP_PORT]
LOG.debug('Plugging VIP VRRP port ID: %(port_id)s into compute '
'instance: %(compute_id)s.',
{constants.PORT_ID: vrrp_port.get(constants.ID),
constants.COMPUTE_ID: amphora[constants.COMPUTE_ID]})
db_vrrp_port = self.network_driver.get_port(
vrrp_port.get(constants.ID))
db_amp = self.amphora_repo.get(db_apis.get_session(),
id=amphora[constants.ID])
self.network_driver.plug_port(db_amp, db_vrrp_port)
def revert(self, result, amphora, amphorae_network_config,
*args, **kwargs):
vrrp_port = None
try:
vrrp_port = amphorae_network_config.get(
amphora.get(constants.ID))[constants.VRRP_PORT]
db_vrrp_port = self.network_driver.get_port(
vrrp_port.get(constants.ID))
db_amp = self.amphora_repo.get(db_apis.get_session(),
id=amphora[constants.ID])
self.network_driver.unplug_port(db_amp, db_vrrp_port)
except Exception:
LOG.warning('Failed to unplug vrrp port: %(port)s from amphora: '
'%(amp)s',
{'port': vrrp_port, 'amp': amphora[constants.ID]})
class WaitForPortDetach(BaseNetworkTask):
"""Task to wait for the neutron ports to detach from an amphora."""
def execute(self, amphora):
LOG.debug('Waiting for ports to detach from amphora: %(amp_id)s.',
{'amp_id': amphora.get(constants.ID)})
db_amp = self.amphora_repo.get(db_apis.get_session(),
id=amphora.get(constants.ID))
self.network_driver.wait_for_port_detach(db_amp)
class ApplyQos(BaseNetworkTask):
"""Apply Quality of Services to the VIP"""
def _apply_qos_on_vrrp_ports(self, loadbalancer, amps_data, qos_policy_id,
is_revert=False, request_qos_id=None):
"""Call network driver to apply QoS Policy on the vrrp ports."""
if not amps_data:
db_lb = self.loadbalancer_repo.get(
db_apis.get_session(),
@ -688,12 +671,21 @@ class ApplyQos(BaseNetworkTask):
def execute(self, loadbalancer, amps_data=None, update_dict=None):
"""Apply qos policy on the vrrp ports which are related with vip."""
qos_policy_id = loadbalancer['vip_qos_policy_id']
db_lb = self.loadbalancer_repo.get(
db_apis.get_session(),
id=loadbalancer[constants.LOADBALANCER_ID])
qos_policy_id = db_lb.vip.qos_policy_id
if not qos_policy_id and (
not update_dict or (
'vip' not in update_dict or
'qos_policy_id' not in update_dict[constants.VIP])):
return
if update_dict and update_dict.get(constants.VIP):
vip_dict = update_dict[constants.VIP]
if vip_dict.get(constants.QOS_POLICY_ID):
qos_policy_id = vip_dict[constants.QOS_POLICY_ID]
self._apply_qos_on_vrrp_ports(loadbalancer, amps_data, qos_policy_id)
def revert(self, result, loadbalancer, amps_data=None, update_dict=None,
@ -756,3 +748,147 @@ class ApplyQosAmphora(BaseNetworkTask):
LOG.error('Failed to remove QoS policy: %s from port: %s due '
'to error: %s', orig_qos_id,
amp_data[constants.VRRP_PORT_ID], e)
class DeletePort(BaseNetworkTask):
"""Task to delete a network port."""
@tenacity.retry(retry=tenacity.retry_if_exception_type(),
stop=tenacity.stop_after_attempt(
CONF.networking.max_retries),
wait=tenacity.wait_exponential(
multiplier=CONF.networking.retry_backoff,
min=CONF.networking.retry_interval,
max=CONF.networking.retry_max), reraise=True)
def execute(self, port_id, passive_failure=False):
"""Delete the network port."""
if port_id is None:
return
if self.execute.retry.statistics.get(constants.ATTEMPT_NUMBER, 1) == 1:
LOG.debug("Deleting network port %s", port_id)
else:
LOG.warning('Retrying network port %s delete attempt %s of %s.',
port_id,
self.execute.retry.statistics[
constants.ATTEMPT_NUMBER],
self.execute.retry.stop.max_attempt_number)
# Let the Taskflow engine know we are working and alive
# Don't use get with a default for 'attempt_number', we need to fail
# if that number is missing.
self.update_progress(
self.execute.retry.statistics[constants.ATTEMPT_NUMBER] /
self.execute.retry.stop.max_attempt_number)
try:
self.network_driver.delete_port(port_id)
except Exception:
if (self.execute.retry.statistics[constants.ATTEMPT_NUMBER] !=
self.execute.retry.stop.max_attempt_number):
LOG.warning('Network port delete for port id: %s failed. '
'Retrying.', port_id)
raise
if passive_failure:
LOG.exception('Network port delete for port ID: %s failed. '
'This resource will be abandoned and should '
'manually be cleaned up once the '
'network service is functional.', port_id)
# Let's at least attempt to disable it so if the instance
# comes back from the dead it doesn't conflict with anything.
try:
self.network_driver.admin_down_port(port_id)
LOG.info('Successfully disabled (admin down) network port '
'%s that failed to delete.', port_id)
except Exception:
LOG.warning('Attempt to disable (admin down) network port '
'%s failed. The network service has failed. '
'Continuing.', port_id)
else:
LOG.exception('Network port delete for port ID: %s failed. '
'The network service has failed. '
'Aborting and reverting.', port_id)
raise
class CreateVIPBasePort(BaseNetworkTask):
"""Task to create the VIP base port for an amphora."""
@tenacity.retry(retry=tenacity.retry_if_exception_type(),
stop=tenacity.stop_after_attempt(
CONF.networking.max_retries),
wait=tenacity.wait_exponential(
multiplier=CONF.networking.retry_backoff,
min=CONF.networking.retry_interval,
max=CONF.networking.retry_max), reraise=True)
def execute(self, vip, vip_sg_id, amphora_id):
port_name = constants.AMP_BASE_PORT_PREFIX + amphora_id
fixed_ips = [{constants.SUBNET_ID: vip[constants.SUBNET_ID]}]
sg_id = []
if vip_sg_id:
sg_id = [vip_sg_id]
port = self.network_driver.create_port(
vip[constants.NETWORK_ID], name=port_name, fixed_ips=fixed_ips,
secondary_ips=[vip[constants.IP_ADDRESS]],
security_group_ids=sg_id,
qos_policy_id=vip[constants.QOS_POLICY_ID])
LOG.info('Created port %s with ID %s for amphora %s',
port_name, port.id, amphora_id)
return port.to_dict(recurse=True)
def revert(self, result, vip, vip_sg_id, amphora_id, *args, **kwargs):
if isinstance(result, failure.Failure):
return
try:
port_name = constants.AMP_BASE_PORT_PREFIX + amphora_id
for port in result:
self.network_driver.delete_port(port.id)
LOG.info('Deleted port %s with ID %s for amphora %s due to a '
'revert.', port_name, port.id, amphora_id)
except Exception as e:
LOG.error('Failed to delete port %s. Resources may still be in '
'use for a port intended for amphora %s due to error '
'%s. Search for a port named %s',
result, amphora_id, str(e), port_name)
class AdminDownPort(BaseNetworkTask):
def execute(self, port_id):
try:
self.network_driver.set_port_admin_state_up(port_id, False)
except base.PortNotFound:
return
for i in range(CONF.networking.max_retries):
port = self.network_driver.get_port(port_id)
if port.status == constants.DOWN:
LOG.debug('Disabled port: %s', port_id)
return
LOG.debug('Port %s is %s instead of DOWN, waiting.',
port_id, port.status)
time.sleep(CONF.networking.retry_interval)
LOG.error('Port %s failed to go DOWN. Port status is still %s. '
'Ignoring and continuing.', port_id, port.status)
def revert(self, result, port_id, *args, **kwargs):
if isinstance(result, failure.Failure):
return
try:
self.network_driver.set_port_admin_state_up(port_id, True)
except Exception as e:
LOG.error('Failed to bring port %s admin up on revert due to: %s.',
port_id, str(e))
class GetVIPSecurityGroupID(BaseNetworkTask):
def execute(self, loadbalancer_id):
sg_name = utils.get_vip_security_group_name(loadbalancer_id)
try:
security_group = self.network_driver.get_security_group(sg_name)
if security_group:
return security_group.id
except base.SecurityGroupNotFound:
with excutils.save_and_reraise_exception() as ctxt:
if self.network_driver.sec_grp_enabled:
LOG.error('VIP security group %s was not found.', sg_name)
else:
ctxt.reraise = False
return None

73
octavia/controller/worker/v2/tasks/retry_tasks.py Normal file
View File

@ -0,0 +1,73 @@
# Copyright 2019 Red Hat, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import time
from oslo_log import log as logging
from taskflow import retry
LOG = logging.getLogger(__name__)
class SleepingRetryTimesController(retry.Times):
"""A retry controller to attempt subflow retries a number of times.
This retry controller overrides the Times on_failure to inject a
sleep interval between retries.
It also adds a log message when all of the retries are exhausted.
:param attempts: number of attempts to retry the associated subflow
before giving up
:type attempts: int
:param name: Meaningful name for this atom, should be something that is
distinguishable and understandable for notification,
debugging, storing and any other similar purposes.
:param provides: A set, string or list of items that
this will be providing (or could provide) to others, used
to correlate and associate the thing/s this atom
produces, if it produces anything at all.
:param requires: A set or list of required inputs for this atom's
``execute`` method.
:param rebind: A dict of key/value pairs used to define argument
name conversions for inputs to this atom's ``execute``
method.
:param revert_all: when provided this will cause the full flow to revert
when the number of attempts that have been tried
has been reached (when false, it will only locally
revert the associated subflow)
:type revert_all: bool
:param interval: Interval, in seconds, between retry attempts.
:type interval: int
"""
def __init__(self, attempts=1, name=None, provides=None, requires=None,
auto_extract=True, rebind=None, revert_all=False, interval=1):
super().__init__(attempts, name, provides, requires, auto_extract,
rebind, revert_all)
self._interval = interval
def on_failure(self, history, *args, **kwargs):
if len(history) < self._attempts:
LOG.warning('%s attempt %s of %s failed. Sleeping %s seconds and '
'retrying.',
self.name[self.name.startswith('retry-') and
len('retry-'):], len(history),
self._attempts, self._interval)
time.sleep(self._interval)
return retry.RETRY
return self._revert_action
def revert(self, history, *args, **kwargs):
LOG.error('%s retries with interval %s seconds have failed for %s. '
'Giving up.', len(history), self._interval, self.name)

22
octavia/tests/unit/controller/worker/v1/flows/test_amphora_flows.py
View File

@ -315,21 +315,9 @@ class TestAmphoraFlows(base.TestCase):
self.assertIsInstance(amp_flow, flow.Flow)
# TODO(johnsom) Uncomment after amphora failover builds a replacement
# amphora.
# self.assertIn(constants.AVAILABILITY_ZONE, amp_flow.requires)
# self.assertIn(constants.BUILD_TYPE_PRIORITY, amp_flow.requires)
# self.assertIn(constants.FLAVOR, amp_flow.requires)
# self.assertEqual(5, len(amp_flow.requires))
self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires)
# self.assertIn(constants.AMPHORA, amp_flow.provides)
# self.assertIn(constants.AMPHORA_ID, amp_flow.provides)
# self.assertIn(constants.COMPUTE_ID, amp_flow.provides)
# self.assertIn(constants.COMPUTE_OBJ, amp_flow.provides)
# self.assertIn(constants.SERVER_PEM, amp_flow.provides)
# self.assertIn(constants.VIP_SG_ID, amp_flow.provides)
# self.assertEqual(6, len(amp_flow.provides))
self.assertIn(constants.VIP_SG_ID, amp_flow.provides)
self.assertEqual(1, len(amp_flow.requires))
self.assertEqual(1, len(amp_flow.provides))
@ -340,14 +328,10 @@ class TestAmphoraFlows(base.TestCase):
self.assertIsInstance(amp_flow, flow.Flow)
# TODO(johnsom) Uncomment after amphora failover builds a replacement
# amphora.
# self.assertIn(constants.AVAILABILITY_ZONE, amp_flow.requires)
# self.assertIn(constants.BUILD_TYPE_PRIORITY, amp_flow.requires)
# self.assertEqual(5, len(amp_flow.requires))
# self.assertEqual(6, len(amp_flow.provides))
self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires)
self.assertIn(constants.VIP_SG_ID, amp_flow.provides)
self.assertEqual(1, len(amp_flow.requires))
self.assertEqual(1, len(amp_flow.provides))

21
octavia/tests/unit/controller/worker/v1/tasks/test_amphora_driver_tasks.py
View File

@ -203,13 +203,13 @@ class TestAmphoraDriverTasks(base.TestCase):
# Test no listeners
mock_lb.listeners = None
listeners_reload_obj.execute(mock_lb, None, 0)
listeners_reload_obj.execute(mock_lb, 0, None)
mock_driver.reload.assert_not_called()
# Test with listeners
mock_driver.start.reset_mock()
mock_lb.listeners = [mock_listener]
listeners_reload_obj.execute(mock_lb, [amphora_mock], 0,
listeners_reload_obj.execute(mock_lb, 0, [amphora_mock],
timeout_dict=self.timeout_dict)
mock_driver.reload.assert_called_once_with(mock_lb, amphora_mock,
self.timeout_dict)
@ -620,7 +620,7 @@ class TestAmphoraDriverTasks(base.TestCase):
amphora_update_vrrp_interface_obj = (
amphora_driver_tasks.AmphoraIndexUpdateVRRPInterface())
amphora_update_vrrp_interface_obj.execute(
[_amphora_mock], 0, timeout_dict)
0, [_amphora_mock], timeout_dict)
mock_driver.get_interface_from_ip.assert_called_once_with(
_amphora_mock, _amphora_mock.vrrp_ip, timeout_dict=timeout_dict)
mock_amphora_repo_update.assert_called_once_with(
@ -629,7 +629,7 @@ class TestAmphoraDriverTasks(base.TestCase):
# Test with an exception
mock_amphora_repo_update.reset_mock()
amphora_update_vrrp_interface_obj.execute(
[_amphora_mock], 0, timeout_dict)
0, [_amphora_mock], timeout_dict)
mock_amphora_repo_update.assert_called_once_with(
_session_mock, _amphora_mock.id, status=constants.ERROR)
@ -692,19 +692,6 @@ class TestAmphoraDriverTasks(base.TestCase):
mock_amphora_repo_update.assert_called_once_with(
_session_mock, _amphora_mock.id, status=constants.ERROR)
def test_amphora_vrrp_stop(self,
mock_driver,
mock_generate_uuid,
mock_log,
mock_get_session,
mock_listener_repo_get,
mock_listener_repo_update,
mock_amphora_repo_update):
amphora_vrrp_stop_obj = (
amphora_driver_tasks.AmphoraVRRPStop())
amphora_vrrp_stop_obj.execute(_LB_mock)
mock_driver.stop_vrrp_service.assert_called_once_with(_LB_mock)
def test_amphora_vrrp_start(self,
mock_driver,
mock_generate_uuid,

48
octavia/tests/unit/controller/worker/v1/test_controller_worker.py
View File

@ -1599,54 +1599,6 @@ class TestControllerWorker(base.TestCase):
assert_called_once_with(FAKE_FLOW, store=expected_stored_params))
_flow_mock.run.assert_called_once_with()
@mock.patch(
'octavia.db.repositories.AmphoraRepository.get_lb_for_amphora',
return_value=None)
@mock.patch('octavia.controller.worker.v1.flows.'
'amphora_flows.AmphoraFlows.get_failover_amphora_flow',
return_value=_flow_mock)
def test_failover_spare_amphora(self,
mock_get_failover_flow,
mock_get_lb_for_amphora,
mock_api_get_session,
mock_dyn_log_listener,
mock_taskflow_load,
mock_pool_repo_get,
mock_member_repo_get,
mock_l7rule_repo_get,
mock_l7policy_repo_get,
mock_listener_repo_get,
mock_lb_repo_get,
mock_health_mon_repo_get,
mock_amp_repo_get):
_flow_mock.reset_mock()
# simulate a spare amphora (amphora not attached to any load_balancer)
mock_amphora = mock.MagicMock()
mock_amphora.id = AMP_ID
mock_amphora.status = constants.AMPHORA_READY
mock_amphora.load_balancer_id = None
mock_amp_repo_get.return_value = mock_amphora
cw = controller_worker.ControllerWorker()
cw.failover_amphora(AMP_ID)
(base_taskflow.BaseTaskFlowEngine.taskflow_load.
assert_called_once_with(
_flow_mock,
store={constants.LOADBALANCER: None,
constants.LOADBALANCER_ID: None,
constants.BUILD_TYPE_PRIORITY:
constants.LB_CREATE_FAILOVER_PRIORITY,
constants.FLAVOR: {},
constants.SERVER_GROUP_ID: None,
constants.AVAILABILITY_ZONE: {},
constants.VIP: None
}))
_flow_mock.run.assert_called_once_with()
@mock.patch('octavia.db.repositories.AmphoraHealthRepository.delete')
def test_failover_deleted_amphora(self,
mock_delete,

169
octavia/tests/unit/controller/worker/v2/flows/test_amphora_flows.py
View File

@ -16,6 +16,7 @@ from unittest import mock
from oslo_config import cfg
from oslo_config import fixture as oslo_fixture
from oslo_utils import uuidutils
from taskflow.patterns import linear_flow as flow
from octavia.common import constants
@ -42,6 +43,7 @@ class TestAmphoraFlows(base.TestCase):
self.amp1 = data_models.Amphora(id=1)
self.amp2 = data_models.Amphora(id=2)
self.amp3 = data_models.Amphora(id=3, status=constants.DELETED)
self.amp4 = data_models.Amphora(id=uuidutils.generate_uuid())
self.lb = data_models.LoadBalancer(
id=4, amphorae=[self.amp1, self.amp2, self.amp3])
@ -62,7 +64,7 @@ class TestAmphoraFlows(base.TestCase):
self.assertIn(constants.AVAILABILITY_ZONE, amp_flow.requires)
self.assertEqual(5, len(amp_flow.provides))
self.assertEqual(3, len(amp_flow.requires))
self.assertEqual(4, len(amp_flow.requires))
def test_get_create_amphora_flow_cert(self, mock_get_net_driver):
self.AmpFlow = amphora_flows.AmphoraFlows()
@ -82,7 +84,7 @@ class TestAmphoraFlows(base.TestCase):
self.assertIn(constants.AVAILABILITY_ZONE, amp_flow.requires)
self.assertEqual(5, len(amp_flow.provides))
self.assertEqual(3, len(amp_flow.requires))
self.assertEqual(4, len(amp_flow.requires))
def test_get_create_amphora_for_lb_flow(self, mock_get_net_driver):
@ -103,7 +105,7 @@ class TestAmphoraFlows(base.TestCase):
self.assertIn(constants.SERVER_PEM, amp_flow.provides)
self.assertEqual(5, len(amp_flow.provides))
self.assertEqual(4, len(amp_flow.requires))
self.assertEqual(5, len(amp_flow.requires))
def test_get_cert_create_amphora_for_lb_flow(self, mock_get_net_driver):
@ -126,7 +128,7 @@ class TestAmphoraFlows(base.TestCase):
self.assertIn(constants.SERVER_PEM, amp_flow.provides)
self.assertEqual(5, len(amp_flow.provides))
self.assertEqual(4, len(amp_flow.requires))
self.assertEqual(5, len(amp_flow.requires))
def test_get_cert_master_create_amphora_for_lb_flow(
self, mock_get_net_driver):
@ -150,7 +152,7 @@ class TestAmphoraFlows(base.TestCase):
self.assertIn(constants.SERVER_PEM, amp_flow.provides)
self.assertEqual(5, len(amp_flow.provides))
self.assertEqual(4, len(amp_flow.requires))
self.assertEqual(5, len(amp_flow.requires))
def test_get_cert_master_rest_anti_affinity_create_amphora_for_lb_flow(
self, mock_get_net_driver):
@ -200,7 +202,7 @@ class TestAmphoraFlows(base.TestCase):
self.assertIn(constants.SERVER_PEM, amp_flow.provides)
self.assertEqual(5, len(amp_flow.provides))
self.assertEqual(4, len(amp_flow.requires))
self.assertEqual(5, len(amp_flow.requires))
def test_get_cert_bogus_create_amphora_for_lb_flow(
self, mock_get_net_driver):
@ -223,7 +225,7 @@ class TestAmphoraFlows(base.TestCase):
self.assertIn(constants.SERVER_PEM, amp_flow.provides)
self.assertEqual(5, len(amp_flow.provides))
self.assertEqual(4, len(amp_flow.requires))
self.assertEqual(5, len(amp_flow.requires))
def test_get_cert_backup_rest_anti_affinity_create_amphora_for_lb_flow(
self, mock_get_net_driver):
@ -253,14 +255,14 @@ class TestAmphoraFlows(base.TestCase):
def test_get_delete_amphora_flow(self, mock_get_net_driver):
amp_flow = self.AmpFlow.get_delete_amphora_flow()
amp_flow = self.AmpFlow.get_delete_amphora_flow(self.amp4.to_dict())
self.assertIsInstance(amp_flow, flow.Flow)
self.assertIn(constants.AMPHORA, amp_flow.requires)
# This flow injects the required data at flow compile time.
self.assertEqual(0, len(amp_flow.provides))
self.assertEqual(1, len(amp_flow.requires))
self.assertEqual(0, len(amp_flow.requires))
def test_allocate_amp_to_lb_decider(self, mock_get_net_driver):
history = mock.MagicMock()
@ -282,102 +284,103 @@ class TestAmphoraFlows(base.TestCase):
result = self.AmpFlow._create_new_amp_for_lb_decider(history)
self.assertFalse(result)
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
@mock.patch('octavia.db.api.get_session', return_value=mock.MagicMock())
def test_get_failover_flow_allocated(self, mock_session, mock_get_lb,
mock_get_net_driver):
mock_get_lb.return_value = self.lb
provider_lb = {constants.LOADBALANCER_ID: '1234'}
amp_flow = self.AmpFlow.get_failover_flow(
load_balancer=provider_lb)
def test_get_failover_flow_act_stdby(self, mock_get_net_driver):
failed_amphora = data_models.Amphora(
id=uuidutils.generate_uuid(), role=constants.ROLE_MASTER,
load_balancer_id=uuidutils.generate_uuid()).to_dict()
amp_flow = self.AmpFlow.get_failover_amphora_flow(
failed_amphora, 2)
self.assertIsInstance(amp_flow, flow.Flow)
self.assertIn(constants.FAILED_AMPHORA, amp_flow.requires)
self.assertIn(constants.AVAILABILITY_ZONE, amp_flow.requires)
self.assertIn(constants.BUILD_TYPE_PRIORITY, amp_flow.requires)
self.assertIn(constants.FLAVOR, amp_flow.requires)
self.assertIn(constants.LOADBALANCER, amp_flow.requires)
self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires)
self.assertIn(constants.VIP, amp_flow.requires)
self.assertIn(constants.AMP_DATA, amp_flow.provides)
self.assertIn(constants.ADDED_PORTS, amp_flow.provides)
self.assertIn(constants.AMP_VRRP_INT, amp_flow.provides)
self.assertIn(constants.AMPHORA, amp_flow.provides)
self.assertIn(constants.AMPHORA_ID, amp_flow.provides)
self.assertIn(constants.AMPHORAE, amp_flow.provides)
self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, amp_flow.provides)
self.assertIn(constants.BASE_PORT, amp_flow.provides)
self.assertIn(constants.COMPUTE_ID, amp_flow.provides)
self.assertIn(constants.COMPUTE_OBJ, amp_flow.provides)
self.assertIn(constants.LISTENERS, amp_flow.provides)
self.assertIn(constants.DELTA, amp_flow.provides)
self.assertIn(constants.LOADBALANCER, amp_flow.provides)
self.assertIn(constants.SERVER_PEM, amp_flow.provides)
self.assertIn(constants.VIP_SG_ID, amp_flow.provides)
self.assertEqual(5, len(amp_flow.requires))
self.assertEqual(12, len(amp_flow.provides))
self.assertEqual(7, len(amp_flow.requires))
self.assertEqual(13, len(amp_flow.provides))
amp_flow = self.AmpFlow.get_failover_flow(
role=constants.ROLE_MASTER, load_balancer=provider_lb)
def test_get_failover_flow_standalone(self, mock_get_net_driver):
failed_amphora = data_models.Amphora(
id=uuidutils.generate_uuid(), role=constants.ROLE_STANDALONE,
load_balancer_id=uuidutils.generate_uuid(),
vrrp_ip='2001:3b8::32').to_dict()
amp_flow = self.AmpFlow.get_failover_amphora_flow(
failed_amphora, 1)
self.assertIsInstance(amp_flow, flow.Flow)
self.assertIn(constants.FAILED_AMPHORA, amp_flow.requires)
self.assertIn(constants.AVAILABILITY_ZONE, amp_flow.requires)
self.assertIn(constants.BUILD_TYPE_PRIORITY, amp_flow.requires)
self.assertIn(constants.FLAVOR, amp_flow.requires)
self.assertIn(constants.LOADBALANCER, amp_flow.requires)
self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires)
self.assertIn(constants.VIP, amp_flow.requires)
self.assertIn(constants.AMP_DATA, amp_flow.provides)
self.assertIn(constants.ADDED_PORTS, amp_flow.provides)
self.assertIn(constants.AMPHORA, amp_flow.provides)
self.assertIn(constants.AMPHORA_ID, amp_flow.provides)
self.assertIn(constants.AMPHORAE, amp_flow.provides)
self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, amp_flow.provides)
self.assertIn(constants.BASE_PORT, amp_flow.provides)
self.assertIn(constants.COMPUTE_ID, amp_flow.provides)
self.assertIn(constants.COMPUTE_OBJ, amp_flow.provides)
self.assertIn(constants.LISTENERS, amp_flow.provides)
self.assertIn(constants.DELTA, amp_flow.provides)
self.assertIn(constants.LOADBALANCER, amp_flow.provides)
self.assertIn(constants.SERVER_PEM, amp_flow.provides)
self.assertIn(constants.VIP_SG_ID, amp_flow.provides)
self.assertEqual(5, len(amp_flow.requires))
self.assertEqual(7, len(amp_flow.requires))
self.assertEqual(12, len(amp_flow.provides))
amp_flow = self.AmpFlow.get_failover_flow(
role=constants.ROLE_BACKUP, load_balancer=provider_lb)
def test_get_failover_flow_bogus_role(self, mock_get_net_driver):
failed_amphora = data_models.Amphora(id=uuidutils.generate_uuid(),
role='bogus').to_dict()
amp_flow = self.AmpFlow.get_failover_amphora_flow(
failed_amphora, 1)
self.assertIsInstance(amp_flow, flow.Flow)
self.assertIn(constants.FAILED_AMPHORA, amp_flow.requires)
self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires)
self.assertIn(constants.AMP_DATA, amp_flow.provides)
self.assertIn(constants.AMPHORA, amp_flow.provides)
self.assertIn(constants.AMPHORA_ID, amp_flow.provides)
self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, amp_flow.provides)
self.assertIn(constants.COMPUTE_ID, amp_flow.provides)
self.assertIn(constants.COMPUTE_OBJ, amp_flow.provides)
self.assertIn(constants.LISTENERS, amp_flow.provides)
self.assertIn(constants.LOADBALANCER, amp_flow.provides)
self.assertIn(constants.VIP_SG_ID, amp_flow.provides)
self.assertEqual(5, len(amp_flow.requires))
self.assertEqual(12, len(amp_flow.provides))
amp_flow = self.AmpFlow.get_failover_flow(
role='BOGUSROLE', load_balancer=provider_lb)
self.assertIsInstance(amp_flow, flow.Flow)
self.assertIn(constants.FAILED_AMPHORA, amp_flow.requires)
self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires)
self.assertIn(constants.AMP_DATA, amp_flow.provides)
self.assertIn(constants.AMPHORA, amp_flow.provides)
self.assertIn(constants.AMPHORA_ID, amp_flow.provides)
self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, amp_flow.provides)
self.assertIn(constants.COMPUTE_ID, amp_flow.provides)
self.assertIn(constants.COMPUTE_OBJ, amp_flow.provides)
self.assertIn(constants.LISTENERS, amp_flow.provides)
self.assertIn(constants.LOADBALANCER, amp_flow.provides)
self.assertEqual(5, len(amp_flow.requires))
self.assertEqual(12, len(amp_flow.provides))
self.assertEqual(1, len(amp_flow.requires))
self.assertEqual(1, len(amp_flow.provides))
def test_get_failover_flow_spare(self, mock_get_net_driver):
amp_flow = self.AmpFlow.get_failover_flow()
amp_flow = self.AmpFlow.get_failover_amphora_flow(
self.amp4.to_dict(), 0)
self.assertIsInstance(amp_flow, flow.Flow)
self.assertIn(constants.FAILED_AMPHORA, amp_flow.requires)
self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires)
self.assertIn(constants.VIP_SG_ID, amp_flow.provides)
self.assertEqual(1, len(amp_flow.requires))
self.assertEqual(0, len(amp_flow.provides))
self.assertEqual(1, len(amp_flow.provides))
def test_cert_rotate_amphora_flow(self, mock_get_net_driver):
self.AmpFlow = amphora_flows.AmphoraFlows()
@ -396,12 +399,30 @@ class TestAmphoraFlows(base.TestCase):
self.assertIsInstance(vrrp_subflow, flow.Flow)
self.assertIn(constants.LOADBALANCER, vrrp_subflow.provides)
self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, vrrp_subflow.provides)
self.assertIn(constants.AMP_VRRP_INT, vrrp_subflow.provides)
self.assertIn(constants.LOADBALANCER, vrrp_subflow.requires)
self.assertIn(constants.LOADBALANCER_ID, vrrp_subflow.requires)
self.assertIn(constants.AMPHORAE, vrrp_subflow.requires)
self.assertEqual(2, len(vrrp_subflow.provides))
self.assertEqual(1, len(vrrp_subflow.requires))
self.assertEqual(2, len(vrrp_subflow.requires))
def test_get_vrrp_subflow_dont_create_vrrp_group(
self, mock_get_net_driver):
vrrp_subflow = self.AmpFlow.get_vrrp_subflow('123',
create_vrrp_group=False)
self.assertIsInstance(vrrp_subflow, flow.Flow)
self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, vrrp_subflow.provides)
self.assertIn(constants.AMP_VRRP_INT, vrrp_subflow.provides)
self.assertIn(constants.LOADBALANCER_ID, vrrp_subflow.requires)
self.assertIn(constants.AMPHORAE, vrrp_subflow.requires)
self.assertEqual(2, len(vrrp_subflow.provides))
self.assertEqual(2, len(vrrp_subflow.requires))
def test_get_post_map_lb_subflow(self, mock_get_net_driver):
@ -414,9 +435,8 @@ class TestAmphoraFlows(base.TestCase):
self.assertIn(constants.FLAVOR, amp_flow.requires)
self.assertIn(constants.AMPHORA, amp_flow.requires)
self.assertIn(constants.AMPHORA, amp_flow.provides)
self.assertEqual(1, len(amp_flow.provides))
self.assertEqual(0, len(amp_flow.provides))
self.assertEqual(2, len(amp_flow.requires))
amp_flow = self.AmpFlow._get_post_map_lb_subflow(
@ -426,9 +446,8 @@ class TestAmphoraFlows(base.TestCase):
self.assertIn(constants.FLAVOR, amp_flow.requires)
self.assertIn(constants.AMPHORA, amp_flow.requires)
self.assertIn(constants.AMPHORA, amp_flow.provides)
self.assertEqual(1, len(amp_flow.provides))
self.assertEqual(0, len(amp_flow.provides))
self.assertEqual(2, len(amp_flow.requires))
amp_flow = self.AmpFlow._get_post_map_lb_subflow(
@ -438,9 +457,8 @@ class TestAmphoraFlows(base.TestCase):
self.assertIn(constants.FLAVOR, amp_flow.requires)
self.assertIn(constants.AMPHORA, amp_flow.requires)
self.assertIn(constants.AMPHORA, amp_flow.provides)
self.assertEqual(1, len(amp_flow.provides))
self.assertEqual(0, len(amp_flow.provides))
self.assertEqual(2, len(amp_flow.requires))
amp_flow = self.AmpFlow._get_post_map_lb_subflow(
@ -450,9 +468,8 @@ class TestAmphoraFlows(base.TestCase):
self.assertIn(constants.FLAVOR, amp_flow.requires)
self.assertIn(constants.AMPHORA, amp_flow.requires)
self.assertIn(constants.AMPHORA, amp_flow.provides)
self.assertEqual(1, len(amp_flow.provides))
self.assertEqual(0, len(amp_flow.provides))
self.assertEqual(2, len(amp_flow.requires))
def test_update_amphora_config_flow(self, mock_get_net_driver):

8
octavia/tests/unit/controller/worker/v2/flows/test_listener_flows.py
View File

@ -14,6 +14,7 @@
#
from unittest import mock
from oslo_utils import uuidutils
from taskflow.patterns import linear_flow as flow
from octavia.common import constants
@ -57,15 +58,16 @@ class TestListenerFlows(base.TestCase):
self.assertEqual(0, len(listener_flow.provides))
def test_get_delete_listener_internal_flow(self, mock_get_net_driver):
fake_listener = {constants.LISTENER_ID: uuidutils.generate_uuid()}
listener_flow = self.ListenerFlow.get_delete_listener_internal_flow(
'test-listener')
fake_listener)
self.assertIsInstance(listener_flow, flow.Flow)
self.assertIn('test-listener', listener_flow.requires)
self.assertIn(constants.LOADBALANCER_ID, listener_flow.requires)
self.assertIn(constants.PROJECT_ID, listener_flow.requires)
self.assertEqual(3, len(listener_flow.requires))
self.assertEqual(2, len(listener_flow.requires))
self.assertEqual(0, len(listener_flow.provides))
def test_get_update_listener_flow(self, mock_get_net_driver):

274
octavia/tests/unit/controller/worker/v2/flows/test_load_balancer_flows.py
View File

@ -16,10 +16,12 @@ from unittest import mock
from oslo_config import cfg
from oslo_config import fixture as oslo_fixture
from oslo_utils import uuidutils
from taskflow.patterns import linear_flow as flow
from octavia.common import constants
from octavia.common import exceptions
from octavia.controller.worker.v2.flows import flow_utils
from octavia.controller.worker.v2.flows import load_balancer_flows
import octavia.tests.unit.base as base
@ -106,34 +108,38 @@ class TestLoadBalancerFlows(base.TestCase):
lb_mock = mock.Mock()
listener_mock = mock.Mock()
listener_mock.id = '123'
listener_dict = {constants.LISTENER_ID: '123'}
listener_mock.to_dict.return_value = {'id': '123'}
lb_mock.listeners = [listener_mock]
lb_mock.id = '321'
lb_mock.project_id = '876'
pool_mock = mock.Mock()
pool_mock.id = '345'
pool_mock.to_dict.return_value = {constants.ID: pool_mock.id}
pool_mock.listeners = None
pool_mock.health_monitor = None
pool_mock.members = None
lb_mock.pools = [pool_mock]
l7_mock = mock.Mock()
l7_mock.id = '678'
listener_mock.l7policies = [l7_mock]
mock_get_lb.return_value = lb_mock
lb_dict = {constants.LOADBALANCER_ID: lb_mock.id}
store = self.LBFlow.get_delete_listeners_store(lb_mock)
store.update(self.LBFlow.get_delete_pools_store(lb_mock))
listeners = flow_utils.get_listeners_on_lb(lb_mock)
pools = flow_utils.get_pools_on_lb(lb_mock)
lb_flow = self.LBFlow.get_cascade_delete_load_balancer_flow(
lb_dict)
lb_dict, listeners, pools)
self.assertIsInstance(lb_flow, flow.Flow)
self.assertEqual({'listener_123': listener_dict,
constants.LOADBALANCER_ID: lb_mock.id,
constants.PROJECT_ID: lb_mock.project_id,
'pool345': pool_mock.id}, store)
self.assertIn(constants.LOADBALANCER, lb_flow.requires)
self.assertIn(constants.LOADBALANCER_ID, lb_flow.requires)
self.assertIn(constants.PROJECT_ID, lb_flow.requires)
self.assertIn(constants.SERVER_GROUP_ID, lb_flow.requires)
self.assertEqual(1, len(lb_flow.provides))
self.assertEqual(6, len(lb_flow.requires))
self.assertEqual(4, len(lb_flow.requires))
def test_get_update_load_balancer_flow(self, mock_get_net_driver):
@ -168,10 +174,14 @@ class TestLoadBalancerFlows(base.TestCase):
self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires)
self.assertIn(constants.UPDATE_DICT, amp_flow.requires)
self.assertIn(constants.AMPHORAE, amp_flow.provides)
self.assertIn(constants.AMP_VRRP_INT, amp_flow.provides)
self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, amp_flow.provides)
self.assertIn(constants.LOADBALANCER, amp_flow.provides)
self.assertEqual(2, len(amp_flow.provides))
self.assertEqual(2, len(amp_flow.requires))
self.assertEqual(2, len(amp_flow.requires), amp_flow.requires)
self.assertEqual(4, len(amp_flow.provides), amp_flow.provides)
# Test mark_active=False
amp_flow = self.LBFlow.get_post_lb_amp_association_flow(
@ -181,10 +191,14 @@ class TestLoadBalancerFlows(base.TestCase):
self.assertIn(constants.LOADBALANCER_ID, amp_flow.requires)
self.assertIn(constants.UPDATE_DICT, amp_flow.requires)
self.assertIn(constants.AMPHORAE, amp_flow.provides)
self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, amp_flow.provides)
self.assertIn(constants.AMP_VRRP_INT, amp_flow.provides)
self.assertIn(constants.LOADBALANCER, amp_flow.provides)
self.assertEqual(2, len(amp_flow.provides))
self.assertEqual(2, len(amp_flow.requires))
self.assertEqual(2, len(amp_flow.requires), amp_flow.requires)
self.assertEqual(4, len(amp_flow.provides), amp_flow.provides)
def test_get_create_load_balancer_flows_single_listeners(
self, mock_get_net_driver):
@ -200,16 +214,18 @@ class TestLoadBalancerFlows(base.TestCase):
self.assertIn(constants.LISTENERS, create_flow.provides)
self.assertIn(constants.AMPHORA, create_flow.provides)
self.assertIn(constants.AMPHORA_ID, create_flow.provides)
self.assertIn(constants.AMPHORA_NETWORK_CONFIG, create_flow.provides)
self.assertIn(constants.AMP_DATA, create_flow.provides)
self.assertIn(constants.COMPUTE_ID, create_flow.provides)
self.assertIn(constants.COMPUTE_OBJ, create_flow.provides)
self.assertIn(constants.LOADBALANCER, create_flow.provides)
self.assertIn(constants.DELTAS, create_flow.provides)
self.assertIn(constants.ADDED_PORTS, create_flow.provides)
self.assertIn(constants.SERVER_PEM, create_flow.provides)
self.assertIn(constants.SUBNET, create_flow.provides)
self.assertIn(constants.VIP, create_flow.provides)
self.assertIn(constants.AMP_DATA, create_flow.provides)
self.assertIn(constants.AMPHORA_NETWORK_CONFIG, create_flow.provides)
self.assertEqual(5, len(create_flow.requires))
self.assertEqual(6, len(create_flow.requires))
self.assertEqual(13, len(create_flow.provides),
create_flow.provides)
@ -221,22 +237,232 @@ class TestLoadBalancerFlows(base.TestCase):
)
)
self.assertIsInstance(create_flow, flow.Flow)
self.assertIn(constants.AVAILABILITY_ZONE, create_flow.requires)
self.assertIn(constants.BUILD_TYPE_PRIORITY, create_flow.requires)
self.assertIn(constants.FLAVOR, create_flow.requires)
self.assertIn(constants.LOADBALANCER_ID, create_flow.requires)
self.assertIn(constants.SERVER_GROUP_ID, create_flow.requires)
self.assertIn(constants.UPDATE_DICT, create_flow.requires)
self.assertIn(constants.LISTENERS, create_flow.provides)
self.assertIn(constants.ADDED_PORTS, create_flow.provides)
self.assertIn(constants.AMP_DATA, create_flow.provides)
self.assertIn(constants.AMP_VRRP_INT, create_flow.provides)
self.assertIn(constants.AMPHORA, create_flow.provides)
self.assertIn(constants.AMPHORAE, create_flow.provides)
self.assertIn(constants.AMPHORA_ID, create_flow.provides)
self.assertIn(constants.AMPHORA_NETWORK_CONFIG, create_flow.provides)
self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, create_flow.provides)
self.assertIn(constants.COMPUTE_ID, create_flow.provides)
self.assertIn(constants.COMPUTE_OBJ, create_flow.provides)
self.assertIn(constants.LOADBALANCER, create_flow.provides)
self.assertIn(constants.DELTAS, create_flow.provides)
self.assertIn(constants.ADDED_PORTS, create_flow.provides)
self.assertIn(constants.LOADBALANCER, create_flow.provides)
self.assertIn(constants.LISTENERS, create_flow.provides)
self.assertIn(constants.SERVER_PEM, create_flow.provides)
self.assertIn(constants.SUBNET, create_flow.provides)
self.assertIn(constants.VIP, create_flow.provides)
self.assertIn(constants.AMP_DATA, create_flow.provides)
self.assertIn(constants.AMPHORAE_NETWORK_CONFIG,
self.assertEqual(6, len(create_flow.requires), create_flow.requires)
self.assertEqual(16, len(create_flow.provides),
create_flow.provides)
self.assertEqual(5, len(create_flow.requires))
self.assertEqual(14, len(create_flow.provides),
create_flow.provides)
def _test_get_failover_LB_flow_single(self, amphorae):
lb_mock = mock.MagicMock()
lb_mock.id = uuidutils.generate_uuid()
lb_mock.topology = constants.TOPOLOGY_SINGLE
failover_flow = self.LBFlow.get_failover_LB_flow(amphorae, lb_mock)
self.assertIsInstance(failover_flow, flow.Flow)
self.assertIn(constants.AVAILABILITY_ZONE, failover_flow.requires)
self.assertIn(constants.BUILD_TYPE_PRIORITY, failover_flow.requires)
self.assertIn(constants.FLAVOR, failover_flow.requires)
self.assertIn(constants.LOADBALANCER, failover_flow.requires)
self.assertIn(constants.LOADBALANCER_ID, failover_flow.requires)
self.assertIn(constants.ADDED_PORTS, failover_flow.provides)
self.assertIn(constants.AMPHORA, failover_flow.provides)
self.assertIn(constants.AMPHORA_ID, failover_flow.provides)
self.assertIn(constants.AMPHORAE_NETWORK_CONFIG,
failover_flow.provides)
self.assertIn(constants.BASE_PORT, failover_flow.provides)
self.assertIn(constants.COMPUTE_ID, failover_flow.provides)
self.assertIn(constants.COMPUTE_OBJ, failover_flow.provides)
self.assertIn(constants.DELTA, failover_flow.provides)
self.assertIn(constants.LOADBALANCER, failover_flow.provides)
self.assertIn(constants.SERVER_PEM, failover_flow.provides)
self.assertIn(constants.VIP, failover_flow.provides)
self.assertIn(constants.VIP_SG_ID, failover_flow.provides)
self.assertEqual(6, len(failover_flow.requires),
failover_flow.requires)
self.assertEqual(12, len(failover_flow.provides),
failover_flow.provides)
def test_get_failover_LB_flow_no_amps_single(self, mock_get_net_driver):
self._test_get_failover_LB_flow_single([])
def test_get_failover_LB_flow_one_amp_single(self, mock_get_net_driver):
amphora_dict = {constants.ID: uuidutils.generate_uuid(),
constants.ROLE: constants.ROLE_STANDALONE,
constants.COMPUTE_ID: uuidutils.generate_uuid(),
constants.VRRP_PORT_ID: None, constants.VRRP_IP: None}
self._test_get_failover_LB_flow_single([amphora_dict])
def test_get_failover_LB_flow_one_spare_amp_single(self,
mock_get_net_driver):
amphora_dict = {constants.ID: uuidutils.generate_uuid(),
constants.ROLE: 'bogus',
constants.COMPUTE_ID: uuidutils.generate_uuid(),
constants.VRRP_PORT_ID: None, constants.VRRP_IP: None}
self._test_get_failover_LB_flow_single([amphora_dict])
def test_get_failover_LB_flow_one_bogus_amp_single(self,
mock_get_net_driver):
amphora_dict = {constants.ID: uuidutils.generate_uuid(),
constants.ROLE: 'bogus',
constants.COMPUTE_ID: uuidutils.generate_uuid(),
constants.VRRP_PORT_ID: None, constants.VRRP_IP: None}
self._test_get_failover_LB_flow_single([amphora_dict])
def test_get_failover_LB_flow_two_amp_single(self, mock_get_net_driver):
amphora_dict = {constants.ID: uuidutils.generate_uuid()}
amphora2_dict = {constants.ID: uuidutils.generate_uuid(),
constants.ROLE: constants.ROLE_STANDALONE,
constants.COMPUTE_ID: uuidutils.generate_uuid(),
constants.VRRP_PORT_ID: None, constants.VRRP_IP: None}
self._test_get_failover_LB_flow_single([amphora_dict, amphora2_dict])
def _test_get_failover_LB_flow_no_amps_act_stdby(self, amphorae):
lb_mock = mock.MagicMock()
lb_mock.id = uuidutils.generate_uuid()
lb_mock.topology = constants.TOPOLOGY_ACTIVE_STANDBY
failover_flow = self.LBFlow.get_failover_LB_flow(amphorae, lb_mock)
self.assertIsInstance(failover_flow, flow.Flow)
self.assertIn(constants.AVAILABILITY_ZONE, failover_flow.requires)
self.assertIn(constants.BUILD_TYPE_PRIORITY, failover_flow.requires)
self.assertIn(constants.FLAVOR, failover_flow.requires)
self.assertIn(constants.LOADBALANCER, failover_flow.requires)
self.assertIn(constants.LOADBALANCER_ID, failover_flow.requires)
self.assertIn(constants.ADDED_PORTS, failover_flow.provides)
self.assertIn(constants.AMPHORA, failover_flow.provides)
self.assertIn(constants.AMPHORA_ID, failover_flow.provides)
self.assertIn(constants.AMPHORAE_NETWORK_CONFIG,
failover_flow.provides)
self.assertIn(constants.BASE_PORT, failover_flow.provides)
self.assertIn(constants.COMPUTE_ID, failover_flow.provides)
self.assertIn(constants.COMPUTE_OBJ, failover_flow.provides)
self.assertIn(constants.DELTA, failover_flow.provides)
self.assertIn(constants.LOADBALANCER, failover_flow.provides)
self.assertIn(constants.SERVER_PEM, failover_flow.provides)
self.assertIn(constants.VIP, failover_flow.provides)
self.assertIn(constants.VIP_SG_ID, failover_flow.provides)
self.assertEqual(6, len(failover_flow.requires),
failover_flow.requires)
self.assertEqual(12, len(failover_flow.provides),
failover_flow.provides)
def test_get_failover_LB_flow_no_amps_act_stdby(self, mock_get_net_driver):
self._test_get_failover_LB_flow_no_amps_act_stdby([])
def test_get_failover_LB_flow_one_amps_act_stdby(self, amphorae):
amphora_dict = {constants.ID: uuidutils.generate_uuid(),
constants.ROLE: constants.ROLE_MASTER,
constants.COMPUTE_ID: uuidutils.generate_uuid(),
constants.VRRP_PORT_ID: None, constants.VRRP_IP: None}
self._test_get_failover_LB_flow_no_amps_act_stdby([amphora_dict])
def test_get_failover_LB_flow_two_amps_act_stdby(self,
mock_get_net_driver):
amphora_dict = {constants.ID: uuidutils.generate_uuid(),
constants.ROLE: constants.ROLE_MASTER,
constants.COMPUTE_ID: uuidutils.generate_uuid(),
constants.VRRP_PORT_ID: uuidutils.generate_uuid(),
constants.VRRP_IP: '192.0.2.46'}
amphora2_dict = {constants.ID: uuidutils.generate_uuid(),
constants.ROLE: constants.ROLE_BACKUP,
constants.COMPUTE_ID: uuidutils.generate_uuid(),
constants.VRRP_PORT_ID: uuidutils.generate_uuid(),
constants.VRRP_IP: '2001:db8::46'}
self._test_get_failover_LB_flow_no_amps_act_stdby([amphora_dict,
amphora2_dict])
def test_get_failover_LB_flow_three_amps_act_stdby(self,
mock_get_net_driver):
amphora_dict = {constants.ID: uuidutils.generate_uuid(),
constants.ROLE: constants.ROLE_MASTER,
constants.COMPUTE_ID: uuidutils.generate_uuid(),
constants.VRRP_PORT_ID: uuidutils.generate_uuid(),
constants.VRRP_IP: '192.0.2.46'}
amphora2_dict = {constants.ID: uuidutils.generate_uuid(),
constants.ROLE: constants.ROLE_BACKUP,
constants.COMPUTE_ID: uuidutils.generate_uuid(),
constants.VRRP_PORT_ID: uuidutils.generate_uuid(),
constants.VRRP_IP: '2001:db8::46'}
amphora3_dict = {constants.ID: uuidutils.generate_uuid(),
constants.ROLE: 'bogus',
constants.COMPUTE_ID: uuidutils.generate_uuid(),
constants.VRRP_PORT_ID: None, constants.VRRP_IP: None}
self._test_get_failover_LB_flow_no_amps_act_stdby(
[amphora_dict, amphora2_dict, amphora3_dict])
def test_get_failover_LB_flow_two_amps_bogus_act_stdby(
self, mock_get_net_driver):
amphora_dict = {constants.ID: uuidutils.generate_uuid(),
constants.ROLE: 'bogus',
constants.COMPUTE_ID: uuidutils.generate_uuid(),
constants.VRRP_PORT_ID: uuidutils.generate_uuid(),
constants.VRRP_IP: '192.0.2.46'}
amphora2_dict = {constants.ID: uuidutils.generate_uuid(),
constants.ROLE: constants.ROLE_MASTER,
constants.COMPUTE_ID: uuidutils.generate_uuid(),
constants.VRRP_PORT_ID: uuidutils.generate_uuid(),
constants.VRRP_IP: '2001:db8::46'}
self._test_get_failover_LB_flow_no_amps_act_stdby([amphora_dict,
amphora2_dict])
def test_get_failover_LB_flow_two_amps_spare_act_stdby(
self, mock_get_net_driver):
amphora_dict = {constants.ID: uuidutils.generate_uuid(),
constants.ROLE: None,
constants.COMPUTE_ID: uuidutils.generate_uuid(),
constants.VRRP_PORT_ID: uuidutils.generate_uuid(),
constants.VRRP_IP: '192.0.2.46'}
amphora2_dict = {constants.ID: uuidutils.generate_uuid(),
constants.ROLE: constants.ROLE_MASTER,
constants.COMPUTE_ID: uuidutils.generate_uuid(),
constants.VRRP_PORT_ID: uuidutils.generate_uuid(),
constants.VRRP_IP: '2001:db8::46'}
self._test_get_failover_LB_flow_no_amps_act_stdby([amphora_dict,
amphora2_dict])
def test_get_failover_LB_flow_two_amps_standalone_act_stdby(
self, mock_get_net_driver):
amphora_dict = {constants.ID: uuidutils.generate_uuid(),
constants.ROLE: constants.ROLE_STANDALONE,
constants.COMPUTE_ID: uuidutils.generate_uuid(),
constants.VRRP_PORT_ID: uuidutils.generate_uuid(),
constants.VRRP_IP: '192.0.2.46'}
amphora2_dict = {constants.ID: uuidutils.generate_uuid(),
constants.ROLE: constants.ROLE_MASTER,
constants.COMPUTE_ID: uuidutils.generate_uuid(),
constants.VRRP_PORT_ID: uuidutils.generate_uuid(),
constants.VRRP_IP: '2001:db8::46'}
self._test_get_failover_LB_flow_no_amps_act_stdby([amphora_dict,
amphora2_dict])

3
octavia/tests/unit/controller/worker/v2/flows/test_pool_flows.py
View File

@ -60,10 +60,9 @@ class TestPoolFlows(base.TestCase):
pool_flow = self.PoolFlow.get_delete_pool_flow_internal('test')
self.assertIsInstance(pool_flow, flow.Flow)
self.assertIn('test', pool_flow.requires)
self.assertIn(constants.PROJECT_ID, pool_flow.requires)
self.assertEqual(2, len(pool_flow.requires))
self.assertEqual(1, len(pool_flow.requires))
self.assertEqual(1, len(pool_flow.provides))
def test_get_update_pool_flow(self):

254
octavia/tests/unit/controller/worker/v2/tasks/test_amphora_driver_tasks.py
View File

@ -41,6 +41,7 @@ FAKE_CONFIG_FILE = 'fake config file'
_db_amphora_mock = mock.MagicMock()
_db_amphora_mock.id = AMP_ID
_db_amphora_mock.status = constants.AMPHORA_ALLOCATED
_db_amphora_mock.vrrp_ip = '198.51.100.65'
_amphora_mock = {
constants.ID: AMP_ID,
constants.STATUS: constants.AMPHORA_ALLOCATED,
@ -81,10 +82,14 @@ class TestAmphoraDriverTasks(base.TestCase):
active_connection_rety_interval=CONN_RETRY_INTERVAL)
conf.config(group="controller_worker",
loadbalancer_topology=constants.TOPOLOGY_SINGLE)
self.timeout_dict = {constants.REQ_CONN_TIMEOUT: 1,
constants.REQ_READ_TIMEOUT: 2,
constants.CONN_MAX_RETRIES: 3,
constants.CONN_RETRY_INTERVAL: 4}
super().setUp()
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
def test_amp_listener_update(self,
def test_amp_listeners_update(self,
mock_lb_get,
mock_driver,
mock_generate_uuid,
@ -95,23 +100,46 @@ class TestAmphoraDriverTasks(base.TestCase):
mock_amphora_repo_get,
mock_amphora_repo_update):
timeout_dict = {constants.REQ_CONN_TIMEOUT: 1,
constants.REQ_READ_TIMEOUT: 2,
constants.CONN_MAX_RETRIES: 3,
constants.CONN_RETRY_INTERVAL: 4}
mock_amphora_repo_get.return_value = _db_amphora_mock
mock_lb_get.return_value = _db_load_balancer_mock
amp_list_update_obj = amphora_driver_tasks.AmpListenersUpdate()
amp_list_update_obj.execute(_LB_mock, 0,
[_amphora_mock], timeout_dict)
amp_list_update_obj.execute(_LB_mock, _amphora_mock, self.timeout_dict)
mock_driver.update_amphora_listeners.assert_called_once_with(
_db_load_balancer_mock, _db_amphora_mock, timeout_dict)
_db_load_balancer_mock, _db_amphora_mock, self.timeout_dict)
mock_driver.update_amphora_listeners.side_effect = Exception('boom')
amp_list_update_obj.execute(_LB_mock, _amphora_mock, self.timeout_dict)
mock_amphora_repo_update.assert_called_once_with(
_session_mock, AMP_ID, status=constants.ERROR)
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
def test_amp_index_listener_update(self,
mock_lb_get,
mock_driver,
mock_generate_uuid,
mock_log,
mock_get_session,
mock_listener_repo_get,
mock_listener_repo_update,
mock_amphora_repo_get,
mock_amphora_repo_update):
mock_amphora_repo_get.return_value = _db_amphora_mock
mock_lb_get.return_value = _db_load_balancer_mock
amp_list_update_obj = amphora_driver_tasks.AmphoraIndexListenerUpdate()
amp_list_update_obj.execute(_LB_mock, 0, [_amphora_mock],
self.timeout_dict)
mock_driver.update_amphora_listeners.assert_called_once_with(
_db_load_balancer_mock, _db_amphora_mock, self.timeout_dict)
mock_driver.update_amphora_listeners.side_effect = Exception('boom')
amp_list_update_obj.execute(_LB_mock, 0,
[_amphora_mock], timeout_dict)
[_amphora_mock], self.timeout_dict)
mock_amphora_repo_update.assert_called_once_with(
_session_mock, AMP_ID, status=constants.ERROR)
@ -155,6 +183,40 @@ class TestAmphoraDriverTasks(base.TestCase):
self.assertEqual(2, repo.ListenerRepository.update.call_count)
self.assertIsNone(amp)
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
@mock.patch('octavia.controller.worker.task_utils.TaskUtils.'
'mark_listener_prov_status_error')
def test_amphora_index_listeners_reload(
self, mock_prov_status_error, mock_lb_repo_get,
mock_driver, mock_generate_uuid, mock_log, mock_get_session,
mock_listener_repo_get, mock_listener_repo_update,
mock_amphora_repo_get, mock_amphora_repo_update):
amphora_mock = mock.MagicMock()
listeners_reload_obj = (
amphora_driver_tasks.AmphoraIndexListenersReload())
mock_lb = mock.MagicMock()
mock_listener = mock.MagicMock()
mock_listener.id = '12345'
mock_amphora_repo_get.return_value = amphora_mock
mock_lb_repo_get.return_value = mock_lb
# Test no listeners
mock_lb.listeners = None
listeners_reload_obj.execute(mock_lb, 0, None)
mock_driver.reload.assert_not_called()
# Test with listeners
mock_driver.start.reset_mock()
mock_lb.listeners = [mock_listener]
listeners_reload_obj.execute(mock_lb, 0, [amphora_mock],
timeout_dict=self.timeout_dict)
mock_driver.reload.assert_called_once_with(mock_lb, amphora_mock,
self.timeout_dict)
# Test revert
mock_lb.listeners = [mock_listener]
listeners_reload_obj.revert(mock_lb)
mock_prov_status_error.assert_called_once_with('12345')
@mock.patch('octavia.controller.worker.task_utils.TaskUtils.'
'mark_listener_prov_status_error')
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
@ -293,6 +355,12 @@ class TestAmphoraDriverTasks(base.TestCase):
status=constants.ERROR)
self.assertIsNone(amp)
# Test revert when this task failed
repo.AmphoraRepository.update.reset_mock()
amp = amphora_finalize_obj.revert(
failure.Failure.from_exception(Exception('boom')), _amphora_mock)
repo.AmphoraRepository.update.assert_not_called()
def test_amphora_post_network_plug(self,
mock_driver,
mock_generate_uuid,
@ -335,6 +403,12 @@ class TestAmphoraDriverTasks(base.TestCase):
self.assertIsNone(amp)
# Test revert when this task failed
repo.AmphoraRepository.update.reset_mock()
amp = amphora_post_network_plug_obj.revert(
failure.Failure.from_exception(Exception('boom')), _amphora_mock)
repo.AmphoraRepository.update.assert_not_called()
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
def test_amphorae_post_network_plug(self, mock_lb_get,
mock_driver,
@ -365,6 +439,14 @@ class TestAmphoraDriverTasks(base.TestCase):
assert_called_once_with(_db_amphora_mock,
network_data_models.Port(**port_mock)))
# Test with no ports to plug
mock_driver.post_network_plug.reset_mock()
_deltas_mock = {'0': [port_mock]}
amphora_post_network_plug_obj.execute(_LB_mock, _deltas_mock)
mock_driver.post_network_plug.assert_not_called()
# Test revert
amp = amphora_post_network_plug_obj.revert(None, _LB_mock,
_deltas_mock)
@ -387,6 +469,13 @@ class TestAmphoraDriverTasks(base.TestCase):
self.assertIsNone(amp)
# Test revert when this task failed
repo.AmphoraRepository.update.reset_mock()
amp = amphora_post_network_plug_obj.revert(
failure.Failure.from_exception(Exception('boom')), _amphora_mock,
None)
repo.AmphoraRepository.update.assert_not_called()
@mock.patch('octavia.db.repositories.LoadBalancerRepository.update')
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
def test_amphora_post_vip_plug(self, mock_lb_get,
@ -446,6 +535,13 @@ class TestAmphoraDriverTasks(base.TestCase):
self.assertIsNone(amp)
# Test revert when this task failed
repo.AmphoraRepository.update.reset_mock()
amp = amphora_post_vip_plug_obj.revert(
failure.Failure.from_exception(Exception('boom')), _amphora_mock,
None)
repo.AmphoraRepository.update.assert_not_called()
@mock.patch('octavia.db.repositories.LoadBalancerRepository.update')
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
def test_amphorae_post_vip_plug(self, mock_lb_get,
@ -495,6 +591,13 @@ class TestAmphoraDriverTasks(base.TestCase):
self.assertIsNone(amp)
# Test revert when this task failed
repo.AmphoraRepository.update.reset_mock()
amp = amphora_post_vip_plug_obj.revert(
failure.Failure.from_exception(Exception('boom')), _amphora_mock,
None)
repo.AmphoraRepository.update.assert_not_called()
def test_amphora_cert_upload(self,
mock_driver,
mock_generate_uuid,
@ -515,9 +618,7 @@ class TestAmphoraDriverTasks(base.TestCase):
mock_driver.upload_cert_amp.assert_called_once_with(
_db_amphora_mock, fer.decrypt(pem_file_mock.encode('utf-8')))
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
def test_amphora_update_vrrp_interface(self,
mock_lb_get,
mock_driver,
mock_generate_uuid,
mock_log,
@ -526,45 +627,57 @@ class TestAmphoraDriverTasks(base.TestCase):
mock_listener_repo_update,
mock_amphora_repo_get,
mock_amphora_repo_update):
_db_load_balancer_mock.amphorae = _amphorae_mock
mock_lb_get.return_value = _db_load_balancer_mock
FAKE_INTERFACE = 'fake0'
mock_amphora_repo_get.return_value = _db_amphora_mock
mock_driver.get_interface_from_ip.side_effect = [FAKE_INTERFACE,
Exception('boom')]
timeout_dict = {constants.CONN_MAX_RETRIES: CONN_MAX_RETRIES,
constants.CONN_RETRY_INTERVAL: CONN_RETRY_INTERVAL}
amphora_update_vrrp_interface_obj = (
amphora_driver_tasks.AmphoraUpdateVRRPInterface())
amphora_update_vrrp_interface_obj.execute(_LB_mock)
amphora_update_vrrp_interface_obj.execute(_amphora_mock, timeout_dict)
mock_driver.get_interface_from_ip.assert_called_once_with(
_db_amphora_mock, _db_amphora_mock.vrrp_ip,
timeout_dict=timeout_dict)
mock_amphora_repo_update.assert_called_once_with(
_session_mock, _db_amphora_mock.id, vrrp_interface=FAKE_INTERFACE)
# Test revert
mock_driver.reset_mock()
_db_load_balancer_mock.amphorae = _amphorae_mock
amphora_update_vrrp_interface_obj.revert("BADRESULT", _LB_mock)
mock_amphora_repo_update.assert_called_with(_session_mock,
_db_amphora_mock.id,
vrrp_interface=None)
mock_driver.reset_mock()
# Test with an exception
mock_amphora_repo_update.reset_mock()
_db_load_balancer_mock.amphorae = _amphorae_mock
amphora_update_vrrp_interface_obj.execute(_amphora_mock, timeout_dict)
mock_amphora_repo_update.assert_called_once_with(
_session_mock, _db_amphora_mock.id, status=constants.ERROR)
failure_obj = failure.Failure.from_exception(Exception("TESTEXCEPT"))
amphora_update_vrrp_interface_obj.revert(failure_obj, _LB_mock)
self.assertFalse(mock_amphora_repo_update.called)
def test_amphora_index_update_vrrp_interface(
self, mock_driver, mock_generate_uuid, mock_log, mock_get_session,
mock_listener_repo_get, mock_listener_repo_update,
mock_amphora_repo_get, mock_amphora_repo_update):
mock_amphora_repo_get.return_value = _db_amphora_mock
FAKE_INTERFACE = 'fake0'
mock_driver.get_interface_from_ip.side_effect = [FAKE_INTERFACE,
Exception('boom')]
# Test revert with exception
mock_driver.reset_mock()
timeout_dict = {constants.CONN_MAX_RETRIES: CONN_MAX_RETRIES,
constants.CONN_RETRY_INTERVAL: CONN_RETRY_INTERVAL}
amphora_update_vrrp_interface_obj = (
amphora_driver_tasks.AmphoraIndexUpdateVRRPInterface())
amphora_update_vrrp_interface_obj.execute(
0, [_amphora_mock], timeout_dict)
mock_driver.get_interface_from_ip.assert_called_once_with(
_db_amphora_mock, _db_amphora_mock.vrrp_ip,
timeout_dict=timeout_dict)
mock_amphora_repo_update.assert_called_once_with(
_session_mock, _db_amphora_mock.id, vrrp_interface=FAKE_INTERFACE)
# Test with an exception
mock_amphora_repo_update.reset_mock()
mock_amphora_repo_update.side_effect = Exception('fail')
amphora_update_vrrp_interface_obj.revert("BADRESULT", _LB_mock)
mock_amphora_repo_update.assert_called_with(_session_mock,
_db_amphora_mock.id,
vrrp_interface=None)
amphora_update_vrrp_interface_obj.execute(
0, [_amphora_mock], timeout_dict)
mock_amphora_repo_update.assert_called_once_with(
_session_mock, _db_amphora_mock.id, status=constants.ERROR)
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
def test_amphora_vrrp_update(self,
@ -578,15 +691,27 @@ class TestAmphoraDriverTasks(base.TestCase):
mock_amphora_repo_get,
mock_amphora_repo_update):
amphorae_network_config = mock.MagicMock()
mock_driver.update_vrrp_conf.side_effect = [mock.DEFAULT,
Exception('boom')]
mock_lb_get.return_value = _db_load_balancer_mock
mock_amphora_repo_get.return_value = _db_amphora_mock
amphora_vrrp_update_obj = (
amphora_driver_tasks.AmphoraVRRPUpdate())
amphora_vrrp_update_obj.execute(_LB_mock, amphorae_network_config)
amphora_vrrp_update_obj.execute(LB_ID, amphorae_network_config,
_amphora_mock, 'fakeint0')
mock_driver.update_vrrp_conf.assert_called_once_with(
_db_load_balancer_mock, amphorae_network_config)
_db_load_balancer_mock, amphorae_network_config,
_db_amphora_mock, None)
# Test with an exception
mock_amphora_repo_update.reset_mock()
amphora_vrrp_update_obj.execute(LB_ID, amphorae_network_config,
_amphora_mock, 'fakeint0')
mock_amphora_repo_update.assert_called_once_with(
_session_mock, _db_amphora_mock.id, status=constants.ERROR)
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
def test_amphora_vrrp_stop(self,
def test_amphora_index_vrrp_update(self,
mock_lb_get,
mock_driver,
mock_generate_uuid,
@ -596,16 +721,29 @@ class TestAmphoraDriverTasks(base.TestCase):
mock_listener_repo_update,
mock_amphora_repo_get,
mock_amphora_repo_update):
amphora_vrrp_stop_obj = (
amphora_driver_tasks.AmphoraVRRPStop())
amphorae_network_config = mock.MagicMock()
mock_driver.update_vrrp_conf.side_effect = [mock.DEFAULT,
Exception('boom')]
mock_lb_get.return_value = _db_load_balancer_mock
amphora_vrrp_stop_obj.execute(_LB_mock)
mock_driver.stop_vrrp_service.assert_called_once_with(
_db_load_balancer_mock)
mock_amphora_repo_get.return_value = _db_amphora_mock
amphora_vrrp_update_obj = (
amphora_driver_tasks.AmphoraIndexVRRPUpdate())
amphora_vrrp_update_obj.execute(LB_ID, amphorae_network_config,
0, [_amphora_mock], 'fakeint0',
timeout_dict=self.timeout_dict)
mock_driver.update_vrrp_conf.assert_called_once_with(
_db_load_balancer_mock, amphorae_network_config, _db_amphora_mock,
self.timeout_dict)
# Test with an exception
mock_amphora_repo_update.reset_mock()
amphora_vrrp_update_obj.execute(LB_ID, amphorae_network_config,
0, [_amphora_mock], 'fakeint0')
mock_amphora_repo_update.assert_called_once_with(
_session_mock, _db_amphora_mock.id, status=constants.ERROR)
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
def test_amphora_vrrp_start(self,
mock_lb_get,
mock_driver,
mock_generate_uuid,
mock_log,
@ -614,12 +752,30 @@ class TestAmphoraDriverTasks(base.TestCase):
mock_listener_repo_update,
mock_amphora_repo_get,
mock_amphora_repo_update):
mock_amphora_repo_get.return_value = _db_amphora_mock
amphora_vrrp_start_obj = (
amphora_driver_tasks.AmphoraVRRPStart())
mock_lb_get.return_value = _db_load_balancer_mock
amphora_vrrp_start_obj.execute(_LB_mock)
amphora_vrrp_start_obj.execute(_amphora_mock,
timeout_dict=self.timeout_dict)
mock_driver.start_vrrp_service.assert_called_once_with(
_db_load_balancer_mock)
_db_amphora_mock, self.timeout_dict)
def test_amphora_index_vrrp_start(self,
mock_driver,
mock_generate_uuid,
mock_log,
mock_get_session,
mock_listener_repo_get,
mock_listener_repo_update,
mock_amphora_repo_get,
mock_amphora_repo_update):
mock_amphora_repo_get.return_value = _db_amphora_mock
amphora_vrrp_start_obj = (
amphora_driver_tasks.AmphoraIndexVRRPStart())
amphora_vrrp_start_obj.execute(0, [_amphora_mock],
timeout_dict=self.timeout_dict)
mock_driver.start_vrrp_service.assert_called_once_with(
_db_amphora_mock, self.timeout_dict)
def test_amphora_compute_connectivity_wait(self,
mock_driver,

67
octavia/tests/unit/controller/worker/v2/tasks/test_compute_tasks.py
View File

@ -18,6 +18,7 @@ from cryptography import fernet
from oslo_config import cfg
from oslo_config import fixture as oslo_fixture
from oslo_utils import uuidutils
import tenacity
from octavia.common import constants
from octavia.common import exceptions
@ -178,7 +179,8 @@ class TestComputeTasks(base.TestCase):
mock_driver.build.return_value = COMPUTE_ID
# Test execute()
compute_id = createcompute.execute(_db_amphora_mock.id, ports=[_port])
compute_id = createcompute.execute(_db_amphora_mock.id, ports=[_port],
server_group_id=None)
# Validate that the build method was called properly
mock_driver.build.assert_called_once_with(
@ -508,20 +510,54 @@ class TestComputeTasks(base.TestCase):
def test_delete_amphorae_on_load_balancer(self, mock_lb_get, mock_session,
mock_driver):
mock_driver.delete.side_effect = [mock.DEFAULT,
exceptions.OctaviaException('boom')]
delete_amps = compute_tasks.DeleteAmphoraeOnLoadBalancer()
mock_lb_get.return_value = _db_load_balancer_mock
delete_amps.execute(_load_balancer_mock)
mock_driver.delete.assert_called_once_with(COMPUTE_ID)
# Test compute driver exception is raised
self.assertRaises(exceptions.OctaviaException, delete_amps.execute,
_load_balancer_mock)
@mock.patch('stevedore.driver.DriverManager.driver')
def test_compute_delete(self, mock_driver):
mock_driver.delete.side_effect = [
mock.DEFAULT, exceptions.OctaviaException('boom'),
mock.DEFAULT, exceptions.OctaviaException('boom'),
exceptions.OctaviaException('boom'),
exceptions.OctaviaException('boom'),
exceptions.OctaviaException('boom')]
delete_compute = compute_tasks.ComputeDelete()
# Limit the retry attempts for the test run to save time
delete_compute.execute.retry.stop = tenacity.stop_after_attempt(2)
delete_compute.execute(_amphora_mock)
mock_driver.delete.assert_called_once_with(COMPUTE_ID)
# Test retry after a compute exception
mock_driver.reset_mock()
delete_compute.execute(_amphora_mock)
mock_driver.delete.assert_has_calls([mock.call(COMPUTE_ID),
mock.call(COMPUTE_ID)])
# Test passive failure
mock_driver.reset_mock()
delete_compute.execute(_amphora_mock, passive_failure=True)
mock_driver.delete.assert_has_calls([mock.call(COMPUTE_ID),
mock.call(COMPUTE_ID)])
# Test non-passive failure
mock_driver.reset_mock()
self.assertRaises(exceptions.OctaviaException, delete_compute.execute,
_amphora_mock, passive_failure=False)
@mock.patch('stevedore.driver.DriverManager.driver')
def test_nova_server_group_create(self, mock_driver):
nova_sever_group_obj = compute_tasks.NovaServerGroupCreate()
@ -567,3 +603,32 @@ class TestComputeTasks(base.TestCase):
sg_id = None
nova_sever_group_obj.execute(sg_id)
self.assertFalse(mock_driver.delete_server_group.called, sg_id)
@mock.patch('stevedore.driver.DriverManager.driver')
def test_attach_port(self, mock_driver):
COMPUTE_ID = uuidutils.generate_uuid()
PORT_ID = uuidutils.generate_uuid()
amphora_dict = {constants.COMPUTE_ID: COMPUTE_ID}
port_dict = {constants.ID: PORT_ID}
attach_port_obj = compute_tasks.AttachPort()
# Test execute
attach_port_obj.execute(amphora_dict, port_dict)
mock_driver.attach_network_or_port.assert_called_once_with(
COMPUTE_ID, port_id=PORT_ID)
# Test revert
mock_driver.reset_mock()
attach_port_obj.revert(amphora_dict, port_dict)
mock_driver.detach_port.assert_called_once_with(COMPUTE_ID, PORT_ID)
# Test rever exception
mock_driver.reset_mock()
mock_driver.detach_port.side_effect = [Exception('boom')]
# should not raise
attach_port_obj.revert(amphora_dict, port_dict)

37
octavia/tests/unit/controller/worker/v2/tasks/test_database_tasks.py
View File

@ -46,10 +46,10 @@ HA_PORT_ID = uuidutils.generate_uuid()
L7POLICY_ID = uuidutils.generate_uuid()
L7RULE_ID = uuidutils.generate_uuid()
VIP_IP = '192.0.5.2'
VRRP_ID = 1
VRRP_IP = '192.0.5.3'
HA_IP = '192.0.5.4'
AMP_ROLE = 'FAKE_ROLE'
VRRP_ID = random.randrange(255)
VRRP_PRIORITY = random.randrange(100)
CACHED_ZONE = 'zone1'
IMAGE_ID = uuidutils.generate_uuid()
@ -542,8 +542,15 @@ class TestDatabaseTasks(base.TestCase):
mock_amphora_repo_update,
mock_amphora_repo_delete):
amphora_dict = {constants.ID: AMP_ID}
vip_dict = {constants.IP_ADDRESS: HA_IP,
constants.PORT_ID: HA_PORT_ID}
fixed_ips = [{constants.IP_ADDRESS: VRRP_IP}]
base_port_dict = {constants.ID: VRRP_PORT_ID,
constants.FIXED_IPS: fixed_ips}
update_amp_fo_details = database_tasks.UpdateAmpFailoverDetails()
update_amp_fo_details.execute(self.amphora, self.amphora)
update_amp_fo_details.execute(amphora_dict, vip_dict, base_port_dict)
mock_amphora_repo_update.assert_called_once_with(
'TEST',
@ -1337,16 +1344,16 @@ class TestDatabaseTasks(base.TestCase):
mock_amphora_repo_update,
mock_amphora_repo_delete):
unused_pool = data_models.Pool(id='unused_pool')
members1 = [{constants.MEMBER_ID: 'member1'},
{constants.MEMBER_ID: 'member2'}]
members1 = [data_models.Member(id='member1'),
data_models.Member(id='member2')]
health_monitor = data_models.HealthMonitor(id='hm1')
default_pool = data_models.Pool(id='default_pool',
members=members1,
health_monitor=health_monitor)
listener1 = data_models.Listener(id='listener1',
default_pool=default_pool)
members2 = [{constants.MEMBER_ID: 'member3'},
{constants.MEMBER_ID: 'member4'}]
members2 = [data_models.Member(id='member3'),
data_models.Member(id='member4')]
redirect_pool = data_models.Pool(id='redirect_pool',
members=members2)
l7rules = [data_models.L7Rule(id='rule1')]
@ -1954,6 +1961,22 @@ class TestDatabaseTasks(base.TestCase):
result = get_vip_from_lb_obj.execute(self.loadbalancer_mock)
self.assertEqual(_vip_mock.to_dict(), result)
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
def test_get_loadbalancer(self, mock_lb_get, mock_generate_uuid, mock_LOG,
mock_get_session, mock_loadbalancer_repo_update,
mock_listener_repo_update,
mock_amphora_repo_update,
mock_amphora_repo_delete):
LB_ID = uuidutils.generate_uuid()
get_loadbalancer_obj = database_tasks.GetLoadBalancer()
mock_lb_get.return_value = _db_loadbalancer_mock
result = get_loadbalancer_obj.execute(LB_ID)
self.assertEqual(self.loadbalancer_mock, result)
mock_lb_get.assert_called_once_with('TEST', id=LB_ID)
@mock.patch('octavia.db.repositories.VRRPGroupRepository.create')
def test_create_vrrp_group_for_lb(self,
mock_vrrp_group_create,
@ -1968,7 +1991,7 @@ class TestDatabaseTasks(base.TestCase):
mock_get_session.side_effect = ['TEST',
odb_exceptions.DBDuplicateEntry]
create_vrrp_group = database_tasks.CreateVRRPGroupForLB()
create_vrrp_group.execute(self.loadbalancer_mock)
create_vrrp_group.execute(LB_ID)
mock_vrrp_group_create.assert_called_once_with(
'TEST', load_balancer_id=LB_ID,
vrrp_group_name=LB_ID.replace('-', ''),

436
octavia/tests/unit/controller/worker/v2/tasks/test_network_tasks.py
View File

@ -18,10 +18,12 @@ from oslo_config import cfg
from oslo_config import fixture as oslo_fixture
from oslo_utils import uuidutils
from taskflow.types import failure
import tenacity
from octavia.api.drivers import utils as provider_utils
from octavia.common import constants
from octavia.common import data_models as o_data_models
from octavia.common import exceptions
from octavia.controller.worker.v2.tasks import network_tasks
from octavia.network import base as net_base
from octavia.network import data_models
@ -81,6 +83,11 @@ class TestNetworkTasks(base.TestCase):
self.db_amphora_mock.id = AMPHORA_ID
self.db_amphora_mock.compute_id = COMPUTE_ID
self.db_amphora_mock.status = constants.AMPHORA_ALLOCATED
self.boot_net_id = NETWORK_ID
conf = self.useFixture(oslo_fixture.Config(cfg.CONF))
conf.config(group="controller_worker",
amp_boot_network_list=[self.boot_net_id])
conf.config(group="networking", max_retries=1)
self.amphora_mock = {constants.ID: AMPHORA_ID,
constants.COMPUTE_ID: COMPUTE_ID,
constants.LB_NETWORK_IP: IP_ADDRESS,
@ -94,10 +101,85 @@ class TestNetworkTasks(base.TestCase):
}
conf = oslo_fixture.Config(cfg.CONF)
conf.config(group="controller_worker", amp_boot_network_list=['netid'])
conf.config(group="controller_worker",
amp_boot_network_list=[self.boot_net_id])
super().setUp()
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
@mock.patch('octavia.db.api.get_session', return_value=_session_mock)
def test_calculate_amphora_delta(self, mock_get_session, mock_lb_repo_get,
mock_get_net_driver):
LB_ID = uuidutils.generate_uuid()
DELETE_NETWORK_ID = uuidutils.generate_uuid()
MEMBER_NETWORK_ID = uuidutils.generate_uuid()
MEMBER_SUBNET_ID = uuidutils.generate_uuid()
VRRP_PORT_ID = uuidutils.generate_uuid()
mock_driver = mock.MagicMock()
mock_get_net_driver.return_value = mock_driver
member_mock = mock.MagicMock()
member_mock.subnet_id = MEMBER_SUBNET_ID
pool_mock = mock.MagicMock()
pool_mock.members = [member_mock]
lb_mock = mock.MagicMock()
lb_mock.pools = [pool_mock]
lb_dict = {constants.LOADBALANCER_ID: LB_ID}
amphora_dict = {constants.ID: AMPHORA_ID,
constants.COMPUTE_ID: COMPUTE_ID,
constants.VRRP_PORT_ID: VRRP_PORT_ID}
vrrp_port_mock = mock.MagicMock()
vrrp_port_mock.network_id = self.boot_net_id
vrrp_port_dict = {constants.NETWORK_ID: self.boot_net_id}
mock_subnet = mock.MagicMock()
mock_subnet.network_id = MEMBER_NETWORK_ID
nic1_delete_mock = mock.MagicMock()
nic1_delete_mock.network_id = DELETE_NETWORK_ID
nic2_keep_mock = mock.MagicMock()
nic2_keep_mock.network_id = self.boot_net_id
mock_lb_repo_get.return_value = lb_mock
mock_driver.get_port.return_value = vrrp_port_mock
mock_driver.get_subnet.return_value = mock_subnet
mock_driver.get_plugged_networks.return_value = [nic1_delete_mock,
nic2_keep_mock]
calc_amp_delta = network_tasks.CalculateAmphoraDelta()
# Test vrrp_port_id is None
result = calc_amp_delta.execute(lb_dict, amphora_dict, {})
self.assertEqual(AMPHORA_ID, result[constants.AMPHORA_ID])
self.assertEqual(COMPUTE_ID, result[constants.COMPUTE_ID])
self.assertEqual(1, len(result[constants.ADD_NICS]))
self.assertEqual(MEMBER_NETWORK_ID,
result[constants.ADD_NICS][0][constants.NETWORK_ID])
self.assertEqual(1, len(result[constants.DELETE_NICS]))
self.assertEqual(
DELETE_NETWORK_ID,
result[constants.DELETE_NICS][0][constants.NETWORK_ID])
mock_driver.get_port.assert_called_once_with(VRRP_PORT_ID)
mock_driver.get_subnet.assert_called_once_with(MEMBER_SUBNET_ID)
mock_driver.get_plugged_networks.assert_called_once_with(COMPUTE_ID)
# Test with vrrp_port_id
mock_driver.reset_mock()
result = calc_amp_delta.execute(lb_dict, amphora_dict, {},
vrrp_port=vrrp_port_dict)
self.assertEqual(AMPHORA_ID, result[constants.AMPHORA_ID])
self.assertEqual(COMPUTE_ID, result[constants.COMPUTE_ID])
self.assertEqual(1, len(result[constants.ADD_NICS]))
self.assertEqual(MEMBER_NETWORK_ID,
result[constants.ADD_NICS][0][constants.NETWORK_ID])
self.assertEqual(1, len(result[constants.DELETE_NICS]))
self.assertEqual(
DELETE_NETWORK_ID,
result[constants.DELETE_NICS][0][constants.NETWORK_ID])
mock_driver.get_port.assert_not_called()
mock_driver.get_subnet.assert_called_once_with(MEMBER_SUBNET_ID)
mock_driver.get_plugged_networks.assert_called_once_with(COMPUTE_ID)
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
@mock.patch('octavia.db.api.get_session', return_value=_session_mock)
def test_calculate_delta(self, mock_get_session, mock_get_lb,
@ -110,9 +192,9 @@ class TestNetworkTasks(base.TestCase):
constants.VRRP_PORT_ID: PORT_ID}
mock_get_net_driver.return_value = mock_driver
mock_driver.get_plugged_networks.return_value = [
data_models.Interface(network_id='netid')]
data_models.Interface(network_id=self.boot_net_id)]
mock_driver.get_port.return_value = data_models.Port(
network_id='netid')
network_id=self.boot_net_id)
EMPTY = {}
empty_deltas = {self.db_amphora_mock.id: data_models.Delta(
amphora_id=AMPHORA_ID,
@ -179,7 +261,7 @@ class TestNetworkTasks(base.TestCase):
pool_mock.members = [member_mock]
mock_driver.get_plugged_networks.return_value = [
data_models.Interface(network_id=3),
data_models.Interface(network_id='netid')]
data_models.Interface(network_id=self.boot_net_id)]
self.assertEqual(empty_deltas,
calc_delta.execute(self.load_balancer_mock, {}))
@ -192,7 +274,7 @@ class TestNetworkTasks(base.TestCase):
pool_mock.members = [member_mock]
mock_driver.get_plugged_networks.return_value = [
data_models.Interface(network_id=2),
data_models.Interface(network_id='netid')]
data_models.Interface(network_id=self.boot_net_id)]
ndm = data_models.Delta(amphora_id=self.db_amphora_mock.id,
compute_id=self.db_amphora_mock.compute_id,
@ -210,7 +292,7 @@ class TestNetworkTasks(base.TestCase):
pool_mock.members = []
mock_driver.get_plugged_networks.return_value = [
data_models.Interface(network_id=2),
data_models.Interface(network_id='netid')
data_models.Interface(network_id=self.boot_net_id)
]
ndm = data_models.Delta(amphora_id=self.db_amphora_mock.id,
@ -648,6 +730,7 @@ class TestNetworkTasks(base.TestCase):
t_constants.MOCK_QOS_POLICY_ID1, mock.ANY)
self.assertEqual(2, mock_driver.apply_qos_on_port.call_count)
mock_get_lb.return_value = null_qos_lb
mock_driver.reset_mock()
update_dict = {}
net.execute(null_qos_lb_dict, update_dict=update_dict)
@ -685,7 +768,7 @@ class TestNetworkTasks(base.TestCase):
net.revert(None, pr_tm_dict, update_dict=update_dict)
mock_driver.apply_qos_on_port.assert_called_with(
t_constants.MOCK_QOS_POLICY_ID2, mock.ANY)
self.assertEqual(2, mock_driver.apply_qos_on_port.call_count)
self.assertEqual(1, mock_driver.apply_qos_on_port.call_count)
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
@mock.patch('octavia.db.api.get_session', return_value=_session_mock)
@ -770,6 +853,28 @@ class TestNetworkTasks(base.TestCase):
net_task.execute(listener)
mock_driver.update_vip.assert_called_once_with(lb, for_delete=True)
@mock.patch('octavia.db.api.get_session', return_value='TEST')
@mock.patch('octavia.db.repositories.AmphoraRepository.get')
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
def test_get_amphora_network_configs_by_id(
self, mock_lb_get, mock_amp_get,
mock_get_session, mock_get_net_driver):
LB_ID = uuidutils.generate_uuid()
AMP_ID = uuidutils.generate_uuid()
mock_driver = mock.MagicMock()
mock_get_net_driver.return_value = mock_driver
mock_amp_get.return_value = 'mock amphora'
mock_lb_get.return_value = 'mock load balancer'
net_task = network_tasks.GetAmphoraNetworkConfigsByID()
net_task.execute(LB_ID, AMP_ID)
mock_driver.get_network_configs.assert_called_once_with(
'mock load balancer', amphora='mock amphora')
mock_amp_get.assert_called_once_with('TEST', id=AMP_ID)
mock_lb_get.assert_called_once_with('TEST', id=LB_ID)
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
@mock.patch('octavia.db.api.get_session', return_value=_session_mock)
def test_get_amphorae_network_configs(self, mock_session, mock_lb_get,
@ -854,49 +959,6 @@ class TestNetworkTasks(base.TestCase):
mock_driver.plug_port.assert_any_call(self.db_amphora_mock, port1)
mock_driver.plug_port.assert_any_call(self.db_amphora_mock, port2)
@mock.patch('octavia.db.repositories.AmphoraRepository.get')
@mock.patch('octavia.db.api.get_session', return_value=mock.MagicMock())
def test_plug_vip_port(self, mock_session, mock_get, mock_get_net_driver):
mock_driver = mock.MagicMock()
mock_get.return_value = self.db_amphora_mock
mock_get_net_driver.return_value = mock_driver
vrrp_port = mock.MagicMock()
amphorae_network_config = mock.MagicMock()
mock_driver.get_port.return_value = vrrp_port
plugvipport = network_tasks.PlugVIPPort()
amp = {constants.ID: AMPHORA_ID,
constants.COMPUTE_ID: '1234'}
plugvipport.execute(amp, amphorae_network_config)
mock_driver.plug_port.assert_called_once_with(self.db_amphora_mock,
vrrp_port)
dict_amp_config = {
AMPHORA_ID: {constants.VRRP_PORT: {constants.ID: 5555}}
}
# test revert
plugvipport.revert(None, amp, dict_amp_config)
mock_driver.unplug_port.assert_called_with(self.db_amphora_mock,
vrrp_port)
@mock.patch('octavia.db.repositories.AmphoraRepository.get')
@mock.patch('octavia.db.api.get_session', return_value=mock.MagicMock())
def test_wait_for_port_detach(self, mock_session, mock_get,
mock_get_net_driver):
mock_driver = mock.MagicMock()
mock_get.return_value = self.db_amphora_mock
mock_get_net_driver.return_value = mock_driver
amphora = {constants.ID: AMPHORA_ID,
constants.LB_NETWORK_IP: IP_ADDRESS}
waitforportdetach = network_tasks.WaitForPortDetach()
waitforportdetach.execute(amphora)
mock_driver.wait_for_port_detach.assert_called_once_with(
self.db_amphora_mock)
@mock.patch('octavia.db.repositories.LoadBalancerRepository.get')
@mock.patch('octavia.db.api.get_session', return_value=_session_mock)
def test_update_vip_sg(self, mock_session, mock_lb_get,
@ -928,7 +990,7 @@ class TestNetworkTasks(base.TestCase):
mock_lb_get.return_value = LB
mock_get.return_value = self.db_amphora_mock
mock_get_net_driver.return_value = mock_driver
net = network_tasks.PlugVIPAmpphora()
net = network_tasks.PlugVIPAmphora()
mockSubnet = mock_driver.get_subnet()
net.execute(self.load_balancer_mock, amphora, mockSubnet)
mock_driver.plug_aap_port.assert_called_once_with(
@ -943,7 +1005,7 @@ class TestNetworkTasks(base.TestCase):
mock_lb_get.return_value = LB
mock_get.return_value = self.db_amphora_mock
mock_get_net_driver.return_value = mock_driver
net = network_tasks.PlugVIPAmpphora()
net = network_tasks.PlugVIPAmphora()
mockSubnet = mock.MagicMock()
amphora = {constants.ID: AMPHORA_ID,
constants.LB_NETWORK_IP: IP_ADDRESS}
@ -951,3 +1013,273 @@ class TestNetworkTasks(base.TestCase):
amphora, mockSubnet)
mock_driver.unplug_aap_port.assert_called_once_with(
LB.vip, self.db_amphora_mock, mockSubnet)
@mock.patch('octavia.controller.worker.v2.tasks.network_tasks.DeletePort.'
'update_progress')
def test_delete_port(self, mock_update_progress, mock_get_net_driver):
PORT_ID = uuidutils.generate_uuid()
mock_driver = mock.MagicMock()
mock_get_net_driver.return_value = mock_driver
mock_driver.delete_port.side_effect = [
mock.DEFAULT, exceptions.OctaviaException('boom'), mock.DEFAULT,
exceptions.OctaviaException('boom'),
exceptions.OctaviaException('boom'),
exceptions.OctaviaException('boom'),
exceptions.OctaviaException('boom'),
exceptions.OctaviaException('boom'),
exceptions.OctaviaException('boom')]
mock_driver.admin_down_port.side_effect = [
mock.DEFAULT, exceptions.OctaviaException('boom')]
net_task = network_tasks.DeletePort()
# Limit the retry attempts for the test run to save time
net_task.execute.retry.stop = tenacity.stop_after_attempt(2)
# Test port ID is None (no-op)
net_task.execute(None)
mock_update_progress.assert_not_called()
mock_driver.delete_port.assert_not_called()
# Test successful delete
mock_update_progress.reset_mock()
mock_driver.reset_mock()
net_task.execute(PORT_ID)
mock_update_progress.assert_called_once_with(0.5)
mock_driver.delete_port.assert_called_once_with(PORT_ID)
# Test exception and successful retry
mock_update_progress.reset_mock()
mock_driver.reset_mock()
net_task.execute(PORT_ID)
mock_update_progress.assert_has_calls([mock.call(0.5), mock.call(1.0)])
mock_driver.delete_port.assert_has_calls([mock.call(PORT_ID),
mock.call(PORT_ID)])
# Test passive failure
mock_update_progress.reset_mock()
mock_driver.reset_mock()
net_task.execute(PORT_ID, passive_failure=True)
mock_update_progress.assert_has_calls([mock.call(0.5), mock.call(1.0)])
mock_driver.delete_port.assert_has_calls([mock.call(PORT_ID),
mock.call(PORT_ID)])
mock_driver.admin_down_port.assert_called_once_with(PORT_ID)
# Test passive failure admin down failure
mock_update_progress.reset_mock()
mock_driver.reset_mock()
mock_driver.admin_down_port.reset_mock()
net_task.execute(PORT_ID, passive_failure=True)
mock_update_progress.assert_has_calls([mock.call(0.5), mock.call(1.0)])
mock_driver.delete_port.assert_has_calls([mock.call(PORT_ID),
mock.call(PORT_ID)])
mock_driver.admin_down_port.assert_called_once_with(PORT_ID)
# Test non-passive failure
mock_update_progress.reset_mock()
mock_driver.reset_mock()
mock_driver.admin_down_port.reset_mock()
mock_driver.admin_down_port.side_effect = [
exceptions.OctaviaException('boom')]
self.assertRaises(exceptions.OctaviaException, net_task.execute,
PORT_ID)
mock_update_progress.assert_has_calls([mock.call(0.5), mock.call(1.0)])
mock_driver.delete_port.assert_has_calls([mock.call(PORT_ID),
mock.call(PORT_ID)])
mock_driver.admin_down_port.assert_not_called()
def test_create_vip_base_port(self, mock_get_net_driver):
AMP_ID = uuidutils.generate_uuid()
PORT_ID = uuidutils.generate_uuid()
VIP_NETWORK_ID = uuidutils.generate_uuid()
VIP_QOS_ID = uuidutils.generate_uuid()
VIP_SG_ID = uuidutils.generate_uuid()
VIP_SUBNET_ID = uuidutils.generate_uuid()
VIP_IP_ADDRESS = '203.0.113.81'
mock_driver = mock.MagicMock()
mock_get_net_driver.return_value = mock_driver
vip_dict = {constants.IP_ADDRESS: VIP_IP_ADDRESS,
constants.NETWORK_ID: VIP_NETWORK_ID,
constants.QOS_POLICY_ID: VIP_QOS_ID,
constants.SUBNET_ID: VIP_SUBNET_ID}
port_mock = mock.MagicMock()
port_mock.id = PORT_ID
mock_driver.create_port.side_effect = [
port_mock, exceptions.OctaviaException('boom'),
exceptions.OctaviaException('boom'),
exceptions.OctaviaException('boom')]
mock_driver.delete_port.side_effect = [mock.DEFAULT, Exception('boom')]
net_task = network_tasks.CreateVIPBasePort()
# Limit the retry attempts for the test run to save time
net_task.execute.retry.stop = tenacity.stop_after_attempt(2)
# Test execute
result = net_task.execute(vip_dict, VIP_SG_ID, AMP_ID)
self.assertEqual(port_mock.to_dict(), result)
mock_driver.create_port.assert_called_once_with(
VIP_NETWORK_ID, name=constants.AMP_BASE_PORT_PREFIX + AMP_ID,
fixed_ips=[{constants.SUBNET_ID: VIP_SUBNET_ID}],
secondary_ips=[VIP_IP_ADDRESS], security_group_ids=[VIP_SG_ID],
qos_policy_id=VIP_QOS_ID)
# Test execute exception
mock_driver.reset_mock()
self.assertRaises(exceptions.OctaviaException, net_task.execute,
vip_dict, None, AMP_ID)
# Test revert when this task failed
mock_driver.reset_mock()
net_task.revert(failure.Failure.from_exception(Exception('boom')),
vip_dict, VIP_SG_ID, AMP_ID)
mock_driver.delete_port.assert_not_called()
# Test revert
mock_driver.reset_mock()
net_task.revert([port_mock], vip_dict, VIP_SG_ID, AMP_ID)
mock_driver.delete_port.assert_called_once_with(PORT_ID)
# Test revert exception
mock_driver.reset_mock()
net_task.revert([port_mock], vip_dict, VIP_SG_ID, AMP_ID)
mock_driver.delete_port.assert_called_once_with(PORT_ID)
@mock.patch('time.sleep')
def test_admin_down_port(self, mock_sleep, mock_get_net_driver):
PORT_ID = uuidutils.generate_uuid()
mock_driver = mock.MagicMock()
mock_get_net_driver.return_value = mock_driver
port_down_mock = mock.MagicMock()
port_down_mock.status = constants.DOWN
port_up_mock = mock.MagicMock()
port_up_mock.status = constants.UP
mock_driver.set_port_admin_state_up.side_effect = [
mock.DEFAULT, net_base.PortNotFound, mock.DEFAULT, mock.DEFAULT,
Exception('boom')]
mock_driver.get_port.side_effect = [port_down_mock, port_up_mock]
net_task = network_tasks.AdminDownPort()
# Test execute
net_task.execute(PORT_ID)
mock_driver.set_port_admin_state_up.assert_called_once_with(PORT_ID,
False)
mock_driver.get_port.assert_called_once_with(PORT_ID)
# Test passive fail on port not found
mock_driver.reset_mock()
net_task.execute(PORT_ID)
mock_driver.set_port_admin_state_up.assert_called_once_with(PORT_ID,
False)
mock_driver.get_port.assert_not_called()
# Test passive fail on port stays up
mock_driver.reset_mock()
net_task.execute(PORT_ID)
mock_driver.set_port_admin_state_up.assert_called_once_with(PORT_ID,
False)
mock_driver.get_port.assert_called_once_with(PORT_ID)
# Test revert when this task failed
mock_driver.reset_mock()
net_task.revert(failure.Failure.from_exception(Exception('boom')),
PORT_ID)
mock_driver.set_port_admin_state_up.assert_not_called()
# Test revert
mock_driver.reset_mock()
net_task.revert(None, PORT_ID)
mock_driver.set_port_admin_state_up.assert_called_once_with(PORT_ID,
True)
# Test revert exception passive failure
mock_driver.reset_mock()
net_task.revert(None, PORT_ID)
mock_driver.set_port_admin_state_up.assert_called_once_with(PORT_ID,
True)
@mock.patch('octavia.common.utils.get_vip_security_group_name')
def test_get_vip_security_group_id(self, mock_get_sg_name,
mock_get_net_driver):
LB_ID = uuidutils.generate_uuid()
SG_ID = uuidutils.generate_uuid()
SG_NAME = 'fake_SG_name'
mock_driver = mock.MagicMock()
mock_get_net_driver.return_value = mock_driver
mock_get_sg_name.return_value = SG_NAME
sg_mock = mock.MagicMock()
sg_mock.id = SG_ID
mock_driver.get_security_group.side_effect = [
sg_mock, None, net_base.SecurityGroupNotFound,
net_base.SecurityGroupNotFound]
net_task = network_tasks.GetVIPSecurityGroupID()
# Test execute
result = net_task.execute(LB_ID)
mock_driver.get_security_group.assert_called_once_with(SG_NAME)
mock_get_sg_name.assert_called_once_with(LB_ID)
# Test execute with empty get subnet response
mock_driver.reset_mock()
mock_get_sg_name.reset_mock()
result = net_task.execute(LB_ID)
self.assertIsNone(result)
mock_get_sg_name.assert_called_once_with(LB_ID)
# Test execute no security group found, security groups enabled
mock_driver.reset_mock()
mock_get_sg_name.reset_mock()
mock_driver.sec_grp_enabled = True
self.assertRaises(net_base.SecurityGroupNotFound, net_task.execute,
LB_ID)
mock_driver.get_security_group.assert_called_once_with(SG_NAME)
mock_get_sg_name.assert_called_once_with(LB_ID)
# Test execute no security group found, security groups disabled
mock_driver.reset_mock()
mock_get_sg_name.reset_mock()
mock_driver.sec_grp_enabled = False
result = net_task.execute(LB_ID)
self.assertIsNone(result)
mock_driver.get_security_group.assert_called_once_with(SG_NAME)
mock_get_sg_name.assert_called_once_with(LB_ID)

47
octavia/tests/unit/controller/worker/v2/tasks/test_retry_tasks.py Normal file
View File

@ -0,0 +1,47 @@
# Copyright 2020 Red Hat, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from unittest import mock
from taskflow import retry
from octavia.controller.worker.v2.tasks import retry_tasks
import octavia.tests.unit.base as base
class TestRetryTasks(base.TestCase):
def setUp(self):
super(TestRetryTasks, self).setUp()
@mock.patch('time.sleep')
def test_sleeping_retry_times_controller(self, mock_sleep):
retry_ctrlr = retry_tasks.SleepingRetryTimesController(
attempts=2, name='test_retry')
# Test on_failure that should RETRY
history = ['boom']
result = retry_ctrlr.on_failure(history)
self.assertEqual(retry.RETRY, result)
# Test on_failure retries exhausted, should REVERT
history = ['boom', 'bang', 'pow']
result = retry_ctrlr.on_failure(history)
self.assertEqual(retry.REVERT, result)
# Test revert - should not raise
retry_ctrlr.revert(history)

943
octavia/tests/unit/controller/worker/v2/test_controller_worker.py
View File

File diff suppressed because it is too large Load Diff

27
tools/create_flow_docs.py
View File

@ -21,6 +21,7 @@ import os
import graphviz
from taskflow import engines
from octavia.api.drivers import utils
from octavia.common import constants
from octavia.tests.common import data_model_helpers as dmh
@ -56,6 +57,9 @@ def generate(flow_list, output_directory):
amp1 = dmh.generate_amphora()
amp2 = dmh.generate_amphora()
lb = dmh.generate_load_balancer(amphorae=[amp1, amp2])
if 'v2' in current_tuple[0]:
lb = utils.lb_dict_to_provider_dict(lb.to_dict())
amp1 = amp1.to_dict()
current_engine = engines.load(
get_flow_method(amp1, 2))
elif (current_tuple[1] == 'LoadBalancerFlows' and
@ -66,11 +70,27 @@ def generate(flow_list, output_directory):
elif (current_tuple[1] == 'LoadBalancerFlows' and
current_tuple[2] == 'get_delete_load_balancer_flow'):
lb = dmh.generate_load_balancer()
if 'v2' in current_tuple[0]:
lb = utils.lb_dict_to_provider_dict(lb.to_dict())
delete_flow = get_flow_method(lb)
else:
delete_flow, store = get_flow_method(lb)
current_engine = engines.load(delete_flow)
elif (current_tuple[1] == 'LoadBalancerFlows' and
current_tuple[2] == 'get_cascade_delete_load_balancer_flow'):
listeners = [{constants.LISTENER_ID:
'368dffc7-7440-4ee0-aca5-11052d001b05'},
{constants.LISTENER_ID:
'd9c45ec4-9dbe-491b-9f21-6886562348bf'}]
pools = [{constants.POOL_ID:
'6886a40b-1f2a-41a3-9ece-5c51845a7ac4'},
{constants.POOL_ID:
'08ada7a2-3eff-42c6-bdd8-b6f2ecd73358'}]
lb = dmh.generate_load_balancer()
if 'v2' in current_tuple[0]:
lb = utils.lb_dict_to_provider_dict(lb.to_dict())
delete_flow = get_flow_method(lb, listeners, pools)
else:
delete_flow, store = get_flow_method(lb)
current_engine = engines.load(delete_flow)
elif (current_tuple[1] == 'LoadBalancerFlows' and
@ -80,6 +100,13 @@ def generate(flow_list, output_directory):
lb = dmh.generate_load_balancer(
amphorae=[amp1, amp2],
topology=constants.TOPOLOGY_ACTIVE_STANDBY)
if 'v2' in current_tuple[0]:
lb = utils.lb_dict_to_provider_dict(lb.to_dict())
flavor = {constants.LOADBALANCER_TOPOLOGY:
constants.TOPOLOGY_ACTIVE_STANDBY}
lb[constants.FLAVOR] = flavor
amp1 = amp1.to_dict()
amp2 = amp2.to_dict()
current_engine = engines.load(
get_flow_method([amp1, amp2], lb))
elif (current_tuple[1] == 'MemberFlows' and

32
tools/flow-list-v2.txt Normal file
View File

@ -0,0 +1,32 @@
# List of TaskFlow flows that should be documented
# Some flows are used by other flows, so just list the primary flows here
# Format:
# module class flow
octavia.controller.worker.v2.flows.amphora_flows AmphoraFlows get_create_amphora_flow
octavia.controller.worker.v2.flows.amphora_flows AmphoraFlows get_failover_amphora_flow
octavia.controller.worker.v2.flows.amphora_flows AmphoraFlows cert_rotate_amphora_flow
octavia.controller.worker.v2.flows.load_balancer_flows LoadBalancerFlows get_create_load_balancer_flow
octavia.controller.worker.v2.flows.load_balancer_flows LoadBalancerFlows get_delete_load_balancer_flow
octavia.controller.worker.v2.flows.load_balancer_flows LoadBalancerFlows get_cascade_delete_load_balancer_flow
octavia.controller.worker.v2.flows.load_balancer_flows LoadBalancerFlows get_update_load_balancer_flow
octavia.controller.worker.v2.flows.load_balancer_flows LoadBalancerFlows get_failover_LB_flow
octavia.controller.worker.v2.flows.listener_flows ListenerFlows get_create_listener_flow
octavia.controller.worker.v2.flows.listener_flows ListenerFlows get_create_all_listeners_flow
octavia.controller.worker.v2.flows.listener_flows ListenerFlows get_delete_listener_flow
octavia.controller.worker.v2.flows.listener_flows ListenerFlows get_update_listener_flow
octavia.controller.worker.v2.flows.pool_flows PoolFlows get_create_pool_flow
octavia.controller.worker.v2.flows.pool_flows PoolFlows get_delete_pool_flow
octavia.controller.worker.v2.flows.pool_flows PoolFlows get_update_pool_flow
octavia.controller.worker.v2.flows.member_flows MemberFlows get_create_member_flow
octavia.controller.worker.v2.flows.member_flows MemberFlows get_delete_member_flow
octavia.controller.worker.v2.flows.member_flows MemberFlows get_update_member_flow
octavia.controller.worker.v2.flows.member_flows MemberFlows get_batch_update_members_flow
octavia.controller.worker.v2.flows.health_monitor_flows HealthMonitorFlows get_create_health_monitor_flow
octavia.controller.worker.v2.flows.health_monitor_flows HealthMonitorFlows get_delete_health_monitor_flow
octavia.controller.worker.v2.flows.health_monitor_flows HealthMonitorFlows get_update_health_monitor_flow
octavia.controller.worker.v2.flows.l7policy_flows L7PolicyFlows get_create_l7policy_flow
octavia.controller.worker.v2.flows.l7policy_flows L7PolicyFlows get_delete_l7policy_flow
octavia.controller.worker.v2.flows.l7policy_flows L7PolicyFlows get_update_l7policy_flow
octavia.controller.worker.v2.flows.l7rule_flows L7RuleFlows get_create_l7rule_flow
octavia.controller.worker.v2.flows.l7rule_flows L7RuleFlows get_delete_l7rule_flow
octavia.controller.worker.v2.flows.l7rule_flows L7RuleFlows get_update_l7rule_flow