diff --git a/etc/neutron.conf b/etc/neutron.conf index 6e1c3cc4a64..a76ec536bd3 100644 --- a/etc/neutron.conf +++ b/etc/neutron.conf @@ -224,6 +224,22 @@ # ports - number of ports associated with the networks hosted on the agent # dhcp_load_type = networks +# Availability Zone support +# +# Default value of availability zone hints. The availability zone aware +# schedulers use this when the resources availability_zone_hints is empty. +# Multiple availability zones can be specified by a comma separated string. +# This value can be empty. In this case, even if availability_zone_hints for +# a resource is empty, availability zone is considered for high availability +# while scheduling the resource. +# default_availability_zones = +# +# Make network scheduler availability zone aware. +# If multiple availability zones are used, set network_scheduler_driver = +# neutron.scheduler.dhcp_agent_scheduler.AZAwareWeightScheduler +# This scheduler selects agent depending on WeightScheduler logic within an +# availability zone so that considers the weight of agent. + # Allow auto scheduling networks to DHCP agent. It will schedule non-hosted # networks to first DHCP agent which sends get_active_networks message to # neutron server diff --git a/neutron/api/v2/attributes.py b/neutron/api/v2/attributes.py index 3587fb3d0f0..506ee2f3e26 100644 --- a/neutron/api/v2/attributes.py +++ b/neutron/api/v2/attributes.py @@ -120,6 +120,21 @@ def _validate_string(data, max_len=None): return msg +def validate_list_of_unique_strings(data, max_string_len=None): + if not isinstance(data, list): + msg = _("'%s' is not a list") % data + return msg + + if len(set(data)) != len(data): + msg = _("Duplicate items in the list: '%s'") % ', '.join(data) + return msg + + for item in data: + msg = _validate_string(item, max_string_len) + if msg: + return msg + + def _validate_boolean(data, valid_values=None): try: convert_to_boolean(data) @@ -635,7 +650,8 @@ validators = {'type:dict': _validate_dict, 'type:uuid_or_none': _validate_uuid_or_none, 'type:uuid_list': _validate_uuid_list, 'type:values': _validate_values, - 'type:boolean': _validate_boolean} + 'type:boolean': _validate_boolean, + 'type:list_of_unique_strings': validate_list_of_unique_strings} # Define constants for base resource name NETWORK = 'network' diff --git a/neutron/common/config.py b/neutron/common/config.py index 49631abefd0..5f0515462ad 100644 --- a/neutron/common/config.py +++ b/neutron/common/config.py @@ -63,6 +63,16 @@ core_opts = [ help=_("The maximum number of items returned in a single " "response, value was 'infinite' or negative integer " "means no limit")), + cfg.ListOpt('default_availability_zones', default=[], + help=_("Default value of availability zone hints. The " + "availability zone aware schedulers use this when " + "the resources availability_zone_hints is empty. " + "Multiple availability zones can be specified by a " + "comma separated string. This value can be empty. " + "In this case, even if availability_zone_hints for " + "a resource is empty, availability zone is " + "considered for high availability while scheduling " + "the resource.")), cfg.IntOpt('max_dns_nameservers', default=5, help=_("Maximum number of DNS nameservers")), cfg.IntOpt('max_subnet_host_routes', default=20, diff --git a/neutron/db/agentschedulers_db.py b/neutron/db/agentschedulers_db.py index 924cdb41699..1bc04bd63c7 100644 --- a/neutron/db/agentschedulers_db.py +++ b/neutron/db/agentschedulers_db.py @@ -29,6 +29,7 @@ from neutron.common import constants from neutron.common import utils from neutron import context as ncontext from neutron.db import agents_db +from neutron.db.availability_zone import network as network_az from neutron.db import model_base from neutron.extensions import agent as ext_agent from neutron.extensions import dhcpagentscheduler @@ -459,6 +460,21 @@ class DhcpAgentSchedulerDbMixin(dhcpagentscheduler self.network_scheduler.auto_schedule_networks(self, context, host) +class AZDhcpAgentSchedulerDbMixin(DhcpAgentSchedulerDbMixin, + network_az.NetworkAvailabilityZoneMixin): + """Mixin class to add availability_zone supported DHCP agent scheduler.""" + + def get_network_availability_zones(self, network_id): + context = ncontext.get_admin_context() + with context.session.begin(): + query = context.session.query(agents_db.Agent.availability_zone) + query = query.join(NetworkDhcpAgentBinding) + query = query.filter( + NetworkDhcpAgentBinding.network_id == network_id) + query = query.group_by(agents_db.Agent.availability_zone) + return [item[0] for item in query] + + # helper functions for readability. def services_available(admin_state_up): if cfg.CONF.enable_services_on_agents_with_admin_state_down: diff --git a/neutron/db/availability_zone/__init__.py b/neutron/db/availability_zone/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/neutron/db/availability_zone/network.py b/neutron/db/availability_zone/network.py new file mode 100644 index 00000000000..c08d0da6d2b --- /dev/null +++ b/neutron/db/availability_zone/network.py @@ -0,0 +1,35 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging + +from neutron.api.v2 import attributes +from neutron.db import common_db_mixin +from neutron.extensions import availability_zone as az_ext +from neutron.extensions import network_availability_zone as net_az + + +LOG = logging.getLogger(__name__) + + +class NetworkAvailabilityZoneMixin(net_az.NetworkAvailabilityZonePluginBase): + """Mixin class to enable network's availability zone attributes.""" + + def _extend_availability_zone(self, net_res, net_db): + net_res[az_ext.AZ_HINTS] = az_ext.convert_az_string_to_list( + net_db[az_ext.AZ_HINTS]) + net_res[az_ext.AVAILABILITY_ZONES] = ( + self.get_network_availability_zones(net_db['id'])) + + common_db_mixin.CommonDbMixin.register_dict_extend_funcs( + attributes.NETWORKS, ['_extend_availability_zone']) diff --git a/neutron/db/migration/alembic_migrations/versions/EXPAND_HEAD b/neutron/db/migration/alembic_migrations/versions/EXPAND_HEAD index ce183c9bb19..2e7ac1ec3ca 100644 --- a/neutron/db/migration/alembic_migrations/versions/EXPAND_HEAD +++ b/neutron/db/migration/alembic_migrations/versions/EXPAND_HEAD @@ -1 +1 @@ -32e5974ada25 +ec7fcfbf72ee diff --git a/neutron/db/migration/alembic_migrations/versions/mitaka/expand/ec7fcfbf72ee_network_az.py b/neutron/db/migration/alembic_migrations/versions/mitaka/expand/ec7fcfbf72ee_network_az.py new file mode 100644 index 00000000000..e6582d4cd1b --- /dev/null +++ b/neutron/db/migration/alembic_migrations/versions/mitaka/expand/ec7fcfbf72ee_network_az.py @@ -0,0 +1,33 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""Add network availability zone + +Revision ID: ec7fcfbf72ee +Revises: 32e5974ada25 +Create Date: 2015-09-17 09:21:51.257579 + +""" + +# revision identifiers, used by Alembic. +revision = 'ec7fcfbf72ee' +down_revision = '32e5974ada25' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('networks', + sa.Column('availability_zone_hints', sa.String(length=255))) diff --git a/neutron/db/models_v2.py b/neutron/db/models_v2.py index 7f0750a0e77..fbcf6128790 100644 --- a/neutron/db/models_v2.py +++ b/neutron/db/models_v2.py @@ -266,3 +266,4 @@ class Network(model_base.HasStandardAttributes, model_base.BASEV2, rbac_entries = orm.relationship(rbac_db_models.NetworkRBAC, backref='network', lazy='joined', cascade='all, delete, delete-orphan') + availability_zone_hints = sa.Column(sa.String(255)) diff --git a/neutron/extensions/availability_zone.py b/neutron/extensions/availability_zone.py index 2c06c18337e..d0cdf1bd928 100644 --- a/neutron/extensions/availability_zone.py +++ b/neutron/extensions/availability_zone.py @@ -14,6 +14,8 @@ import abc +from oslo_serialization import jsonutils + from neutron.api import extensions from neutron.api.v2 import attributes as attr from neutron.api.v2 import base @@ -21,9 +23,36 @@ from neutron.common import exceptions from neutron import manager +AZ_HINTS_DB_LEN = 255 + + +# resource independent common methods +def convert_az_list_to_string(az_list): + return jsonutils.dumps(az_list) + + +def convert_az_string_to_list(az_string): + return jsonutils.loads(az_string) if az_string else [] + + +def _validate_availability_zone_hints(data, valid_value=None): + # syntax check only here. existence of az will be checked later. + msg = attr.validate_list_of_unique_strings(data) + if msg: + return msg + az_string = convert_az_list_to_string(data) + if len(az_string) > AZ_HINTS_DB_LEN: + msg = _("Too many availability_zone_hints specified") + raise exceptions.InvalidInput(error_message=msg) + + +attr.validators['type:availability_zone_hints'] = ( + _validate_availability_zone_hints) + # Attribute Map RESOURCE_NAME = 'availability_zone' AVAILABILITY_ZONES = 'availability_zones' +AZ_HINTS = 'availability_zone_hints' # name: name of availability zone (string) # resource: type of resource: 'network' or 'router' # state: state of availability zone: 'available' or 'unavailable' @@ -99,9 +128,9 @@ class AvailabilityZonePluginBase(object): def get_availability_zones(self, context, filters=None, fields=None, sorts=None, limit=None, marker=None, page_reverse=False): - pass + """Return availability zones which a resource belongs to""" @abc.abstractmethod def validate_availability_zones(self, context, resource_type, availability_zones): - pass + """Verify that the availability zones exist.""" diff --git a/neutron/extensions/network_availability_zone.py b/neutron/extensions/network_availability_zone.py new file mode 100644 index 00000000000..192a6c299a3 --- /dev/null +++ b/neutron/extensions/network_availability_zone.py @@ -0,0 +1,68 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc + +import six + +from neutron.api import extensions +from neutron.extensions import availability_zone as az_ext + + +EXTENDED_ATTRIBUTES_2_0 = { + 'networks': { + az_ext.AVAILABILITY_ZONES: {'allow_post': False, 'allow_put': False, + 'is_visible': True}, + az_ext.AZ_HINTS: { + 'allow_post': True, 'allow_put': False, 'is_visible': True, + 'validate': {'type:availability_zone_hints': None}, + 'default': []}}, +} + + +class Network_availability_zone(extensions.ExtensionDescriptor): + """Network availability zone extension.""" + + @classmethod + def get_name(cls): + return "Network Availability Zone" + + @classmethod + def get_alias(cls): + return "network_availability_zone" + + @classmethod + def get_description(cls): + return "Availability zone support for network." + + @classmethod + def get_updated(cls): + return "2015-01-01T10:00:00-00:00" + + def get_required_extensions(self): + return ["availability_zone"] + + def get_extended_resources(self, version): + if version == "2.0": + return EXTENDED_ATTRIBUTES_2_0 + else: + return {} + + +@six.add_metaclass(abc.ABCMeta) +class NetworkAvailabilityZonePluginBase(object): + + @abc.abstractmethod + def get_network_availability_zones(self, network_id): + """Return availability zones which a network belongs to""" diff --git a/neutron/plugins/ml2/plugin.py b/neutron/plugins/ml2/plugin.py index e03cb2f8c4c..c72622fcd35 100644 --- a/neutron/plugins/ml2/plugin.py +++ b/neutron/plugins/ml2/plugin.py @@ -60,6 +60,7 @@ from neutron.db import securitygroups_db from neutron.db import securitygroups_rpc_base as sg_db_rpc from neutron.db import vlantransparent_db from neutron.extensions import allowedaddresspairs as addr_pair +from neutron.extensions import availability_zone as az_ext from neutron.extensions import extra_dhcp_opt as edo_ext from neutron.extensions import portbindings from neutron.extensions import portsecurity as psec @@ -88,7 +89,7 @@ class Ml2Plugin(db_base_plugin_v2.NeutronDbPluginV2, dvr_mac_db.DVRDbMixin, external_net_db.External_net_db_mixin, sg_db_rpc.SecurityGroupServerRpcMixin, - agentschedulers_db.DhcpAgentSchedulerDbMixin, + agentschedulers_db.AZDhcpAgentSchedulerDbMixin, addr_pair_db.AllowedAddressPairsMixin, vlantransparent_db.Vlantransparent_db_mixin, extradhcpopt_db.ExtraDhcpOptMixin, @@ -119,7 +120,8 @@ class Ml2Plugin(db_base_plugin_v2.NeutronDbPluginV2, "extra_dhcp_opt", "subnet_allocation", "net-mtu", "vlan-transparent", "address-scope", "dns-integration", - "availability_zone"] + "availability_zone", + "network_availability_zone"] @property def supported_extension_aliases(self): @@ -641,6 +643,15 @@ class Ml2Plugin(db_base_plugin_v2.NeutronDbPluginV2, result['id'], {'network': {api.MTU: net_data[api.MTU]}}) result[api.MTU] = res.get(api.MTU, 0) + if az_ext.AZ_HINTS in net_data: + self.validate_availability_zones(context, 'network', + net_data[az_ext.AZ_HINTS]) + az_hints = az_ext.convert_az_list_to_string( + net_data[az_ext.AZ_HINTS]) + super(Ml2Plugin, self).update_network(context, + result['id'], {'network': {az_ext.AZ_HINTS: az_hints}}) + result[az_ext.AZ_HINTS] = az_hints + return result, mech_context def create_network(self, context, network): diff --git a/neutron/scheduler/dhcp_agent_scheduler.py b/neutron/scheduler/dhcp_agent_scheduler.py index 4dab4f058e7..084fc6fa86f 100644 --- a/neutron/scheduler/dhcp_agent_scheduler.py +++ b/neutron/scheduler/dhcp_agent_scheduler.py @@ -14,6 +14,9 @@ # under the License. +import collections +import heapq + from oslo_config import cfg from oslo_db import exception as db_exc from oslo_log import log as logging @@ -22,6 +25,7 @@ from sqlalchemy import sql from neutron.common import constants from neutron.db import agents_db from neutron.db import agentschedulers_db +from neutron.extensions import availability_zone as az_ext from neutron.i18n import _LI, _LW from neutron.scheduler import base_resource_filter from neutron.scheduler import base_scheduler @@ -64,6 +68,12 @@ class AutoScheduler(object): continue if any(dhcp_agent.id == agent.id for agent in agents): continue + net = plugin.get_network(context, net_id) + az_hints = (net.get(az_ext.AZ_HINTS) or + cfg.CONF.default_availability_zones) + if (az_hints and + dhcp_agent['availability_zone'] not in az_hints): + continue bindings_to_add.append((dhcp_agent, net_id)) # do it outside transaction so particular scheduling results don't # make other to fail @@ -84,6 +94,46 @@ class WeightScheduler(base_scheduler.BaseWeightScheduler, AutoScheduler): super(WeightScheduler, self).__init__(DhcpFilter()) +class AZAwareWeightScheduler(WeightScheduler): + + def select(self, plugin, context, resource_hostable_agents, + resource_hosted_agents, num_agents_needed): + """AZ aware scheduling + If the network has multiple AZs, agents are scheduled as + follows: + - select AZ with least agents scheduled for the network + (nondeterministic for AZs with same amount of agents scheduled) + - choose agent in the AZ with WeightScheduler + """ + hostable_az_agents = collections.defaultdict(list) + num_az_agents = {} + for agent in resource_hostable_agents: + az_agent = agent['availability_zone'] + hostable_az_agents[az_agent].append(agent) + if az_agent not in num_az_agents: + num_az_agents[az_agent] = 0 + if num_agents_needed <= 0: + return [] + for agent in resource_hosted_agents: + az_agent = agent['availability_zone'] + if az_agent in num_az_agents: + num_az_agents[az_agent] += 1 + + num_az_q = [(value, key) for key, value in num_az_agents.items()] + heapq.heapify(num_az_q) + chosen_agents = [] + while num_agents_needed > 0: + num, select_az = heapq.heappop(num_az_q) + select_agent = super(AZAwareWeightScheduler, self).select( + plugin, context, hostable_az_agents[select_az], [], 1) + chosen_agents.append(select_agent[0]) + hostable_az_agents[select_az].remove(select_agent[0]) + if hostable_az_agents[select_az]: + heapq.heappush(num_az_q, (num + 1, select_az)) + num_agents_needed -= 1 + return chosen_agents + + class DhcpFilter(base_resource_filter.BaseResourceFilter): def bind(self, context, agents, network_id): @@ -147,13 +197,15 @@ class DhcpFilter(base_resource_filter.BaseResourceFilter): return return network_hosted_agents - def _get_active_agents(self, plugin, context): + def _get_active_agents(self, plugin, context, az_hints): """Return a list of active dhcp agents.""" with context.session.begin(subtransactions=True): + filters = {'agent_type': [constants.AGENT_TYPE_DHCP], + 'admin_state_up': [True]} + if az_hints: + filters['availability_zone'] = az_hints active_dhcp_agents = plugin.get_agents_db( - context, filters={ - 'agent_type': [constants.AGENT_TYPE_DHCP], - 'admin_state_up': [True]}) + context, filters=filters) if not active_dhcp_agents: LOG.warn(_LW('No more DHCP agents')) return [] @@ -171,7 +223,9 @@ class DhcpFilter(base_resource_filter.BaseResourceFilter): if hosted_agents is None: return {'n_agents': 0, 'hostable_agents': [], 'hosted_agents': []} n_agents = cfg.CONF.dhcp_agents_per_network - len(hosted_agents) - active_dhcp_agents = self._get_active_agents(plugin, context) + az_hints = (network.get(az_ext.AZ_HINTS) or + cfg.CONF.default_availability_zones) + active_dhcp_agents = self._get_active_agents(plugin, context, az_hints) if not active_dhcp_agents: return {'n_agents': 0, 'hostable_agents': [], 'hosted_agents': hosted_agents} diff --git a/neutron/tests/functional/scheduler/test_dhcp_agent_scheduler.py b/neutron/tests/functional/scheduler/test_dhcp_agent_scheduler.py index 0cce1fceb70..e65a3874b39 100644 --- a/neutron/tests/functional/scheduler/test_dhcp_agent_scheduler.py +++ b/neutron/tests/functional/scheduler/test_dhcp_agent_scheduler.py @@ -345,6 +345,10 @@ class TestAutoSchedule(test_dhcp_sch.TestDhcpSchedulerBaseTestCase, 'enable_dhcp': enable_dhcp}) return subnets + def get_network(self, context, net_id): + # TODO(hichihara): add test cases of AZ scheduler + return {'availability_zone_hints': []} + def _get_hosted_networks_on_dhcp_agent(self, agent_id): query = self.ctx.session.query( agentschedulers_db.NetworkDhcpAgentBinding.network_id) diff --git a/neutron/tests/unit/api/v2/test_attributes.py b/neutron/tests/unit/api/v2/test_attributes.py index 4c4a95d1b93..fe3953a8bcf 100644 --- a/neutron/tests/unit/api/v2/test_attributes.py +++ b/neutron/tests/unit/api/v2/test_attributes.py @@ -113,6 +113,24 @@ class TestAttributes(base.BaseTestCase): msg = attributes._validate_string("123456789", None) self.assertIsNone(msg) + def test_validate_list_of_unique_strings(self): + data = "TEST" + msg = attributes.validate_list_of_unique_strings(data, None) + self.assertEqual("'TEST' is not a list", msg) + + data = ["TEST01", "TEST02", "TEST01"] + msg = attributes.validate_list_of_unique_strings(data, None) + self.assertEqual( + "Duplicate items in the list: 'TEST01, TEST02, TEST01'", msg) + + data = ["12345678", "123456789"] + msg = attributes.validate_list_of_unique_strings(data, 8) + self.assertEqual("'123456789' exceeds maximum length of 8", msg) + + data = ["TEST01", "TEST02", "TEST03"] + msg = attributes.validate_list_of_unique_strings(data, None) + self.assertIsNone(msg) + def test_validate_no_whitespace(self): data = 'no_white_space' result = attributes._validate_no_whitespace(data) diff --git a/neutron/tests/unit/db/test_db_base_plugin_v2.py b/neutron/tests/unit/db/test_db_base_plugin_v2.py index 51eac478616..587a5a2f843 100644 --- a/neutron/tests/unit/db/test_db_base_plugin_v2.py +++ b/neutron/tests/unit/db/test_db_base_plugin_v2.py @@ -292,7 +292,8 @@ class NeutronDbPluginV2TestCase(testlib_api.WebTestCase): 'admin_state_up': admin_state_up, 'tenant_id': self._tenant_id}} for arg in (('admin_state_up', 'tenant_id', 'shared', - 'vlan_transparent') + (arg_list or ())): + 'vlan_transparent', + 'availability_zone_hints') + (arg_list or ())): # Arg must be present if arg in kwargs: data['network'][arg] = kwargs[arg] @@ -5610,7 +5611,9 @@ class DbModelTestCase(testlib_api.SqlTestCase): exp_end_with = (" {tenant_id=None, id=None, " "name='net_net', status='OK', " "admin_state_up=True, mtu=None, " - "vlan_transparent=None, standard_attr_id=None}>") + "vlan_transparent=None, " + "availability_zone_hints=None, " + "standard_attr_id=None}>") final_exp = exp_start_with + exp_middle + exp_end_with self.assertEqual(final_exp, actual_repr_output) diff --git a/neutron/tests/unit/extensions/test_availability_zone.py b/neutron/tests/unit/extensions/test_availability_zone.py index d68ea90b977..40b04518489 100644 --- a/neutron/tests/unit/extensions/test_availability_zone.py +++ b/neutron/tests/unit/extensions/test_availability_zone.py @@ -40,7 +40,6 @@ class AZExtensionManager(object): return [] -# This plugin class is just for testing class AZTestPlugin(db_base_plugin_v2.NeutronDbPluginV2, agents_db.AgentDbMixin): supported_extension_aliases = ["agent", "availability_zone"] @@ -96,3 +95,41 @@ class TestAZAgentCase(AZTestCommon): self.assertRaises(az_ext.AvailabilityZoneNotFound, self.plugin.validate_availability_zones, ctx, 'router', ['nova1']) + + +class TestAZNetworkCase(AZTestCommon): + def setUp(self): + plugin = 'neutron.plugins.ml2.plugin.Ml2Plugin' + ext_mgr = AZExtensionManager() + super(TestAZNetworkCase, self).setUp(plugin=plugin, ext_mgr=ext_mgr) + + def test_create_network_with_az(self): + self._register_azs() + az_hints = ['nova1'] + with self.network(availability_zone_hints=az_hints) as net: + res = self._show('networks', net['network']['id']) + self.assertItemsEqual(az_hints, + res['network']['availability_zone_hints']) + + def test_create_network_with_azs(self): + self._register_azs() + az_hints = ['nova1', 'nova2'] + with self.network(availability_zone_hints=az_hints) as net: + res = self._show('networks', net['network']['id']) + self.assertItemsEqual(az_hints, + res['network']['availability_zone_hints']) + + def test_create_network_without_az(self): + with self.network() as net: + res = self._show('networks', net['network']['id']) + self.assertEqual([], res['network']['availability_zone_hints']) + + def test_create_network_with_empty_az(self): + with self.network(availability_zone_hints=[]) as net: + res = self._show('networks', net['network']['id']) + self.assertEqual([], res['network']['availability_zone_hints']) + + def test_create_network_with_not_exist_az(self): + res = self._create_network(self.fmt, 'net', True, + availability_zone_hints=['nova3']) + self.assertEqual(404, res.status_int) diff --git a/neutron/tests/unit/plugins/ml2/test_extension_driver_api.py b/neutron/tests/unit/plugins/ml2/test_extension_driver_api.py index 7faa5e138b2..2297007f497 100644 --- a/neutron/tests/unit/plugins/ml2/test_extension_driver_api.py +++ b/neutron/tests/unit/plugins/ml2/test_extension_driver_api.py @@ -80,7 +80,7 @@ class ExtensionDriverTestCase(test_plugin.Ml2PluginV2TestCase): def test_faulty_extend_dict(self): with mock.patch.object(ext_test.TestExtensionDriver, 'extend_network_dict', - side_effect=TypeError): + side_effect=[None, TypeError]): network, tid = self._verify_network_create(201, None) self._verify_network_update(network, 400, 'ExtensionDriverError') diff --git a/neutron/tests/unit/plugins/ml2/test_plugin.py b/neutron/tests/unit/plugins/ml2/test_plugin.py index a9dcb44f96b..b4e906000cf 100644 --- a/neutron/tests/unit/plugins/ml2/test_plugin.py +++ b/neutron/tests/unit/plugins/ml2/test_plugin.py @@ -1667,6 +1667,7 @@ class TestMl2PluginCreateUpdateDeletePort(base.BaseTestCase): plugin._get_host_port_if_changed = mock.Mock( return_value=new_host_port) plugin._check_mac_update_allowed = mock.Mock(return_value=True) + plugin._extend_availability_zone = mock.Mock() self.notify.side_effect = ( lambda r, e, t, **kwargs: self._ensure_transaction_is_closed()) diff --git a/neutron/tests/unit/scheduler/test_dhcp_agent_scheduler.py b/neutron/tests/unit/scheduler/test_dhcp_agent_scheduler.py index 71bc94c7a3c..f06fd46b185 100644 --- a/neutron/tests/unit/scheduler/test_dhcp_agent_scheduler.py +++ b/neutron/tests/unit/scheduler/test_dhcp_agent_scheduler.py @@ -135,6 +135,7 @@ class TestDhcpScheduler(TestDhcpSchedulerBaseTestCase): plugin = mock.Mock() plugin.get_subnets.return_value = [{"network_id": self.network_id, "enable_dhcp": True}] + plugin.get_network.return_value = self.network if active_hosts_only: plugin.get_dhcp_agents_hosting_networks.return_value = [] else: @@ -180,6 +181,10 @@ class TestAutoScheduleNetworks(TestDhcpSchedulerBaseTestCase): valid_host If true, then an valid host is passed to schedule the network, else an invalid host is passed. + + az_hints + 'availability_zone_hints' of the network. + note that default 'availability_zone' of an agent is 'nova'. """ scenarios = [ ('Network present', @@ -187,42 +192,64 @@ class TestAutoScheduleNetworks(TestDhcpSchedulerBaseTestCase): enable_dhcp=True, scheduled_already=False, agent_down=False, - valid_host=True)), + valid_host=True, + az_hints=[])), ('No network', dict(network_present=False, enable_dhcp=False, scheduled_already=False, agent_down=False, - valid_host=True)), + valid_host=True, + az_hints=[])), ('Network already scheduled', dict(network_present=True, enable_dhcp=True, scheduled_already=True, agent_down=False, - valid_host=True)), + valid_host=True, + az_hints=[])), ('Agent down', dict(network_present=True, enable_dhcp=True, scheduled_already=False, agent_down=False, - valid_host=True)), + valid_host=True, + az_hints=[])), ('dhcp disabled', dict(network_present=True, enable_dhcp=False, scheduled_already=False, agent_down=False, - valid_host=False)), + valid_host=False, + az_hints=[])), ('Invalid host', dict(network_present=True, enable_dhcp=True, scheduled_already=False, agent_down=False, - valid_host=False)), + valid_host=False, + az_hints=[])), + + ('Match AZ', + dict(network_present=True, + enable_dhcp=True, + scheduled_already=False, + agent_down=False, + valid_host=True, + az_hints=['nova'])), + + ('Not match AZ', + dict(network_present=True, + enable_dhcp=True, + scheduled_already=False, + agent_down=False, + valid_host=True, + az_hints=['not-match'])), ] def test_auto_schedule_network(self): @@ -230,6 +257,8 @@ class TestAutoScheduleNetworks(TestDhcpSchedulerBaseTestCase): plugin.get_subnets.return_value = ( [{"network_id": self.network_id, "enable_dhcp": self.enable_dhcp}] if self.network_present else []) + plugin.get_network.return_value = {'availability_zone_hints': + self.az_hints} scheduler = dhcp_agent_scheduler.ChanceScheduler() if self.network_present: down_agent_count = 1 if self.agent_down else 0 @@ -241,6 +270,9 @@ class TestAutoScheduleNetworks(TestDhcpSchedulerBaseTestCase): expected_result = (self.network_present and self.enable_dhcp) expected_hosted_agents = (1 if expected_result and self.valid_host else 0) + if (self.az_hints and + agents[0]['availability_zone'] not in self.az_hints): + expected_hosted_agents = 0 host = "host-a" if self.valid_host else "host-b" observed_ret_value = scheduler.auto_schedule_networks( plugin, self.ctx, host) @@ -448,3 +480,96 @@ class TestDhcpSchedulerFilter(TestDhcpSchedulerBaseTestCase, self._test_get_dhcp_agents_hosting_networks({'host-d'}, active=True, admin_state_up=False) + + +class DHCPAgentAZAwareWeightSchedulerTestCase(TestDhcpSchedulerBaseTestCase): + + def setUp(self): + super(DHCPAgentAZAwareWeightSchedulerTestCase, self).setUp() + DB_PLUGIN_KLASS = 'neutron.plugins.ml2.plugin.Ml2Plugin' + self.setup_coreplugin(DB_PLUGIN_KLASS) + cfg.CONF.set_override("network_scheduler_driver", + 'neutron.scheduler.dhcp_agent_scheduler.AZAwareWeightScheduler') + self.plugin = importutils.import_object('neutron.plugins.ml2.plugin.' + 'Ml2Plugin') + cfg.CONF.set_override('dhcp_agents_per_network', 1) + cfg.CONF.set_override("dhcp_load_type", "networks") + + def test_az_scheduler_one_az_hints(self): + self._save_networks(['1111']) + helpers.register_dhcp_agent('az1-host1', networks=1, az='az1') + helpers.register_dhcp_agent('az1-host2', networks=2, az='az1') + helpers.register_dhcp_agent('az2-host1', networks=3, az='az2') + helpers.register_dhcp_agent('az2-host2', networks=4, az='az2') + self.plugin.network_scheduler.schedule(self.plugin, self.ctx, + {'id': '1111', 'availability_zone_hints': ['az2']}) + agents = self.plugin.get_dhcp_agents_hosting_networks(self.ctx, + ['1111']) + self.assertEqual(1, len(agents)) + self.assertEqual('az2-host1', agents[0]['host']) + + def test_az_scheduler_default_az_hints(self): + cfg.CONF.set_override('default_availability_zones', ['az1']) + self._save_networks(['1111']) + helpers.register_dhcp_agent('az1-host1', networks=1, az='az1') + helpers.register_dhcp_agent('az1-host2', networks=2, az='az1') + helpers.register_dhcp_agent('az2-host1', networks=3, az='az2') + helpers.register_dhcp_agent('az2-host2', networks=4, az='az2') + self.plugin.network_scheduler.schedule(self.plugin, self.ctx, + {'id': '1111', 'availability_zone_hints': []}) + agents = self.plugin.get_dhcp_agents_hosting_networks(self.ctx, + ['1111']) + self.assertEqual(1, len(agents)) + self.assertEqual('az1-host1', agents[0]['host']) + + def test_az_scheduler_two_az_hints(self): + cfg.CONF.set_override('dhcp_agents_per_network', 2) + self._save_networks(['1111']) + helpers.register_dhcp_agent('az1-host1', networks=1, az='az1') + helpers.register_dhcp_agent('az1-host2', networks=2, az='az1') + helpers.register_dhcp_agent('az2-host1', networks=3, az='az2') + helpers.register_dhcp_agent('az2-host2', networks=4, az='az2') + helpers.register_dhcp_agent('az3-host1', networks=5, az='az3') + helpers.register_dhcp_agent('az3-host2', networks=6, az='az3') + self.plugin.network_scheduler.schedule(self.plugin, self.ctx, + {'id': '1111', 'availability_zone_hints': ['az1', 'az3']}) + agents = self.plugin.get_dhcp_agents_hosting_networks(self.ctx, + ['1111']) + self.assertEqual(2, len(agents)) + expected_hosts = set(['az1-host1', 'az3-host1']) + hosts = set([a['host'] for a in agents]) + self.assertEqual(expected_hosts, hosts) + + def test_az_scheduler_two_az_hints_one_available_az(self): + cfg.CONF.set_override('dhcp_agents_per_network', 2) + self._save_networks(['1111']) + helpers.register_dhcp_agent('az1-host1', networks=1, az='az1') + helpers.register_dhcp_agent('az1-host2', networks=2, az='az1') + helpers.register_dhcp_agent('az2-host1', networks=3, alive=False, + az='az2') + helpers.register_dhcp_agent('az2-host2', networks=4, + admin_state_up=False, az='az2') + self.plugin.network_scheduler.schedule(self.plugin, self.ctx, + {'id': '1111', 'availability_zone_hints': ['az1', 'az2']}) + agents = self.plugin.get_dhcp_agents_hosting_networks(self.ctx, + ['1111']) + self.assertEqual(2, len(agents)) + expected_hosts = set(['az1-host1', 'az1-host2']) + hosts = set([a['host'] for a in agents]) + self.assertEqual(expected_hosts, hosts) + + def test_az_scheduler_no_az_hints(self): + cfg.CONF.set_override('dhcp_agents_per_network', 2) + self._save_networks(['1111']) + helpers.register_dhcp_agent('az1-host1', networks=2, az='az1') + helpers.register_dhcp_agent('az1-host2', networks=3, az='az1') + helpers.register_dhcp_agent('az2-host1', networks=2, az='az2') + helpers.register_dhcp_agent('az2-host2', networks=1, az='az2') + self.plugin.network_scheduler.schedule(self.plugin, self.ctx, + {'id': '1111', 'availability_zone_hints': []}) + agents = self.plugin.get_dhcp_agents_hosting_networks(self.ctx, + ['1111']) + self.assertEqual(2, len(agents)) + expected_hosts = set(['az1-host1', 'az2-host2']) + hosts = {a['host'] for a in agents} + self.assertEqual(expected_hosts, hosts) diff --git a/releasenotes/notes/add-availability-zone-4440cf00be7c54ba.yaml b/releasenotes/notes/add-availability-zone-4440cf00be7c54ba.yaml new file mode 100644 index 00000000000..0a873f1f21d --- /dev/null +++ b/releasenotes/notes/add-availability-zone-4440cf00be7c54ba.yaml @@ -0,0 +1,4 @@ +--- +features: + - DHCP agent is assigned to a availability zone. Network can be host on the + DHCP agent with availability zone which users specify. \ No newline at end of file