Allow Juju AZ context information to be used

The change adds an option to the charm to use JUJU_AVAILABILITY_ZONE
environment variable set by Juju for the hook environment based on the
underlying provider's availability zone information for a given machine.

This information is used to configure the availability_zone setting for
Neutron DHCP and L3 agents specifically because they support it
and for other agents (because both neutron.conf and agent-specific
configuration files are loaded) such as metadata agents and lbaas
agents.

Additionally, a setting is added to allow changing the default
availability zone because 'nova' is a default value coming from the
Neutron defaults for agents.

Change-Id: I94303aa70ee3adc6ace0f9af1e7c4f5c0edbcdb5
Closes-Bug: #1796068

config.yaml

@@ -271,3 +271,31 @@ options:
       Only supported in OpenStack Newton and higher. For deployments of Queens or
       later this value is ignored. Please set the corresponding value in the
       nova-cloud-controller charm.
+  default-availability-zone:
+    type: string
+    default: 'nova'
+    description: |
+      Default availability zone to use for agents (l3, dhcp) on this machine.
+      If this option is not set, the default availability zone 'nova' is used.
+      If customize-failure-domain is set to True, it will override this option
+      only if an AZ is set by the Juju provider. If JUJU_AVAILABILITY_ZONE is
+      not set, the value specified by this option will be used regardless of
+      customize-failure-domain's setting.
+
+      .
+      NOTE: Router and Network objects have a property called
+      availability_zone_hints which can be used to restrict dnsmasq
+      and router namespace placement by DHCP and L3 agents to specific
+      neutron availability zones. Neutron AZs are not tied to Nova AZs but
+      their names can match.
+      .
+  customize-failure-domain:
+    type: boolean
+    default: False
+    description: |
+      Juju propagates availability zone information to charms from the
+      underlying machine provider such as MAAS and this option allows the
+      charm to use JUJU_AVAILABILITY_ZONE to set default_availability_zone
+      for Neutron agents (DHCP and L3 agents). This option overrides the
+      default-availability-zone charm config setting only when the Juju
+      provider sets JUJU_AVAILABILITY_ZONE.

hooks/neutron_contexts.py

@@ -19,9 +19,10 @@ from charmhelpers.contrib.openstack.utils import (
     os_release,
     CompareOpenStackReleases,
 )
-from charmhelpers.contrib.hahelpers.cluster import(
+from charmhelpers.contrib.hahelpers.cluster import (
     eligible_leader
 )
+
 from charmhelpers.contrib.network.ip import (
     get_address_in_network,
     get_host_ip,
@@ -48,6 +49,11 @@ CORE_PLUGIN = {
 }
 
 
+def _get_availability_zone():
+    from neutron_utils import get_availability_zone as get_az
+    return get_az()
+
+
 def core_plugin():
     return CORE_PLUGIN[config('plugin')]
 
@@ -122,6 +128,7 @@ class NeutronGatewayContext(NeutronAPIContext):
             'report_interval': api_settings['report_interval'],
             'enable_metadata_network': config('enable-metadata-network'),
             'enable_isolated_metadata': config('enable-isolated-metadata'),
+            'availability_zone': _get_availability_zone(),
         }
 
         ctxt['local_ip'] = get_local_ip()
@@ -197,6 +204,7 @@ class NovaMetadataContext(OSContextGenerator):
             ctxt['nova_metadata_protocol'] = 'http'
         return ctxt
 
+
 SHARED_SECRET = "/etc/{}/secret.txt"
 
 

hooks/neutron_utils.py

@@ -87,6 +87,7 @@ from copy import deepcopy
 def valid_plugin():
     return config('plugin') in CORE_PLUGIN
 
+
 NEUTRON_COMMON = 'neutron-common'
 VERSION_PACKAGE = NEUTRON_COMMON
 
@@ -301,6 +302,7 @@ def determine_l3ha_packages():
 def use_l3ha():
     return NeutronAPIContext()()['enable_l3ha']
 
+
 EXT_PORT_CONF = '/etc/init/ext-port.conf'
 PHY_NIC_MTU_CONF = '/etc/init/os-charm-phy-nic-mtu.conf'
 STOPPED_SERVICES = ['os-charm-phy-nic-mtu', 'ext-port']
@@ -1050,6 +1052,7 @@ def configure_apparmor():
     for profile in profiles:
         context.AppArmorContext(profile).setup_aa_profile()
 
+
 VENDORDATA_FILE = '/etc/nova/vendor_data.json'
 
 
@@ -1062,3 +1065,10 @@ def write_vendordata(vdata):
     with open(VENDORDATA_FILE, 'w') as vdata_file:
         vdata_file.write(json.dumps(json_vdata, sort_keys=True, indent=2))
     return True
+
+
+def get_availability_zone():
+    use_juju_az = config('customize-failure-domain')
+    juju_az = os.environ.get('JUJU_AVAILABILITY_ZONE')
+    return (juju_az if use_juju_az and juju_az
+            else config('default-availability-zone'))

templates/mitaka/neutron.conf

@@ -16,6 +16,7 @@ rpc_response_timeout = {{ rpc_response_timeout }}
 [agent]
 root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
 report_interval = {{ report_interval }}
+{% include "parts/agent" %}
 
 {% include "section-rabbitmq-oslo" %}
 

templates/parts/agent

@@ -0,0 +1,3 @@
+{% if availability_zone -%}
+availability_zone = {{ availability_zone }}
+{% endif -%}

unit_tests/test_neutron_contexts.py

@@ -147,11 +147,12 @@ class TestNeutronGatewayContext(CharmTestCase):
         self.config.side_effect = self.test_config.get
         self.maxDiff = None
 
+    @patch('neutron_utils.config')
     @patch('charmhelpers.contrib.openstack.context.relation_get')
     @patch('charmhelpers.contrib.openstack.context.related_units')
     @patch('charmhelpers.contrib.openstack.context.relation_ids')
     @patch.object(neutron_contexts, 'get_shared_secret')
-    def test_all(self, _secret, _rids, _runits, _rget):
+    def test_all(self, _secret, _rids, _runits, _rget, mock_config):
         rdata = {'l2-population': 'True',
                  'enable-dvr': 'True',
                  'overlay-network-type': 'gre',
@@ -169,6 +170,14 @@ class TestNeutronGatewayContext(CharmTestCase):
         self.test_config.set('vlan-ranges',
                              'physnet1:1000:2000 physnet2:2001:3000')
         self.test_config.set('flat-network-providers', 'physnet3 physnet4')
+
+        def config_side_effect(key):
+            return {
+                'customize-failure-domain': False,
+                'default-availability-zone': 'nova',
+            }[key]
+        mock_config.side_effect = config_side_effect
+
         self.network_get_primary_address.side_effect = NotImplementedError
         self.unit_get.return_value = '10.5.0.1'
         # Provided by neutron-api relation
@@ -204,14 +213,17 @@ class TestNeutronGatewayContext(CharmTestCase):
             'dnsmasq_flags': {
                 'dhcp-userclass': 'set:ipxe,iPXE',
                 'dhcp-match': 'set:ipxe,175'
-            }
+            },
+            'availability_zone': 'nova',
         })
 
+    @patch('neutron_utils.config')
     @patch('charmhelpers.contrib.openstack.context.relation_get')
     @patch('charmhelpers.contrib.openstack.context.related_units')
     @patch('charmhelpers.contrib.openstack.context.relation_ids')
     @patch.object(neutron_contexts, 'get_shared_secret')
-    def test_all_network_spaces(self, _secret, _rids, _runits, _rget):
+    def test_all_network_spaces(self, _secret, _rids, _runits, _rget,
+                                mock_config):
         rdata = {'l2-population': 'True',
                  'enable-dvr': 'True',
                  'overlay-network-type': 'gre',
@@ -228,6 +240,14 @@ class TestNeutronGatewayContext(CharmTestCase):
         self.test_config.set('vlan-ranges',
                              'physnet1:1000:2000 physnet2:2001:3000')
         self.test_config.set('flat-network-providers', 'physnet3 physnet4')
+
+        def config_side_effect(key):
+            return {
+                'customize-failure-domain': False,
+                'default-availability-zone': 'nova',
+            }[key]
+        mock_config.side_effect = config_side_effect
+
         self.network_get_primary_address.return_value = '192.168.20.2'
         self.unit_get.return_value = '10.5.0.1'
         # Provided by neutron-api relation
@@ -263,7 +283,8 @@ class TestNeutronGatewayContext(CharmTestCase):
             'dnsmasq_flags': {
                 'dhcp-userclass': 'set:ipxe,iPXE',
                 'dhcp-match': 'set:ipxe,175'
-            }
+            },
+            'availability_zone': 'nova',
         })
 
     @patch('charmhelpers.contrib.openstack.context.relation_get')
@@ -293,6 +314,84 @@ class TestNeutronGatewayContext(CharmTestCase):
         self.assertTrue(ctxt['enable_isolated_metadata'])
         self.assertTrue(ctxt['enable_metadata_network'])
 
+    @patch('neutron_utils.config')
+    @patch('os.environ.get')
+    @patch('charmhelpers.contrib.openstack.context.relation_get')
+    @patch('charmhelpers.contrib.openstack.context.related_units')
+    @patch('charmhelpers.contrib.openstack.context.relation_ids')
+    @patch.object(neutron_contexts, 'get_shared_secret')
+    def test_availability_zone_no_juju_with_env(self, _secret, _rids,
+                                                _runits, _rget,
+                                                mock_get,
+                                                mock_config):
+        def environ_get_side_effect(key):
+            return {
+                'JUJU_AVAILABILITY_ZONE': 'az1',
+                'PATH': 'foobar',
+            }[key]
+        mock_get.side_effect = environ_get_side_effect
+
+        def config_side_effect(key):
+            return {
+                'customize-failure-domain': False,
+                'default-availability-zone': 'nova',
+            }[key]
+
+        mock_config.side_effect = config_side_effect
+        context = neutron_contexts.NeutronGatewayContext()
+        self.assertEqual(
+            'nova', context()['availability_zone'])
+
+    @patch('neutron_utils.config')
+    @patch('os.environ.get')
+    @patch('charmhelpers.contrib.openstack.context.relation_get')
+    @patch('charmhelpers.contrib.openstack.context.related_units')
+    @patch('charmhelpers.contrib.openstack.context.relation_ids')
+    @patch.object(neutron_contexts, 'get_shared_secret')
+    def test_availability_zone_no_juju_no_env(self, _secret, _rids,
+                                              _runits, _rget,
+                                              mock_get, mock_config):
+        def environ_get_side_effect(key):
+            return {
+                'JUJU_AVAILABILITY_ZONE': '',
+                'PATH': 'foobar',
+            }[key]
+        mock_get.side_effect = environ_get_side_effect
+
+        def config_side_effect(key):
+            return {
+                'customize-failure-domain': False,
+                'default-availability-zone': 'nova',
+            }[key]
+
+        mock_config.side_effect = config_side_effect
+        context = neutron_contexts.NeutronGatewayContext()
+
+        self.assertEqual(
+            'nova', context()['availability_zone'])
+
+    @patch('neutron_utils.config')
+    @patch('os.environ.get')
+    @patch('charmhelpers.contrib.openstack.context.relation_get')
+    @patch('charmhelpers.contrib.openstack.context.related_units')
+    @patch('charmhelpers.contrib.openstack.context.relation_ids')
+    @patch.object(neutron_contexts, 'get_shared_secret')
+    def test_availability_zone_juju(self, _secret, _rids,
+                                    _runits, _rget,
+                                    mock_get, mock_config):
+        def environ_get_side_effect(key):
+            return {
+                'JUJU_AVAILABILITY_ZONE': 'az1',
+                'PATH': 'foobar',
+            }[key]
+        mock_get.side_effect = environ_get_side_effect
+
+        mock_config.side_effect = self.test_config.get
+        self.test_config.set('customize-failure-domain', True)
+        context = neutron_contexts.NeutronGatewayContext()
+        self.assertEqual(
+            'az1', context()['availability_zone'])
+
 
 class TestSharedSecret(CharmTestCase):
 

unit_tests/test_neutron_utils.py

@@ -865,6 +865,62 @@ class TestNeutronUtils(CharmTestCase):
         with patch_open() as (_open, _file):
             self.assertEqual(neutron_utils.write_vendordata(_jdata), False)
 
+    @patch.object(neutron_utils.os.environ, 'get')
+    def test_get_az_customize_with_env(self, os_environ_get_mock):
+        self.config.side_effect = self.test_config.get
+        self.test_config.set('customize-failure-domain', True)
+        self.test_config.set('default-availability-zone', 'nova')
+
+        def os_environ_get_side_effect(key):
+            return {
+                'JUJU_AVAILABILITY_ZONE': 'az1',
+            }[key]
+        os_environ_get_mock.side_effect = os_environ_get_side_effect
+        az = neutron_utils.get_availability_zone()
+        self.assertEqual('az1', az)
+
+    @patch.object(neutron_utils.os.environ, 'get')
+    def test_get_az_customize_without_env(self, os_environ_get_mock):
+        self.config.side_effect = self.test_config.get
+        self.test_config.set('customize-failure-domain', True)
+        self.test_config.set('default-availability-zone', 'mynova')
+
+        def os_environ_get_side_effect(key):
+            return {
+                'JUJU_AVAILABILITY_ZONE': '',
+            }[key]
+        os_environ_get_mock.side_effect = os_environ_get_side_effect
+        az = neutron_utils.get_availability_zone()
+        self.assertEqual('mynova', az)
+
+    @patch.object(neutron_utils.os.environ, 'get')
+    def test_get_az_no_customize_without_env(self, os_environ_get_mock):
+        self.config.side_effect = self.test_config.get
+        self.test_config.set('customize-failure-domain', False)
+        self.test_config.set('default-availability-zone', 'nova')
+
+        def os_environ_get_side_effect(key):
+            return {
+                'JUJU_AVAILABILITY_ZONE': '',
+            }[key]
+        os_environ_get_mock.side_effect = os_environ_get_side_effect
+        az = neutron_utils.get_availability_zone()
+        self.assertEqual('nova', az)
+
+    @patch.object(neutron_utils.os.environ, 'get')
+    def test_get_az_no_customize_with_env(self, os_environ_get_mock):
+        self.config.side_effect = self.test_config.get
+        self.test_config.set('customize-failure-domain', False)
+        self.test_config.set('default-availability-zone', 'nova')
+
+        def os_environ_get_side_effect(key):
+            return {
+                'JUJU_AVAILABILITY_ZONE': 'az1',
+            }[key]
+        os_environ_get_mock.side_effect = os_environ_get_side_effect
+        az = neutron_utils.get_availability_zone()
+        self.assertEqual('nova', az)
+
 network_context = {
     'service_username': 'foo',
     'service_password': 'bar',
@@ -893,6 +949,7 @@ class DummyExternalPortContext():
     def __call__(self):
         return self.return_value
 
+
 cluster1 = ['cluster1-machine1.internal']
 cluster2 = ['cluster2-machine1.internal', 'cluster2-machine2.internal'
             'cluster2-machine3.internal']