Enable support for security group logging

Add support to enable logging of security groups for
OpenStack Queens or later; this feature is enabled via
the neutron-api charm, with local configuration options
provided in the neutron-openvswitch charm.

The feature is only compatible with the openvswitch firewall
driver and will not be enabled if this configuration option
is not set in the neutron-openvswitch charm.

This change is removing unnecessary Neutron config
option "neutron_firewall_driver" since FW drivers are
being handled on agents side (not on API server) since
Mitaka release.

Change-Id: Icadb055b2c5c3216b6d086b44a4823595b2baffa
Closes-Bug: #1787397

.gitignore

@@ -11,3 +11,4 @@ xenial/
 .stestr
 __pycache__
 func-results.json
+.idea

config.yaml

@@ -12,6 +12,16 @@ options:
     default: False
     description: |
       Setting this to True will allow supporting services to log to syslog.
+  enable-security-group-logging:
+    type: boolean
+    default: False
+    description: |
+      Setting this to True will enable logging for Security Groups. (Available from Queens)
+      WARNING: Enabling this may affect your disk I/O performance since this
+      will log ALL traffic being passed via NSG. Logging configuration
+      such as thresholds and a destination log file are available in the neutron-openvswitch charm.
+      Also, an neutron-openvswitch charm config option "firewall-driver" should be explicitly
+      set to "openvswitch", since security group logging works only with OVS firewall driver now.
   openstack-origin:
     type: string
     default: distro

hooks/neutron_api_context.py

@@ -218,6 +218,30 @@ def is_qos_requested_and_valid():
         return False
 
 
+def is_nsg_logging_enabled():
+    """
+    Check, if Neutron security groups logging should be enabled.
+    Works only on >=Queens and with OVS native firewall driver:
+    https://docs.openstack.org/neutron/queens/admin/config-logging.html
+    """
+    if config('enable-security-group-logging'):
+        if config('neutron-plugin') != 'ovs':
+            msg = ("Disabling NSG logging; implementation only exists "
+                   "for the OVS ML2 driver")
+            log(msg, ERROR)
+            return False
+
+        if CompareOpenStackReleases(os_release('neutron-server')) < 'queens':
+            msg = ("The enable-security-group-logging option is only "
+                   "supported on Queens or later")
+            log(msg, ERROR)
+            return False
+
+        return True
+
+    return False
+
+
 def is_vlan_trunking_requested_and_valid():
     """Check whether VLAN trunking should be enabled by checking whether
        it has been requested and, if it has, is it supported in the current
@@ -242,7 +266,6 @@ def is_vlan_trunking_requested_and_valid():
                    "later")
             log(msg, ERROR)
             return False
-        print("release >= newton")
 
         return True
     else:
@@ -514,9 +537,13 @@ class NeutronCCContext(context.NeutronContext):
                          ('neutron_dynamic_routing.'
                           'services.bgp.bgp_plugin.BgpPlugin')],
             }
+
             ctxt['service_plugins'] = service_plugins.get(
                 release, service_plugins['pike'])
 
+            if is_nsg_logging_enabled():
+                ctxt['service_plugins'].append('log')
+
             if is_qos_requested_and_valid():
                 ctxt['service_plugins'].append('qos')
 

hooks/neutron_api_hooks.py

@@ -96,6 +96,7 @@ from neutron_api_context import (
     IdentityServiceContext,
     is_qos_requested_and_valid,
     is_vlan_trunking_requested_and_valid,
+    is_nsg_logging_enabled,
     EtcdContext,
 )
 
@@ -466,6 +467,7 @@ def neutron_plugin_api_relation_joined(rid=None):
             'enable-l3ha': get_l3ha(),
             'enable-qos': is_qos_requested_and_valid(),
             'enable-vlan-trunking': is_vlan_trunking_requested_and_valid(),
+            'enable-nsg-logging': is_nsg_logging_enabled(),
             'overlay-network-type': get_overlay_network_type(),
             'addr': unit_get('private-address'),
             'polling-interval': config('polling-interval'),

templates/mitaka/neutron.conf

@@ -41,11 +41,6 @@ service_plugins = {{ service_plugins }}
 
 {% if neutron_security_groups -%}
 allow_overlapping_ips = True
-{% if neutron_plugin == 'Calico' -%}
-neutron_firewall_driver = neutron.agent.linux.iptables_firewall.IptablesFirewallDriver
-{% else -%}
-neutron_firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
-{% endif -%}
 {% endif -%}
 
 dhcp_agents_per_network = {{ dhcp_agents_per_network }}

templates/newton/neutron.conf

@@ -41,11 +41,6 @@ service_plugins = {{ service_plugins }}
 
 {% if neutron_security_groups -%}
 allow_overlapping_ips = True
-{% if neutron_plugin == 'Calico' -%}
-neutron_firewall_driver = neutron.agent.linux.iptables_firewall.IptablesFirewallDriver
-{% else -%}
-neutron_firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
-{% endif -%}
 {% endif -%}
 
 dhcp_agents_per_network = {{ dhcp_agents_per_network }}

templates/ocata/neutron.conf

@@ -41,11 +41,6 @@ service_plugins = {{ service_plugins }}
 
 {% if neutron_security_groups -%}
 allow_overlapping_ips = True
-{% if neutron_plugin == 'Calico' -%}
-neutron_firewall_driver = neutron.agent.linux.iptables_firewall.IptablesFirewallDriver
-{% else -%}
-neutron_firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
-{% endif -%}
 {% endif -%}
 
 dhcp_agents_per_network = {{ dhcp_agents_per_network }}

templates/pike/neutron.conf

@@ -41,11 +41,6 @@ service_plugins = {{ service_plugins }}
 
 {% if neutron_security_groups -%}
 allow_overlapping_ips = True
-{% if neutron_plugin == 'Calico' -%}
-neutron_firewall_driver = neutron.agent.linux.iptables_firewall.IptablesFirewallDriver
-{% else -%}
-neutron_firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
-{% endif -%}
 {% endif -%}
 
 dhcp_agents_per_network = {{ dhcp_agents_per_network }}

templates/queens/ml2_conf.ini

@@ -0,0 +1,61 @@
+# queens
+###############################################################################
+# [ WARNING ]
+# Configuration file maintained by Juju. Local changes may be overwritten.  
+###############################################################################
+[ml2]
+{% if extension_drivers -%}
+extension_drivers={{ extension_drivers }}
+{% endif -%}
+
+{% if neutron_plugin == 'Calico' -%}
+type_drivers = local,flat
+mechanism_drivers = calico
+{% else -%}
+type_drivers = {{ tenant_network_types }}
+tenant_network_types = {{ tenant_network_types }}
+mechanism_drivers = {{ mechanism_drivers }}
+
+{% if physical_network_mtus -%}
+physical_network_mtus = {{ physical_network_mtus }}
+{% endif -%}
+{% if path_mtu -%}
+path_mtu = {{ path_mtu }}
+{% endif -%}
+
+[ml2_type_gre]
+tunnel_id_ranges = 1:1000
+
+[ml2_type_vxlan]
+vni_ranges = {{ vni_ranges }}
+
+[ml2_type_vlan]
+network_vlan_ranges = {{ vlan_ranges }}
+
+[ml2_type_flat]
+flat_networks = {{ network_providers }}
+
+[ovs]
+enable_tunneling = True
+local_ip = {{ local_ip }}
+
+[agent]
+tunnel_types = {{ overlay_network_type }}
+{% if 'log' in service_plugins -%}
+extensions = log
+
+{% endif -%}
+
+{% endif -%}
+
+[securitygroup]
+{% if neutron_security_groups -%}
+enable_security_group = True
+{% else -%}
+enable_security_group = False
+{% endif -%}
+
+{% if supported_pci_vendor_devs %}
+[ml2_sriov]
+supported_pci_vendor_devs = {{ supported_pci_vendor_devs }}
+{% endif -%}

templates/rocky/neutron.conf

@@ -41,11 +41,6 @@ service_plugins = {{ service_plugins }}
 
 {% if neutron_security_groups -%}
 allow_overlapping_ips = True
-{% if neutron_plugin == 'Calico' -%}
-neutron_firewall_driver = neutron.agent.linux.iptables_firewall.IptablesFirewallDriver
-{% else -%}
-neutron_firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
-{% endif -%}
 {% endif -%}
 
 dhcp_agents_per_network = {{ dhcp_agents_per_network }}

unit_tests/test_neutron_api_hooks.py

@@ -90,6 +90,7 @@ TO_PATCH = [
     'status_set',
     'get_relation_ip',
     'update_dns_ha_resource_params',
+    'is_nsg_logging_enabled',
 ]
 NEUTRON_CONF_DIR = "/etc/neutron"
 
@@ -133,6 +134,7 @@ class NeutronAPIHooksTests(CharmTestCase):
         self.test_config.set('openstack-origin', 'distro')
         self.test_config.set('neutron-plugin', 'ovs')
         self.neutron_plugin_attribute.side_effect = _mock_nuage_npa
+        self.is_nsg_logging_enabled.return_value = False
 
     def _fake_relids(self, rel_name):
         return [randrange(100) for _count in range(2)]
@@ -488,6 +490,7 @@ class NeutronAPIHooksTests(CharmTestCase):
             'service_username': None,
             'service_host': None,
             'neutron-api-ready': 'no',
+            'enable-nsg-logging': False,
         }
         self.is_qos_requested_and_valid.return_value = False
         self.is_vlan_trunking_requested_and_valid.return_value = False
@@ -502,6 +505,54 @@ class NeutronAPIHooksTests(CharmTestCase):
             **_relation_data
         )
 
+    def test_neutron_plugin_api_relation_joined_nsg_logging(self):
+        self.unit_get.return_value = '172.18.18.18'
+        self.IdentityServiceContext.return_value = \
+            DummyContext(return_value={})
+        _relation_data = {
+            'neutron-security-groups': False,
+            'enable-dvr': False,
+            'enable-l3ha': False,
+            'enable-qos': False,
+            'enable-vlan-trunking': False,
+            'addr': '172.18.18.18',
+            'polling-interval': 2,
+            'rpc-response-timeout': 60,
+            'report-interval': 30,
+            'l2-population': False,
+            'overlay-network-type': 'vxlan',
+            'service_protocol': None,
+            'auth_protocol': None,
+            'service_tenant': None,
+            'service_port': None,
+            'region': 'RegionOne',
+            'service_password': None,
+            'auth_port': None,
+            'auth_host': None,
+            'service_username': None,
+            'service_host': None,
+            'neutron-api-ready': 'no',
+            'enable-nsg-logging': True,
+        }
+
+        self.is_qos_requested_and_valid.return_value = False
+        self.is_vlan_trunking_requested_and_valid.return_value = False
+        self.get_dvr.return_value = False
+        self.get_l3ha.return_value = False
+        self.get_l2population.return_value = False
+        self.get_overlay_network_type.return_value = 'vxlan'
+        self.get_dns_domain.return_value = ''
+
+        self.test_config.set('enable-security-group-logging', True)
+        self.is_nsg_logging_enabled.return_value = True
+
+        self._call_hook('neutron-plugin-api-relation-joined')
+
+        self.relation_set.assert_called_with(
+            relation_id=None,
+            **_relation_data
+        )
+
     def test_neutron_plugin_api_relation_joined_dvr(self):
         self.unit_get.return_value = '172.18.18.18'
         self.IdentityServiceContext.return_value = \
@@ -529,6 +580,7 @@ class NeutronAPIHooksTests(CharmTestCase):
             'service_username': None,
             'service_host': None,
             'neutron-api-ready': 'no',
+            'enable-nsg-logging': False,
         }
         self.is_qos_requested_and_valid.return_value = False
         self.is_vlan_trunking_requested_and_valid.return_value = False
@@ -570,6 +622,7 @@ class NeutronAPIHooksTests(CharmTestCase):
             'service_username': None,
             'service_host': None,
             'neutron-api-ready': 'no',
+            'enable-nsg-logging': False,
         }
         self.is_qos_requested_and_valid.return_value = False
         self.is_vlan_trunking_requested_and_valid.return_value = False
@@ -613,6 +666,7 @@ class NeutronAPIHooksTests(CharmTestCase):
             'service_username': None,
             'service_host': None,
             'neutron-api-ready': 'no',
+            'enable-nsg-logging': False,
         }
         self.is_qos_requested_and_valid.return_value = False
         self.is_vlan_trunking_requested_and_valid.return_value = False
@@ -654,7 +708,8 @@ class NeutronAPIHooksTests(CharmTestCase):
             'service_username': None,
             'service_host': None,
             'neutron-api-ready': 'no',
-            'dns-domain': 'openstack.example.'
+            'dns-domain': 'openstack.example.',
+            'enable-nsg-logging': False,
         }
         self.is_qos_requested_and_valid.return_value = False
         self.is_vlan_trunking_requested_and_valid.return_value = False