Merge "Add support for FWaaS v2 logging"

config.yaml

@@ -308,3 +308,29 @@ options:
       for Neutron agents (DHCP and L3 agents). This option overrides the
       default-availability-zone charm config setting only when the Juju
       provider sets JUJU_AVAILABILITY_ZONE.
+  firewall-group-log-output-base:
+    type: string
+    default:
+    description: |
+      This option allows setting a path for Firewall Group logs.
+      A valid file system path must be provided. If this option is not
+      provided Neutron will use syslog as a destination.
+      (Available from Stein)
+  firewall-group-log-rate-limit:
+    type: int
+    default:
+    description: |
+      Log entries are queued for writing to a log file when a packet rate
+      exceeds the limit set by this option.
+      Possible values: null (no rate limitation), integer values greater than 100.
+      WARNING: Should be NOT LESS than 100, if set (if null logging will not be
+      rate limited).
+      (Available from Stein)
+  firewall-group-log-burst-limit:
+    type: int
+    default: 25
+    description: |
+      This option sets the maximum queue size for log entries.
+      Can be used to avoid excessive memory consumption.
+      WARNING: Should be NOT LESS than 25.
+      (Available from Stein)

hooks/charmhelpers/contrib/network/ovs/__init__.py

@@ -217,19 +217,35 @@ def full_restart():
         service('force-reload-kmod', 'openvswitch-switch')
 
 
-def enable_ipfix(bridge, target):
+def enable_ipfix(bridge, target,
-    '''Enable IPfix on bridge to target.
+                 cache_active_timeout=60,
+                 cache_max_flows=128,
+                 sampling=64):
+    '''Enable IPFIX on bridge to target.
     :param bridge: Bridge to monitor
-    :param target: IPfix remote endpoint
+    :param target: IPFIX remote endpoint
+    :param cache_active_timeout: The maximum period in seconds for
+                                 which an IPFIX flow record is cached
+                                 and aggregated before being sent
+    :param cache_max_flows: The maximum number of IPFIX flow records
+                            that can be cached at a time
+    :param sampling: The rate at which packets should be sampled and
+                     sent to each target collector
     '''
-    cmd = ['ovs-vsctl', 'set', 'Bridge', bridge, 'ipfix=@i', '--',
+    cmd = [
-           '--id=@i', 'create', 'IPFIX', 'targets="{}"'.format(target)]
+        'ovs-vsctl', 'set', 'Bridge', bridge, 'ipfix=@i', '--',
+        '--id=@i', 'create', 'IPFIX',
+        'targets="{}"'.format(target),
+        'sampling={}'.format(sampling),
+        'cache_active_timeout={}'.format(cache_active_timeout),
+        'cache_max_flows={}'.format(cache_max_flows),
+    ]
     log('Enabling IPfix on {}.'.format(bridge))
     subprocess.check_call(cmd)
 
 
 def disable_ipfix(bridge):
-    '''Diable IPfix on target bridge.
+    '''Diable IPFIX on target bridge.
     :param bridge: Bridge to modify
     '''
     cmd = ['ovs-vsctl', 'clear', 'Bridge', bridge, 'ipfix']

hooks/charmhelpers/contrib/openstack/audits/openstack_security_guide.py

@@ -126,7 +126,11 @@ def _config_ini(path):
     :returns: Configuration contained in path
     :rtype: Dict
     """
-    conf = configparser.ConfigParser()
+    # When strict is enabled, duplicate options are not allowed in the
+    # parsed INI; however, Oslo allows duplicate values. This change
+    # causes us to ignore the duplicate values which is acceptable as
+    # long as we don't validate any multi-value options
+    conf = configparser.ConfigParser(strict=False)
     conf.read(path)
     return dict(conf)
 
@@ -204,7 +208,7 @@ def validate_file_ownership(config):
                     "Invalid ownership configuration: {}".format(key))
         owner = options.get('owner', config.get('owner', 'root'))
         group = options.get('group', config.get('group', 'root'))
-        optional = options.get('optional', config.get('optional', 'False'))
+        optional = options.get('optional', config.get('optional', False))
         if '*' in file_name:
             for file in glob.glob(file_name):
                 if file not in files.keys():
@@ -226,7 +230,7 @@ def validate_file_permissions(config):
                 raise RuntimeError(
                     "Invalid ownership configuration: {}".format(key))
         mode = options.get('mode', config.get('permissions', '600'))
-        optional = options.get('optional', config.get('optional', 'False'))
+        optional = options.get('optional', config.get('optional', False))
         if '*' in file_name:
             for file in glob.glob(file_name):
                 if file not in files.keys():

hooks/charmhelpers/contrib/openstack/cert_utils.py

@@ -106,9 +106,11 @@ class CertRequest(object):
             sans = sorted(list(set(entry['addresses'])))
             request[entry['cn']] = {'sans': sans}
         if self.json_encode:
-            return {'cert_requests': json.dumps(request, sort_keys=True)}
+            req = {'cert_requests': json.dumps(request, sort_keys=True)}
         else:
-            return {'cert_requests': request}
+            req = {'cert_requests': request}
+        req['unit_name'] = local_unit().replace('/', '_')
+        return req
 
 
 def get_certificate_request(json_encode=True):

hooks/charmhelpers/contrib/openstack/context.py

@@ -1710,6 +1710,10 @@ class NeutronAPIContext(OSContextGenerator):
                 'rel_key': 'enable-nsg-logging',
                 'default': False,
             },
+            'enable_nfg_logging': {
+                'rel_key': 'enable-nfg-logging',
+                'default': False,
+            },
             'global_physnet_mtu': {
                 'rel_key': 'global-physnet-mtu',
                 'default': 1500,

hooks/charmhelpers/contrib/storage/linux/utils.py

@@ -110,17 +110,19 @@ def is_device_mounted(device):
     return bool(re.search(r'MOUNTPOINT=".+"', out))
 
 
-def mkfs_xfs(device, force=False):
+def mkfs_xfs(device, force=False, inode_size=1024):
     """Format device with XFS filesystem.
 
     By default this should fail if the device already has a filesystem on it.
     :param device: Full path to device to format
     :ptype device: tr
     :param force: Force operation
-    :ptype: force: boolean"""
+    :ptype: force: boolean
+    :param inode_size: XFS inode size in bytes
+    :ptype inode_size: int"""
     cmd = ['mkfs.xfs']
     if force:
         cmd.append("-f")
 
-    cmd += ['-i', 'size=1024', device]
+    cmd += ['-i', "size={}".format(inode_size), device]
     check_call(cmd)

hooks/neutron_contexts.py

@@ -8,6 +8,7 @@ from charmhelpers.core.hookenv import (
     related_units,
     unit_get,
     network_get_primary_address,
+    log,
 )
 from charmhelpers.contrib.openstack.context import (
     OSContextGenerator,
@@ -49,6 +50,9 @@ CORE_PLUGIN = {
     OVS_ODL: NEUTRON_OVS_ODL_PLUGIN,
 }
 
+NFG_LOG_RATE_LIMIT_MIN = 100
+NFG_LOG_BURST_LIMIT_MIN = 25
+
 
 def _get_availability_zone():
     from neutron_utils import get_availability_zone as get_az
@@ -107,6 +111,33 @@ class L3AgentContext(OSContextGenerator):
         return ctxt
 
 
+def validate_nfg_log_path(desired_nfg_log_path):
+    if not desired_nfg_log_path:
+        # None means "we need to use syslog" - no need
+        # to check anything on filesystem
+        return None
+
+    dst_dir, _ = os.path.split(desired_nfg_log_path)
+    path_exists = os.path.exists(dst_dir)
+    if not path_exists:
+        log(
+            "Desired NFG log directory {} not exists! "
+            "falling back to syslog".format(dst_dir),
+            "WARN"
+        )
+        return None
+
+    if path_exists and os.path.isdir(desired_nfg_log_path):
+        log(
+            "Desired NFG log path {} should be file, not directory! "
+            "falling back to syslog".format(desired_nfg_log_path),
+            "WARN"
+        )
+        return None
+
+    return desired_nfg_log_path
+
+
 class NeutronGatewayContext(NeutronAPIContext):
 
     def __call__(self):
@@ -131,6 +162,7 @@ class NeutronGatewayContext(NeutronAPIContext):
             'enable_metadata_network': config('enable-metadata-network'),
             'enable_isolated_metadata': config('enable-isolated-metadata'),
             'availability_zone': _get_availability_zone(),
+            'enable_nfg_logging': api_settings['enable_nfg_logging'],
         }
 
         ctxt['local_ip'] = get_local_ip()
@@ -161,6 +193,26 @@ class NeutronGatewayContext(NeutronAPIContext):
             ctxt['enable_metadata_network'] = True
             ctxt['enable_isolated_metadata'] = True
 
+        ctxt['nfg_log_output_base'] = validate_nfg_log_path(
+            config('firewall-group-log-output-base')
+        )
+        ctxt['nfg_log_rate_limit'] = config(
+            'firewall-group-log-rate-limit'
+        )
+        if ctxt['nfg_log_rate_limit'] is not None:
+            ctxt['nfg_log_rate_limit'] = max(
+                ctxt['nfg_log_rate_limit'],
+                NFG_LOG_RATE_LIMIT_MIN
+            )
+        ctxt['nfg_log_burst_limit'] = config(
+            'firewall-group-log-burst-limit'
+        )
+        if ctxt['nfg_log_burst_limit'] is not None:
+            ctxt['nfg_log_burst_limit'] = max(
+                ctxt['nfg_log_burst_limit'],
+                NFG_LOG_BURST_LIMIT_MIN
+            )
+
         return ctxt
 
 

hooks/neutron_utils.py

@@ -155,6 +155,7 @@ GATEWAY_PKGS = {
         "nova-api-metadata",
         "neutron-metering-agent",
         "neutron-lbaas-agent",
+        "libnetfilter-log1",  # fwaas_v2_log
     ],
     NSX: [
         "neutron-dhcp-agent",
@@ -197,6 +198,7 @@ PY3_PACKAGES = [
     'python3-neutron',
     'python3-neutron-fwaas',
     'python3-neutron-lbaas',
+    'python3-zmq',  # fwaas_v2_log
 ]
 
 EARLY_PACKAGES = {

templates/stein/l3_agent.ini

@@ -30,4 +30,16 @@ ha_vrrp_health_check_interval = 30
 {% endif -%}
 
 [AGENT]
+{% if enable_nfg_logging -%}
+extensions = fwaas_v2,fwaas_v2_log
+[network_log]
+{% if nfg_log_rate_limit -%}
+rate_limit = {{ nfg_log_rate_limit }}
+{% endif -%}
+burst_limit = {{ nfg_log_burst_limit }}
+{% if nfg_log_output_base -%}
+local_output_log_base = {{ nfg_log_output_base }}
+{% endif -%}
+{% else %}
 extensions = fwaas_v2
+{% endif -%}

unit_tests/test_neutron_contexts.py

@@ -156,6 +156,7 @@ class TestNeutronGatewayContext(CharmTestCase):
         self.config.side_effect = self.test_config.get
         self.maxDiff = None
 
+    @patch.object(neutron_contexts, 'validate_nfg_log_path', lambda x: x)
     @patch('neutron_utils.config')
     @patch('charmhelpers.contrib.openstack.context.relation_get')
     @patch('charmhelpers.contrib.openstack.context.related_units')
@@ -168,7 +169,8 @@ class TestNeutronGatewayContext(CharmTestCase):
                  'enable-l3ha': 'True',
                  'enable-qos': 'True',
                  'network-device-mtu': 9000,
-                 'dns-domain': 'openstack.example.'}
+                 'dns-domain': 'openstack.example.',
+                 'enable-nfg-logging': 'True'}
         self.test_config.set('plugin', 'ovs')
         self.test_config.set('debug', False)
         self.test_config.set('verbose', True)
@@ -179,6 +181,10 @@ class TestNeutronGatewayContext(CharmTestCase):
         self.test_config.set('vlan-ranges',
                              'physnet1:1000:2000 physnet2:2001:3000')
         self.test_config.set('flat-network-providers', 'physnet3 physnet4')
+        self.test_config.set('firewall-group-log-output-base',
+                             '/var/log/firewall-logs')
+        self.test_config.set('firewall-group-log-rate-limit', 100)
+        self.test_config.set('firewall-group-log-burst-limit', 50)
 
         def config_side_effect(key):
             return {
@@ -224,8 +230,13 @@ class TestNeutronGatewayContext(CharmTestCase):
                 'dhcp-match': 'set:ipxe,175'
             },
             'availability_zone': 'nova',
+            'enable_nfg_logging': True,
+            'nfg_log_burst_limit': 50,
+            'nfg_log_output_base': '/var/log/firewall-logs',
+            'nfg_log_rate_limit': 100,
         })
 
+    @patch.object(neutron_contexts, 'validate_nfg_log_path', lambda x: x)
     @patch('neutron_utils.config')
     @patch('charmhelpers.contrib.openstack.context.relation_get')
     @patch('charmhelpers.contrib.openstack.context.related_units')
@@ -294,6 +305,10 @@ class TestNeutronGatewayContext(CharmTestCase):
                 'dhcp-match': 'set:ipxe,175'
             },
             'availability_zone': 'nova',
+            'enable_nfg_logging': False,
+            'nfg_log_burst_limit': 25,
+            'nfg_log_output_base': None,
+            'nfg_log_rate_limit': None,
         })
 
     @patch('charmhelpers.contrib.openstack.context.relation_get')
@@ -401,6 +416,19 @@ class TestNeutronGatewayContext(CharmTestCase):
         self.assertEqual(
             'az1', context()['availability_zone'])
 
+    @patch('charmhelpers.contrib.openstack.context.relation_get')
+    @patch('charmhelpers.contrib.openstack.context.related_units')
+    @patch('charmhelpers.contrib.openstack.context.relation_ids')
+    @patch.object(neutron_contexts, 'get_shared_secret')
+    def test_nfg_min_settings(self, _secret, _rids, _runits, _rget):
+        self.test_config.set('firewall-group-log-rate-limit', 90)
+        self.test_config.set('firewall-group-log-burst-limit', 20)
+        self.network_get_primary_address.return_value = '192.168.20.2'
+        self.unit_get.return_value = '10.5.0.1'
+        ctxt = neutron_contexts.NeutronGatewayContext()()
+        self.assertEqual(ctxt['nfg_log_burst_limit'], 25)
+        self.assertEqual(ctxt['nfg_log_rate_limit'], 100)
+
 
 class TestSharedSecret(CharmTestCase):