Add keystone audit middleware API logging

This commit adds Keystone audit middleware API logging to
the Neutron-API charm in versions Yoga and newer to allow users to
configure their environment for CADF compliance. This feature can
be enabled/disabled and is set to 'disabled' by default to avoid
bloat in log files. The logging output writes to
/var/log/neutron/neutron-server.log.
This commit builds on previous discussions:
https://github.com/juju/charm-helpers/pull/808.

Closes-Bug: 1856555
Change-Id: I6d4f471607b11c2a0450d74c8fd68385486ec8d9

config.yaml

@@ -12,6 +12,11 @@ options:
     default: False
     description: |
       Setting this to True will allow supporting services to log to syslog.
+  audit-middleware:
+    type: boolean
+    default: False
+    description: |
+      Enable Keystone auditing middleware for logging API calls.
   enable-security-group-logging:
     type: boolean
     default: False

hooks/neutron_api_utils.py

@@ -144,11 +144,13 @@ API_PORTS = {
     'neutron-server': 9696,
 }
 
+SERVICE = 'neutron'
 NEUTRON_CONF_DIR = "/etc/neutron"
 
 NEUTRON_CONF = '%s/neutron.conf' % NEUTRON_CONF_DIR
 NEUTRON_LBAAS_CONF = '%s/neutron_lbaas.conf' % NEUTRON_CONF_DIR
 NEUTRON_VPNAAS_CONF = '%s/neutron_vpnaas.conf' % NEUTRON_CONF_DIR
+NEUTRON_AUDIT_CONF = '%s/api_audit_map.conf' % NEUTRON_CONF_DIR
 HAPROXY_CONF = '/etc/haproxy/haproxy.cfg'
 APACHE_PORTS_CONF = '/etc/apache2/ports.conf'
 APACHE_CONF = '/etc/apache2/sites-available/openstack_https_frontend'
@@ -187,7 +189,8 @@ BASE_RESOURCE_MAP = OrderedDict([
                      context.InternalEndpointContext(),
                      context.MemcacheContext(),
                      neutron_api_context.DesignateContext(),
-                     neutron_api_context.NeutronInfobloxContext()],
+                     neutron_api_context.NeutronInfobloxContext(),
+                     context.KeystoneAuditMiddleware(service=SERVICE)],
     }),
     (NEUTRON_DEFAULT, {
         'services': ['neutron-server'],
@@ -195,7 +198,12 @@ BASE_RESOURCE_MAP = OrderedDict([
     }),
     (API_PASTE_INI, {
         'services': ['neutron-server'],
-        'contexts': [neutron_api_context.NeutronApiApiPasteContext()],
+        'contexts': [neutron_api_context.NeutronApiApiPasteContext(),
+                     context.KeystoneAuditMiddleware(service=SERVICE)],
+    }),
+    (NEUTRON_AUDIT_CONF, {
+        'services': ['apache2'],
+        'contexts': [context.KeystoneAuditMiddleware(service=SERVICE)]
     }),
     (APACHE_CONF, {
         'contexts': [neutron_api_context.ApacheSSLContext()],

templates/yoga/api-paste.ini

@@ -0,0 +1,64 @@
+[composite:neutron]
+use = egg:Paste#urlmap
+/: neutronversions_composite
+/healthcheck: healthcheck
+/v2.0: neutronapi_v2_0
+
+[composite:neutronapi_v2_0]
+use = call:neutron.auth:pipeline_factory
+noauth = cors http_proxy_to_wsgi request_id catch_errors extensions neutronapiapp_v2_0
+{% if audit_middleware and service_name -%}
+keystone = {% for m in extra_middleware %}{{ m.name }} {% endfor %}cors http_proxy_to_wsgi request_id catch_errors authtoken keystonecontext audit extensions neutronapiapp_v2_0
+{% else -%}
+keystone = {% for m in extra_middleware %}{{ m.name }} {% endfor %}cors http_proxy_to_wsgi request_id catch_errors authtoken keystonecontext extensions neutronapiapp_v2_0
+{% endif %}
+
+{% for m in extra_middleware -%}
+[{{ m.type }}:{{ m.name }}]
+{% for k, v in m.config.items() -%}
+{{ k }} = {{ v }}
+{% endfor %}
+{% endfor -%}
+
+[composite:neutronversions_composite]
+use = call:neutron.auth:pipeline_factory
+noauth = cors http_proxy_to_wsgi neutronversions
+keystone = cors http_proxy_to_wsgi neutronversions
+
+[filter:request_id]
+paste.filter_factory = oslo_middleware:RequestId.factory
+
+[filter:catch_errors]
+paste.filter_factory = oslo_middleware:CatchErrors.factory
+
+[filter:cors]
+paste.filter_factory = oslo_middleware.cors:filter_factory
+oslo_config_project = neutron
+
+[filter:http_proxy_to_wsgi]
+paste.filter_factory = oslo_middleware.http_proxy_to_wsgi:HTTPProxyToWSGI.factory
+
+[filter:keystonecontext]
+paste.filter_factory = neutron.auth:NeutronKeystoneContext.factory
+
+[filter:authtoken]
+paste.filter_factory = keystonemiddleware.auth_token:filter_factory
+
+[filter:extensions]
+paste.filter_factory = neutron.api.extensions:plugin_aware_extension_middleware_factory
+
+[app:neutronversions]
+paste.app_factory = neutron.pecan_wsgi.app:versions_factory
+
+[app:neutronapiapp_v2_0]
+paste.app_factory = neutron.api.v2.router:APIRouter.factory
+
+[filter:osprofiler]
+paste.filter_factory = osprofiler.web:WsgiMiddleware.factory
+
+[app:healthcheck]
+paste.app_factory = oslo_middleware:Healthcheck.app_factory
+backends = disable_by_file
+disable_by_file_path = /var/lib/neutron/healthcheck_disable
+
+{% include "section-filter-audit" %}

templates/yoga/api_audit_map.conf

@@ -0,0 +1,31 @@
+[DEFAULT]
+# default target endpoint type
+# should match the endpoint type defined in service catalog
+target_endpoint_type = None
+
+[custom_actions]
+add_router_interface = update/add
+remove_router_interface = update/remove
+
+# possible end path of api requests
+[path_keywords]
+floatingips = ip
+healthmonitors = healthmonitor
+health_monitors = health_monitor
+lb = None
+members = member
+metering-labels = label
+metering-label-rules = rule
+networks = network
+pools = pool
+ports = port
+routers = router
+quotas = quota
+security-groups = security-group
+security-group-rules = rule
+subnets = subnet
+vips = vip
+
+# map endpoint type defined in service catalog to CADF typeURI
+[service_endpoints]
+network = service/network

templates/yoga/neutron.conf

@@ -0,0 +1,150 @@
+# ussuri
+###############################################################################
+# [ WARNING ]
+# Configuration file maintained by Juju. Local changes may be overwritten.
+# Restart trigger {{ restart_trigger }}
+###############################################################################
+[DEFAULT]
+verbose = {{ verbose }}
+debug = {{ debug }}
+use_syslog = {{ use_syslog }}
+state_path = /var/lib/neutron
+bind_host = {{ bind_host }}
+auth_strategy = keystone
+api_workers = {{ workers }}
+rpc_workers = {{ workers }}
+
+{% if transport_url %}
+transport_url = {{ transport_url }}
+{% endif %}
+
+router_distributed = {{ enable_dvr }}
+
+{% if dns_domain -%}
+dns_domain = {{ dns_domain }}
+{% endif -%}
+
+l3_ha = {{ l3_ha }}
+{% if l3_ha -%}
+max_l3_agents_per_router = {{ max_l3_agents_per_router }}
+{% endif -%}
+
+allow_automatic_l3agent_failover = {{ allow_automatic_l3agent_failover }}
+allow_automatic_dhcp_failover = {{ allow_automatic_dhcp_failover }}
+{% if network_scheduler_driver -%}
+network_scheduler_driver = {{ network_scheduler_driver }}
+dhcp_load_type = {{ dhcp_load_type }}
+{% endif -%}
+{% if router_scheduler_driver -%}
+router_scheduler_driver = {{ router_scheduler_driver }}
+{% endif -%}
+
+{% if neutron_bind_port -%}
+bind_port = {{ neutron_bind_port }}
+{% else -%}
+bind_port = 9696
+{% endif -%}
+
+{% if core_plugin -%}
+core_plugin = {{ core_plugin }}
+{% if service_plugins -%}
+service_plugins = {{ service_plugins }}
+{% endif -%}
+{% endif -%}
+
+{% if neutron_security_groups -%}
+allow_overlapping_ips = True
+{% endif -%}
+
+dhcp_agents_per_network = {{ dhcp_agents_per_network }}
+
+notify_nova_on_port_status_changes = True
+notify_nova_on_port_data_changes = True
+
+{% if sections and 'DEFAULT' in sections -%}
+{% for key, value in sections['DEFAULT'] -%}
+{{ key }} = {{ value }}
+{% endfor -%}
+{% endif %}
+
+{% if user_config_flags -%}
+{% for key, value in user_config_flags.items() -%}
+{{ key }} = {{ value }}
+{% endfor -%}
+{% endif -%}
+
+{% if global_physnet_mtu -%}
+global_physnet_mtu = {{ global_physnet_mtu }}
+{% endif -%}
+
+{% if enable_designate -%}
+external_dns_driver = designate
+{% endif -%}
+
+{% include "section-zeromq" %}
+
+[quotas]
+{% if quota_driver -%}
+quota_driver = {{ quota_driver }}
+{% else -%}
+quota_driver = neutron.db.quota.driver.DbQuotaDriver
+{% endif -%}
+{% if neutron_security_groups -%}
+quota_items = network,subnet,port,security_group,security_group_rule
+quota_security_group = {{ quota_security_group }}
+quota_security_group_rule = {{ quota_security_group_rule }}
+{% else -%}
+quota_items = network,subnet,port
+{% endif -%}
+quota_network = {{ quota_network }}
+quota_subnet = {{ quota_subnet }}
+quota_port = {{ quota_port }}
+quota_vip = {{ quota_vip }}
+quota_pool = {{ quota_pool }}
+quota_member = {{ quota_member }}
+quota_health_monitors = {{ quota_health_monitors }}
+quota_router = {{ quota_router }}
+quota_floatingip = {{ quota_floatingip }}
+
+[agent]
+root_helper = sudo /usr/bin/neutron-rootwrap /etc/neutron/rootwrap.conf
+
+{% include "section-keystone-authtoken-mitaka" %}
+
+{% include "parts/section-database" %}
+
+{% include "section-oslo-messaging-rabbit" %}
+
+{% include "section-oslo-notifications" %}
+
+[ovs]
+igmp_snooping_enable = {{ enable_igmp_snooping }}
+
+[oslo_concurrency]
+lock_path = $state_path/lock
+
+{% include "parts/section-nova" %}
+
+{% if enable_designate -%}
+{% include "parts/section-designate" %}
+{% endif -%}
+
+{% include "parts/section-placement" %}
+
+{% if firewall_v2 -%}
+[service_providers]
+service_provider = FIREWALL_V2:fwaas_db:neutron_fwaas.services.firewall.service_drivers.agents.agents.FirewallAgentDriver:default
+{% endif %}
+
+{% include "section-oslo-middleware" %}
+
+{% for section in sections -%}
+{% if section != 'DEFAULT' -%}
+[{{ section }}]
+{% for key, value in sections[section] -%}
+{{ key }} = {{ value }}
+{% endfor %}
+{% endif %}
+{%- endfor %}
+
+{% include "section-audit-middleware-notifications" %}

tests/tests.yaml

@@ -13,11 +13,15 @@ tests:
   - zaza.openstack.charm_tests.neutron.tests.NeutronCreateNetworkTest
   - zaza.openstack.charm_tests.neutron.tests.SecurityTest
   - zaza.openstack.charm_tests.policyd.tests.NeutronApiTests
+  - zaza.openstack.charm_tests.audit.tests.KeystoneAuditMiddlewareTest
 
 configure:
   - zaza.openstack.charm_tests.keystone.setup.add_demo_user
 
 tests_options:
+  audit-middleware:
+    service: neutron
+    application: neutron-api
   policyd:
     service: neutron
   force_deploy:

unit_tests/test_neutron_api_utils.py

@@ -281,6 +281,7 @@ class TestNeutronAPIUtils(CharmTestCase):
             (nutils.NEUTRON_CONF, ['neutron-server']),
             (nutils.NEUTRON_DEFAULT, ['neutron-server']),
             (nutils.API_PASTE_INI, ['neutron-server']),
+            (nutils.NEUTRON_AUDIT_CONF, ['apache2']),
             (nutils.APACHE_CONF, ['apache2']),
             (nutils.HAPROXY_CONF, ['haproxy']),
             (nutils.APACHE_PORTS_CONF, ['apache2']),
@@ -302,6 +303,7 @@ class TestNeutronAPIUtils(CharmTestCase):
             (nutils.NEUTRON_CONF, ['neutron-server']),
             (nutils.NEUTRON_DEFAULT, ['neutron-server']),
             (nutils.API_PASTE_INI, ['neutron-server']),
+            (nutils.NEUTRON_AUDIT_CONF, ['apache2']),
             (nutils.APACHE_CONF, ['apache2']),
             (nutils.HAPROXY_CONF, ['haproxy']),
             (nutils.APACHE_PORTS_CONF, ['apache2']),
@@ -332,6 +334,7 @@ class TestNeutronAPIUtils(CharmTestCase):
                  '/etc/neutron/api-paste.ini',
                  '/etc/default/neutron-server',
                  '/etc/neutron/plugins/ml2/ml2_conf.ini',
+                 '/etc/neutron/api_audit_map.conf',
                  '/etc/apache2/ports.conf',
                  '/etc/apache2/sites-available/openstack_https_frontend',
                  '/etc/haproxy/haproxy.cfg']