Add auditing support to the Octavia API

This patch adds Cloud Auditing Data Federation (CADF) auditing support to the
Octavia API. This is implemented using the keystonemiddleware audit filter.

Change-Id: I87a7e15171dfaf28b6ed97ca71d4423d18fbdbea

devstack/plugin.sh

@@ -207,6 +207,10 @@ function octavia_configure {
         cp $OCTAVIA_DIR/etc/octavia.conf $OCTAVIA_CONF
     fi
 
+    if ! [ -e $OCTAVIA_AUDIT_MAP ] ; then
+        cp $OCTAVIA_DIR/etc/audit/octavia_api_audit_map.conf.sample $OCTAVIA_AUDIT_MAP
+    fi
+
     # Use devstack logging configuration
     setup_logging $OCTAVIA_CONF
 

devstack/settings

@@ -14,6 +14,7 @@ OCTAVIA_CERTS_DIR=${OCTAVIA_CERTS_DIR:-${OCTAVIA_CONF_DIR}/certs}
 OCTAVIA_DHCLIENT_DIR=${OCTAVIA_DHCLIENT_DIR:-"/etc/dhcp/octavia"}
 OCTAVIA_DHCLIENT_CONF=${OCTAVIA_DHCLIENT_CONF:-${OCTAVIA_DHCLIENT_DIR}/dhclient.conf}
 OCTAVIA_CONF=${OCTAVIA_CONF:-${OCTAVIA_CONF_DIR}/octavia.conf}
+OCTAVIA_AUDIT_MAP=${OCTAVIA_AUDIT_MAP:-${OCTAVIA_CONF_DIR}/octavia_api_audit_map.conf}
 OCTAVIA_TEMPEST_DIR=${OCTAVIA_TEMPEST_DIR:-${OCTAVIA_DIR}/octavia/tests/tempest}
 
 OCTAVIA_AMPHORA_DRIVER=${OCTAVIA_AMPHORA_DRIVER:-"amphora_haproxy_rest_driver"}

doc/source/admin/api-audit.rst

@@ -0,0 +1,194 @@
+
+====================
+Octavia API Auditing
+====================
+
+The `keystonemiddleware audit middleware`_ supports delivery of Cloud Auditing
+Data Federation (CADF) audit events via Oslo messaging notifier capability.
+Based on `notification_driver` configuration, audit events can be routed to
+messaging infrastructure (notification_driver = messagingv2) or can be routed
+to a log file (notification_driver = log).
+
+More information about the CADF format can be found on the `DMTF Cloud Auditing Data Federation website <https://www.dmtf.org/standards/cadf>`_.
+
+Audit middleware creates two events per REST API interaction. First event has
+information extracted from request data and the second one has request outcome
+(response).
+
+.. _keystonemiddleware audit middleware: https://docs.openstack.org/keystonemiddleware/latest/audit.html
+
+Configuring Octavia API Auditing
+================================
+
+Auditing can be enabled by making the following changes to the Octavia
+configuration file on your Octavia API instance(s).
+
+#. Enable auditing::
+
+    [audit]
+    ...
+    enabled = True
+
+#. Optionally specify the location of the audit map file::
+
+    [audit]
+    ...
+    audit_map_file = /etc/octavia/octavia_api_audit_map.conf
+
+   The default audit map file location is /etc/octavia/octavia_api_audit_map.conf.
+
+#. Copy the audit map file from the octavia/etc/audit directory to the
+   location specified in the previous step. A sample file has been provided
+   in octavia/etc/audit/octavia_api_audit_map.conf.sample.
+
+#. Optionally specify the REST HTTP methods you do not want to audit::
+
+    [audit]
+    ...
+    ignore_req_list =
+
+#. Specify the driver to use for sending the audit notifications::
+
+    [audit_middleware_notifications]
+    ...
+    driver = log
+
+   Driver options are: messaging, messagingv2, routing, log, noop
+
+#. Optionally specify the messaging topic::
+
+    [audit_middleware_notifications]
+    ...
+    topics =
+
+#. Optionally specify the messaging transport URL::
+
+    [audit_middleware_notifications]
+    ...
+    transport_url =
+
+#. Restart your Octavia API processes.
+
+Sampe Audit Events
+==================
+
+Request
+-------
+
+.. code-block:: json
+
+  {
+    "event_type": "audit.http.request",
+    "timestamp": "2018-10-11 22:42:22.721025",
+    "payload": {
+      "typeURI": "http://schemas.dmtf.org/cloud/audit/1.0/event",
+      "eventTime": "2018-10-11T22:42:22.720112+0000",
+      "target": {
+        "id": "octavia",
+        "typeURI": "service/load-balancer/loadbalancers",
+        "addresses": [{
+          "url": "http://10.21.21.53/load-balancer",
+          "name": "admin"
+        }, {
+          "url": "http://10.21.21.53/load-balancer",
+          "name": "private"
+        }, {
+          "url": "http://10.21.21.53/load-balancer",
+          "name": "public"
+        }],
+        "name": "octavia"
+      },
+      "observer": {
+        "id": "target"
+      },
+      "tags": ["correlation_id?value=e5b34bc3-4837-54fa-9892-8e65a9a2e73a"],
+      "eventType": "activity",
+      "initiator": {
+        "typeURI": "service/security/account/user",
+        "name": "admin",
+        "credential": {
+          "token": "***",
+          "identity_status": "Confirmed"
+        },
+        "host": {
+          "agent": "openstacksdk/0.17.2 keystoneauth1/3.11.0 python-requests/2.19.1 CPython/2.7.12",
+          "address": "10.21.21.53"
+        },
+        "project_id": "90168d185e504b5580884a235ba31612",
+        "id": "2af901396a424d5ca9dffa725226e8c7"
+      },
+      "action": "read/list",
+      "outcome": "pending",
+      "id": "8cf14af5-246e-5739-a11e-513ca13b7d36",
+      "requestPath": "/load-balancer/v2.0/lbaas/loadbalancers"
+    },
+    "priority": "INFO",
+    "publisher_id": "uwsgi",
+    "message_id": "63264e0e-e60f-4adc-a656-0d87ab5d6329"
+  }
+
+Response
+--------
+
+.. code-block:: json
+
+  {
+    "event_type": "audit.http.response",
+    "timestamp": "2018-10-11 22:42:22.853129",
+    "payload": {
+      "typeURI": "http://schemas.dmtf.org/cloud/audit/1.0/event",
+      "eventTime": "2018-10-11T22:42:22.720112+0000",
+      "target": {
+        "id": "octavia",
+        "typeURI": "service/load-balancer/loadbalancers",
+        "addresses": [{
+          "url": "http://10.21.21.53/load-balancer",
+          "name": "admin"
+        }, {
+          "url": "http://10.21.21.53/load-balancer",
+          "name": "private"
+        }, {
+          "url": "http://10.21.21.53/load-balancer",
+          "name": "public"
+        }],
+        "name": "octavia"
+      },
+      "observer": {
+        "id": "target"
+      },
+      "tags": ["correlation_id?value=e5b34bc3-4837-54fa-9892-8e65a9a2e73a"],
+      "eventType": "activity",
+      "initiator": {
+        "typeURI": "service/security/account/user",
+        "name": "admin",
+        "credential": {
+          "token": "***",
+          "identity_status": "Confirmed"
+        },
+        "host": {
+          "agent": "openstacksdk/0.17.2 keystoneauth1/3.11.0 python-requests/2.19.1 CPython/2.7.12",
+          "address": "10.21.21.53"
+        },
+        "project_id": "90168d185e504b5580884a235ba31612",
+        "id": "2af901396a424d5ca9dffa725226e8c7"
+      },
+      "reason": {
+        "reasonCode": "200",
+        "reasonType": "HTTP"
+      },
+      "reporterchain": [{
+        "reporterTime": "2018-10-11T22:42:22.852613+0000",
+        "role": "modifier",
+        "reporter": {
+          "id": "target"
+        }
+      }],
+      "action": "read/list",
+      "outcome": "success",
+      "id": "8cf14af5-246e-5739-a11e-513ca13b7d36",
+      "requestPath": "/load-balancer/v2.0/lbaas/loadbalancers"
+    },
+    "priority": "INFO",
+    "publisher_id": "uwsgi",
+    "message_id": "7cd89dce-af6e-40c5-8634-e87d1ed32a3c"
+  }

doc/source/admin/index.rst

@@ -37,6 +37,7 @@ Operator Reference
    Anchor.rst
    apache-httpd.rst
    providers.rst
+   api-audit.rst
 
 Indices and Search
 ------------------

etc/audit/octavia_api_audit_map.conf.sample

@@ -0,0 +1,32 @@
+[DEFAULT]
+# default target endpoint type
+# should match the endpoint type defined in service catalog
+target_endpoint_type = load-balancer
+
+[custom_actions]
+failover = failover
+
+# possible end path of API requests
+# path of api requests for CADF target typeURI
+# Just need to include top resource path to identify class
+# of resources. Ex: Log audit event for API requests
+# path containing "nodes" keyword and node uuid.
+[path_keywords]
+amphorae = amphora
+defaults = None
+failover = None
+healthmonitors = healthmonitor
+l7policies = l7policy
+listeners = listener
+loadbalancers = loadbalancer
+members = member
+pools = pool
+providers = None
+quotas = quota
+rules = rule
+stats = None
+status = None
+
+# map endpoint type defined in service catalog to CADF typeURI
+[service_endpoints]
+load-balancer = service/load-balancer

etc/octavia.conf

@@ -423,3 +423,40 @@
 # default_member_quota = -1
 # default_pool_quota = -1
 # default_health_monitor_quota = -1
+
+[audit]
+# Enable auditing of API requests.
+# enabled = False
+
+# Path to audit map file for octavia-api service. Used only
+# when API audit is enabled.
+# audit_map_file = /etc/octavia/octavia_api_audit_map.conf
+
+# Comma separated list of REST API HTTP methods to be
+# ignored during audit. For example: auditing will not be done
+# on any GET or POST requests if this is set to "GET,POST". It
+# is used only when API audit is enabled.
+# ignore_req_list =
+
+[audit_middleware_notifications]
+# Note: This section comes from openstack/keystonemiddleware
+# It is included here for documentation convenience and may be out of date
+
+# Indicate whether to use oslo_messaging as the notifier. If set to False,
+# the local logger will be used as the notifier. If set to True, the
+# oslo_messaging package must also be present. Otherwise, the local will be
+# used instead.
+# use_oslo_messaging = True
+
+# The Driver to handle sending notifications. Possible values are messaging,
+# messagingv2, routing, log, test, noop. If not specified, then value from
+# oslo_messaging_notifications conf section is used.
+# driver =
+
+# List of AMQP topics used for OpenStack notifications. If not specified,
+# then value from oslo_messaging_notifications conf section is used.
+# topics =
+
+# A URL representing messaging driver to use for notification. If not
+# specified, we fall back to the same configuration used for RPC.
+# transport_url =

octavia/api/app.py

@@ -12,6 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import keystonemiddleware.audit as audit_middleware
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_middleware import cors
@@ -21,6 +22,7 @@ import pecan
 from octavia.api import config as app_config
 from octavia.api.drivers import driver_factory
 from octavia.common import constants
+from octavia.common import exceptions
 from octavia.common import keystone
 from octavia.common import service as octavia_service
 
@@ -63,6 +65,21 @@ def setup_app(pecan_config=None, debug=False, argv=None):
 def _wrap_app(app):
     """Wraps wsgi app with additional middlewares."""
     app = request_id.RequestId(app)
+
+    if CONF.audit.enabled:
+        try:
+            app = audit_middleware.AuditMiddleware(
+                app,
+                audit_map_file=CONF.audit.audit_map_file,
+                ignore_req_list=CONF.audit.ignore_req_list
+            )
+        except (EnvironmentError, OSError,
+                audit_middleware.PycadfAuditApiConfigError) as e:
+            raise exceptions.InputFileError(
+                file_name=CONF.audit.audit_map_file,
+                reason=e
+            )
+
     if cfg.CONF.api_settings.auth_strategy == constants.KEYSTONE:
         app = keystone.SkippingAuthProtocol(app, {})
 

octavia/common/config.py

@@ -579,6 +579,20 @@ quota_opts = [
                help=_('Default per project health monitor quota.')),
 ]
 
+audit_opts = [
+    cfg.BoolOpt('enabled', default=False,
+                help=_('Enable auditing of API requests')),
+    cfg.StrOpt('audit_map_file',
+               default='/etc/octavia/octavia_api_audit_map.conf',
+               help=_('Path to audit map file for octavia-api service. '
+                      'Used only when API audit is enabled.')),
+    cfg.StrOpt('ignore_req_list', default='',
+               help=_('Comma separated list of REST API HTTP methods to be '
+                      'ignored during audit. For example: auditing will not '
+                      'be done on any GET or POST requests if this is set to '
+                      '"GET,POST". It is used only when API audit is '
+                      'enabled.')),
+]
 
 # Register the configuration options
 cfg.CONF.register_opts(core_opts)
@@ -599,7 +613,7 @@ cfg.CONF.register_opts(nova_opts, group='nova')
 cfg.CONF.register_opts(glance_opts, group='glance')
 cfg.CONF.register_opts(neutron_opts, group='neutron')
 cfg.CONF.register_opts(quota_opts, group='quotas')
-
+cfg.CONF.register_opts(audit_opts, group='audit')
 
 # Ensure that the control exchange is set correctly
 messaging.set_transport_defaults(control_exchange='octavia')

octavia/common/exceptions.py

@@ -363,3 +363,7 @@ class ProviderUnsupportedOptionError(APIException):
     msg = _("Provider '%(prov)s' does not support a requested option: "
             "%(user_msg)s")
     code = 501
+
+
+class InputFileError(OctaviaException):
+    message = _('Error with file %(file_name)s. Reason: %(reason)s')

octavia/opts.py

@@ -41,6 +41,7 @@ def list_opts():
         ('neutron', octavia.common.config.neutron_opts),
         ('glance', octavia.common.config.glance_opts),
         ('quotas', octavia.common.config.quota_opts),
+        ('audit', octavia.common.config.audit_opts),
         add_auth_opts(),
     ]
 

releasenotes/notes/add_api_audit-58dc16bff517eae7.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    The Octavia API now supports Cloud Auditing Data Federation (CADF)
+    auditing.