Adjust Vitrage api to support Prometheus datasource

This includes adding support to basic mode auth along side to keystone auth.

documentation commits coming after

Change-Id: If99fd31dae55b30f199f261adb6a6de933857ad2

devstack/apache-vitrage.template

@@ -4,6 +4,7 @@ Listen %PORT%
     WSGIDaemonProcess vitrage-api processes=%APIWORKERS% threads=10 user=%USER% display-name=%{GROUP} %VIRTUALENV%
     WSGIProcessGroup vitrage-api
     WSGIScriptAlias / %WSGIAPP%
+    WSGIPassAuthorization On
     WSGIApplicationGroup %{GLOBAL}
     <IfVersion >= 2.4>
         ErrorLogFormat "%{cu}t %M"

etc/vitrage/api-paste.ini

@@ -7,7 +7,7 @@ use = egg:Paste#urlmap
 [composite:vitrage+keystone]
 use = egg:Paste#urlmap
 / = vitrageversions_pipeline
-/v1 = vitragev1_keystone_pipeline
+/v1 = vitrage_event_pipline
 /healthcheck = healthcheck
 
 [composite:vitrage+keycloak]
@@ -32,8 +32,8 @@ root = vitrage.api.controllers.root.VersionsController
 [pipeline:vitragev1_noauth_pipeline]
 pipeline = cors http_proxy_to_wsgi request_id osprofiler vitragev1
 
-[pipeline:vitragev1_keystone_pipeline]
+[pipeline:vitrage_event_pipline]
-pipeline = cors http_proxy_to_wsgi request_id osprofiler keystoneauthtoken vitragev1
+pipeline = cors http_proxy_to_wsgi request_id osprofiler basic_and_keystone_auth vitragev1
 
 [pipeline:vitragev1_keycloak_pipeline]
 pipeline = cors http_proxy_to_wsgi request_id osprofiler keycloakauthtoken vitragev1
@@ -42,8 +42,8 @@ pipeline = cors http_proxy_to_wsgi request_id osprofiler keycloakauthtoken vitra
 paste.app_factory = vitrage.api.app:app_factory
 root = vitrage.api.controllers.v1.root.V1Controller
 
-[filter:keystoneauthtoken]
+[filter:basic_and_keystone_auth]
-paste.filter_factory = keystonemiddleware.auth_token:filter_factory
+paste.filter_factory = vitrage.middleware.basic_and_keystone_auth:filter_factory
 oslo_config_project = vitrage
 
 [filter:keycloakauthtoken]

vitrage/api/controllers/v1/event.py

@@ -18,8 +18,10 @@ from oslo_log import log
 from osprofiler import profiler
 from pecan.core import abort
 
+from datetime import datetime
 from vitrage.api.controllers.rest import RootRestController
 from vitrage.api.policy import enforce
+from vitrage.datasources.prometheus.driver import PROMETHEUS_EVENT_TYPE
 
 
 LOG = log.getLogger(__name__)
@@ -37,9 +39,14 @@ class EventController(RootRestController):
         enforce("event post", pecan.request.headers,
                 pecan.request.enforcer, {})
 
-        event_time = kwargs['time']
+        prom_event_type = None
-        event_type = kwargs['type']
+        user_agent = pecan.request.headers.get('User-Agent')
-        details = kwargs['details']
+        if user_agent and user_agent.startswith("Alertmanager"):
+            prom_event_type = PROMETHEUS_EVENT_TYPE
+
+        event_time = kwargs.get('time', datetime.utcnow())
+        event_type = kwargs.get('type', prom_event_type)
+        details = kwargs.get('details', kwargs)
 
         self.post_event(event_time, event_type, details)
 

vitrage/api_handler/apis/event.py

@@ -42,13 +42,12 @@ class EventApis(EntityGraphApisBase):
             event = {EventProperties.TYPE: event_type,
                      EventProperties.TIME: event_time,
                      EventProperties.DETAILS: details}
-
             if details.get(DoctorDetails.STATUS) == DoctorStatus.UP:
                 notification_type = DoctorProperties.CUSTOM_EVENT_UP
             elif details.get(DoctorDetails.STATUS) == DoctorStatus.DOWN:
                 notification_type = DoctorProperties.CUSTOM_EVENT_DOWN
             else:
-                raise Exception("Unknown status")
+                notification_type = event_type
 
             self.oslo_notifier.info(
                 ctxt={'message_id': uuidutils.generate_uuid(),

vitrage/datasources/prometheus/driver.py

@@ -17,6 +17,7 @@ from oslo_log import log
 
 from vitrage.common.constants import DatasourceAction
 from vitrage.common.constants import DatasourceProperties as DSProps
+from vitrage.common.constants import EventProperties as EProps
 from vitrage.datasources.alarm_driver_base import AlarmDriverBase
 from vitrage.datasources.prometheus import PROMETHEUS_DATASOURCE
 from vitrage.datasources.prometheus.properties import get_alarm_update_time
@@ -67,6 +68,8 @@ class PrometheusDriver(AlarmDriverBase):
         """Get an event from Prometheus and create a list of alarm events
 
         :param event: dictionary of this form:
+            {
+              "details":
                 {
                   "status": "firing",
                   "groupLabels": {
@@ -75,7 +78,7 @@ class PrometheusDriver(AlarmDriverBase):
                   "groupKey": "{}:{alertname=\"HighInodeUsage\"}",
                   "commonAnnotations": {
                     "mount_point": "/%",
-                "description": "\"Consider ssh\"ing into the instance \"\n",
+                    "description": "\"Consider ssh\"ing into instance \"\n",
                     "title": "High number of inode usage",
                     "value": "96.81%",
                     "device": "/dev/vda1%",
@@ -94,15 +97,15 @@ class PrometheusDriver(AlarmDriverBase):
                         "mountpoint": "/"
                       },
                       "endsAt": "0001-01-01T00:00:00Z",
-                  "generatorURL": "http://devstack-rocky-4:9090/graph?g0.htm1",
+                      "generatorURL": "http://devstack-4:9090/graph?g0.htm1",
                       "startsAt": "2018-05-03T12:25:38.231388525Z",
                       "annotations": {
                         "mount_point": "/%",
-                    "description": "\"Consider ssh\"ing into the instance\"\n",
+                        "description": "\"Consider ssh\"ing into instance\"\n",
                         "title": "High number of inode usage",
                         "value": "96.81%",
                         "device": "/dev/vda1%",
-                    "runbook": "troubleshooting/filesystem_alerts_inodes.md"
+                        "runbook": "filesystem_alerts_inodes.md"
                       }
                     }
                   ],
@@ -119,6 +122,7 @@ class PrometheusDriver(AlarmDriverBase):
                     "mountpoint": "/"
                   }
                 }
+            }
 
         :param event_type: The type of the event. Always 'prometheus.alarm'.
         :return: a list of events, one per Prometheus alert
@@ -128,13 +132,16 @@ class PrometheusDriver(AlarmDriverBase):
         LOG.debug('Going to enrich event: %s', str(event))
 
         alarms = []
-
+        details = event.get(EProps.DETAILS)
-        for alarm in event.get(PProps.ALERTS, []):
+        if details:
+            for alarm in details.get(PProps.ALERTS, []):
                 alarm[DSProps.EVENT_TYPE] = event_type
-            alarm[PProps.STATUS] = event[PProps.STATUS]
+                alarm[PProps.STATUS] = details[PProps.STATUS]
 
                 old_alarm = self._old_alarm(alarm)
-            alarm = self._filter_and_cache_alarm(alarm, old_alarm,
+                alarm = \
+                    self._filter_and_cache_alarm(alarm,
+                                                 old_alarm,
                                                  self._filter_get_erroneous,
                                                  get_alarm_update_time(alarm))
 

vitrage/middleware/basic_and_keystone_auth.py

@@ -0,0 +1,138 @@
+# Copyright 2018 - Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import werkzeug.http
+
+from keystoneauth1.identity.generic import password
+from keystoneauth1 import loading
+from keystoneauth1 import session
+from keystonemiddleware.auth_token import AuthProtocol
+from oslo_config import cfg
+from oslo_log import log as logging
+from oslo_serialization import jsonutils
+from oslo_utils import encodeutils
+from six.moves import http_client as httplib
+from webob import exc
+
+
+LOG = logging.getLogger(__name__)
+CFG_GROUP = "service_credentials"
+
+
+class BasicAndKeystoneAuth(AuthProtocol):
+
+    def __init__(self, application, conf):
+        super(BasicAndKeystoneAuth, self).__init__(application, conf)
+
+        self.application = application
+
+        self.oslo_conf = cfg.ConfigOpts()
+        self.oslo_conf([],
+                       project='vitrage',
+                       validate_default_values=True)
+        password_option = loading.get_auth_plugin_conf_options('password')
+        self.oslo_conf.register_opts(password_option, group=CFG_GROUP)
+        self.auth_url = self.oslo_conf.service_credentials.auth_url
+
+    @property
+    def reject_auth_headers(self):
+        header_val = 'Keystone uri=\'%s\'' % self.auth_url
+        return [('WWW-Authenticate', header_val)]
+
+    def process_request(self, req):
+        """Process request.
+
+            Evaluate the headers in a request and attempt to authenticate the
+            request according to authentication mode.
+            If the request comes through /v1/event api path then it can be
+            authenticate either with basic auth by providing username and
+            password or with keystone authentication.
+            If authenticated then additional headers are added to the
+            request for use by applications. If not authenticated the request
+            will be rejected or marked unauthenticated depending on
+            configuration.
+        """
+        if req.path == '/v1/event':
+            basic_auth_info = self._get_basic_authenticator(req)
+            if basic_auth_info:
+                self._basic_authenticate(basic_auth_info, req)
+
+            else:
+                super(BasicAndKeystoneAuth, self).process_request(req)
+        else:
+            super(BasicAndKeystoneAuth, self).process_request(req)
+
+    def _basic_authenticate(self, auth_info, req):
+        try:
+            project_domain_id, project_name, user_domain_id = \
+                self._get_auth_params()
+            auth = password.Password(auth_url=self.auth_url,
+                                     username=auth_info.username,
+                                     password=auth_info.password,
+                                     user_domain_id=user_domain_id,
+                                     project_domain_id=project_domain_id,
+                                     project_name=project_name)
+            sess = session.Session(auth=auth)
+            token = sess.get_token()
+            project_id = str(auth.get_project_id(sess))
+            roles = str(auth.get_auth_ref(sess).role_names[0])
+            self._set_req_headers(req, token, project_id, roles)
+        except Exception as e:
+            to_unicode = encodeutils.exception_to_unicode(e)
+            message = 'Authorization exception: %s' % to_unicode
+            self._unauthorized(message)
+
+    def _get_auth_params(self):
+        user_domain_id = \
+            self.oslo_conf.service_credentials.user_domain_id
+        project_domain_id = \
+            self.oslo_conf.service_credentials.project_domain_id
+        project_name = self.oslo_conf.service_credentials.project_name
+        return project_domain_id, project_name, user_domain_id
+
+    def _unauthorized(self, message):
+        body = {'error': {
+            'code': httplib.UNAUTHORIZED,
+            'title': httplib.responses.get(httplib.UNAUTHORIZED),
+            'message': message,
+        }}
+
+        raise exc.HTTPUnauthorized(body=jsonutils.dumps(body),
+                                   headers=self.reject_auth_headers,
+                                   charset='UTF-8',
+                                   content_type='application/json')
+
+    @staticmethod
+    def _get_basic_authenticator(req):
+        auth = werkzeug.http.parse_authorization_header(
+            req.headers.get("Authorization"))
+        return auth
+
+    @staticmethod
+    def _set_req_headers(req, token, project_id, roles):
+        req.headers['X-Auth-Token'] = token
+        req.headers['X-Identity-Status'] = 'Confirmed'
+        req.headers['X-Project-Id'] = project_id
+        req.headers['X-Roles'] = roles
+
+
+def filter_factory(global_conf, **local_conf):
+    """Return a WSGI filter app for use with paste.deploy."""
+    conf = global_conf.copy()
+    conf.update(local_conf)
+
+    def auth_filter(app):
+        return BasicAndKeystoneAuth(app, conf)
+
+    return auth_filter

vitrage/service.py

@@ -27,6 +27,7 @@ LOG = log.getLogger(__name__)
 
 
 def prepare_service(args=None, conf=None, config_files=None):
+    set_defaults()
     if conf is None:
         conf = cfg.ConfigOpts()
     log.register_options(conf)
@@ -59,3 +60,16 @@ def prepare_service(args=None, conf=None, config_files=None):
     messaging.setup()
 
     return conf
+
+
+def set_defaults():
+    from oslo_middleware import cors
+    cfg.set_defaults(cors.CORS_OPTS,
+                     allow_headers=[
+                         'Authorization',
+                         'X-Auth-Token',
+                         'X-Subject-Token',
+                         'X-User-Id',
+                         'X-Domain-Id',
+                         'X-Project-Id',
+                         'X-Roles'])

vitrage/tests/functional/api/v1/test_basic.py

@@ -0,0 +1,189 @@
+# Copyright 2018 - Nokia Corporation
+# Copyright 2014 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# noinspection PyPackageRequirements
+import uuid
+
+from datetime import datetime
+from mock import mock
+from six.moves import http_client as httplib
+from vitrage.tests.functional.api.v1 import FunctionalTest
+
+HEADERS = {
+    'Authorization': 'Basic dml0cmFnZTpwYXNzd29yZA==',
+    'User-Agent': 'Alertmanager/0.15.0',
+    'Host': '127.0.0.1:8999',
+    'Content-Type': 'application/json'
+}
+
+EVENT_TYPE = 'prometheus.alarm'
+
+VALID_TOKEN = uuid.uuid4().hex
+
+PROJECT_ID = 'best_project'
+
+
+class Role(object):
+    pass
+
+
+ROLES = Role()
+
+ROLES.role_names = ['admin']
+
+EVENT_DETAILS = {
+    "status": "firing",
+    "groupLabels": {
+        "alertname": "HighInodeUsage"
+    },
+    "groupKey": "{}:{alertname=\"HighInodeUsage\"}",
+    "commonAnnotations": {
+        "mount_point": "/%",
+        "description": "\"Consider ssh\"ing into the instance \"\n",
+        "title": "High number of inode usage",
+        "value": "96.81%",
+        "device": "/dev/vda1%",
+        "runbook": "troubleshooting/filesystem_alerts_inodes.md"
+    },
+    "alerts": [
+        {
+            "status": "firing",
+            "labels": {
+                "severity": "critical",
+                "fstype": "ext4",
+                "instance": "localhost:9100",
+                "job": "node",
+                "alertname": "HighInodeUsage",
+                "device": "/dev/vda1",
+                "mountpoint": "/"
+            },
+            "endsAt": "0001-01-01T00:00:00Z",
+            "generatorURL": "http://devstack-rocky-4:9090/graph?g0.htm1",
+            "startsAt": "2018-05-03T12:25:38.231388525Z",
+            "annotations": {
+                "mount_point": "/%",
+                "description": "\"Consider ssh\"ing into the instance\"\n",
+                "title": "High number of inode usage",
+                "value": "96.81%",
+                "device": "/dev/vda1%",
+                "runbook": "troubleshooting/filesystem_alerts_inodes.md"
+            }
+        }
+    ],
+    "version": "4",
+    "receiver": "vitrage",
+    "externalURL": "http://devstack-rocky-4:9093",
+    "commonLabels": {
+        "severity": "critical",
+        "fstype": "ext4",
+        "instance": "localhost:9100",
+        "job": "node",
+        "alertname": "HighInodeUsage",
+        "device": "/dev/vda1",
+        "mountpoint": "/"
+    }
+}
+
+ERR_MSG_MISSING_AUTH = 'The request you have made requires authentication.'
+
+ERR_MSG_MISSING_VERSIONED_IDENTITY_ENDPOINTS = 'Authorization exception: ' \
+                                               'Could not find versioned ' \
+                                               'identity endpoints when ' \
+                                               'attempting to authenticate. ' \
+                                               'Please check that your ' \
+                                               'auth_url is correct.'
+
+ERR_MISSING_AUTH = {'error': {
+    'code': httplib.UNAUTHORIZED,
+    'title': httplib.responses[httplib.UNAUTHORIZED],
+    'message': ERR_MSG_MISSING_AUTH,
+}}
+
+ERR_MISSING_VERSIONED_IDENTITY_ENDPOINTS = {'error': {
+    'code': httplib.UNAUTHORIZED,
+    'title': httplib.responses[httplib.UNAUTHORIZED],
+    'message': ERR_MSG_MISSING_VERSIONED_IDENTITY_ENDPOINTS,
+}}
+
+
+class BasicAuthTest(FunctionalTest):
+
+    def __init__(self, *args, **kwds):
+        super(BasicAuthTest, self).__init__(*args, **kwds)
+        self.auth = 'keystone'
+
+    keystoneauth__identity = 'keystoneauth1.identity'
+
+    @mock.patch('keystoneauth1.session.Session.get_token',
+                return_value=VALID_TOKEN)
+    @mock.patch('%s.base.BaseIdentityPlugin.get_project_id' %
+                keystoneauth__identity,
+                return_value=PROJECT_ID)
+    @mock.patch('%s.generic.base.BaseGenericPlugin.get_auth_ref' %
+                keystoneauth__identity,
+                return_value=ROLES)
+    @mock.patch('pecan.request')
+    def test_header_parsing(self, req_mock, *args):
+        resp = self.post_json('/event',
+                              params={
+                                  'time': datetime.now().isoformat(),
+                                  'type': EVENT_TYPE,
+                                  'details': EVENT_DETAILS
+                              },
+                              headers=HEADERS)
+        req = resp.request
+        self.assertEqual('Confirmed', req.headers['X-Identity-Status'])
+        self.assertEqual(ROLES.role_names[0], req.headers['X-Roles'])
+        self.assertEqual(PROJECT_ID, req.headers['X-Project-Id'])
+        self.assertEqual(VALID_TOKEN, req.headers['X-Auth-Token'])
+        self.assertEqual(1, req_mock.client.call.call_count)
+
+    @mock.patch('keystoneauth1.session.Session.request')
+    def test_basic_mode_auth_wrong_authorization(self, *args):
+        wrong_headers = HEADERS.copy()
+        wrong_headers['Authorization'] = 'Basic dml0cmFnZTpwdml0cmFnZT=='
+        resp = self.post_json('/event',
+                              params={
+                                  'time': datetime.now().isoformat(),
+                                  'type': EVENT_TYPE,
+                                  'details': EVENT_DETAILS
+                              },
+                              headers=wrong_headers,
+                              expect_errors=True)
+        self.assertEqual(httplib.UNAUTHORIZED, resp.status_code)
+        self.assertDictEqual(ERR_MISSING_VERSIONED_IDENTITY_ENDPOINTS,
+                             resp.json)
+
+    def test_in_basic_mode_auth_no_header(self):
+        resp = self.post_json('/event', expect_errors=True)
+
+        self.assertEqual(httplib.UNAUTHORIZED, resp.status_code)
+        self.assertDictEqual(ERR_MISSING_AUTH, resp.json)
+
+    @mock.patch('keystoneauth1.identity.generic.password.Password')
+    @mock.patch('keystoneauth1.session.Session')
+    @mock.patch('pecan.request')
+    def test_in_basic_mode_auth_success(self, req_mock, *args):
+        resp = self.post_json('/event',
+                              params={
+                                  'time': datetime.now().isoformat(),
+                                  'type': EVENT_TYPE,
+                                  'details': EVENT_DETAILS
+                              },
+                              headers=HEADERS)
+
+        self.assertEqual(1, req_mock.client.call.call_count)
+        self.assertEqual(httplib.OK, resp.status_code)

vitrage/tests/resources/mock_configurations/driver/driver_prometheus_update_dynamic.json

@@ -1,3 +1,5 @@
+{
+  "details":
     {
       "status": "firing",
       "groupLabels": {
@@ -50,3 +52,4 @@
         "mountpoint": "/"
       }
     }
+}