From 034a597c9fec31bb4f635ebfb9d75f88ac830931 Mon Sep 17 00:00:00 2001 From: Muhamad Najjar Date: Sun, 22 Jul 2018 15:46:53 +0000 Subject: [PATCH] Adjust Vitrage api to support Prometheus datasource This includes adding support to basic mode auth along side to keystone auth. documentation commits coming after Change-Id: If99fd31dae55b30f199f261adb6a6de933857ad2 --- devstack/apache-vitrage.template | 1 + etc/vitrage/api-paste.ini | 10 +- vitrage/api/controllers/v1/event.py | 13 +- vitrage/api_handler/apis/event.py | 3 +- vitrage/datasources/prometheus/driver.py | 101 +++++----- vitrage/middleware/basic_and_keystone_auth.py | 138 +++++++++++++ vitrage/service.py | 14 ++ vitrage/tests/functional/api/v1/test_basic.py | 189 ++++++++++++++++++ .../driver_prometheus_update_dynamic.json | 81 ++++---- 9 files changed, 454 insertions(+), 96 deletions(-) create mode 100644 vitrage/middleware/basic_and_keystone_auth.py create mode 100644 vitrage/tests/functional/api/v1/test_basic.py diff --git a/devstack/apache-vitrage.template b/devstack/apache-vitrage.template index 0ef49b77f..b276a2f78 100644 --- a/devstack/apache-vitrage.template +++ b/devstack/apache-vitrage.template @@ -4,6 +4,7 @@ Listen %PORT% WSGIDaemonProcess vitrage-api processes=%APIWORKERS% threads=10 user=%USER% display-name=%{GROUP} %VIRTUALENV% WSGIProcessGroup vitrage-api WSGIScriptAlias / %WSGIAPP% + WSGIPassAuthorization On WSGIApplicationGroup %{GLOBAL} = 2.4> ErrorLogFormat "%{cu}t %M" diff --git a/etc/vitrage/api-paste.ini b/etc/vitrage/api-paste.ini index ee1643370..2de82f5ff 100644 --- a/etc/vitrage/api-paste.ini +++ b/etc/vitrage/api-paste.ini @@ -7,7 +7,7 @@ use = egg:Paste#urlmap [composite:vitrage+keystone] use = egg:Paste#urlmap / = vitrageversions_pipeline -/v1 = vitragev1_keystone_pipeline +/v1 = vitrage_event_pipline /healthcheck = healthcheck [composite:vitrage+keycloak] @@ -32,8 +32,8 @@ root = vitrage.api.controllers.root.VersionsController [pipeline:vitragev1_noauth_pipeline] pipeline = cors http_proxy_to_wsgi request_id osprofiler vitragev1 -[pipeline:vitragev1_keystone_pipeline] -pipeline = cors http_proxy_to_wsgi request_id osprofiler keystoneauthtoken vitragev1 +[pipeline:vitrage_event_pipline] +pipeline = cors http_proxy_to_wsgi request_id osprofiler basic_and_keystone_auth vitragev1 [pipeline:vitragev1_keycloak_pipeline] pipeline = cors http_proxy_to_wsgi request_id osprofiler keycloakauthtoken vitragev1 @@ -42,8 +42,8 @@ pipeline = cors http_proxy_to_wsgi request_id osprofiler keycloakauthtoken vitra paste.app_factory = vitrage.api.app:app_factory root = vitrage.api.controllers.v1.root.V1Controller -[filter:keystoneauthtoken] -paste.filter_factory = keystonemiddleware.auth_token:filter_factory +[filter:basic_and_keystone_auth] +paste.filter_factory = vitrage.middleware.basic_and_keystone_auth:filter_factory oslo_config_project = vitrage [filter:keycloakauthtoken] diff --git a/vitrage/api/controllers/v1/event.py b/vitrage/api/controllers/v1/event.py index 42f5c1541..2e0779d94 100644 --- a/vitrage/api/controllers/v1/event.py +++ b/vitrage/api/controllers/v1/event.py @@ -18,8 +18,10 @@ from oslo_log import log from osprofiler import profiler from pecan.core import abort +from datetime import datetime from vitrage.api.controllers.rest import RootRestController from vitrage.api.policy import enforce +from vitrage.datasources.prometheus.driver import PROMETHEUS_EVENT_TYPE LOG = log.getLogger(__name__) @@ -37,9 +39,14 @@ class EventController(RootRestController): enforce("event post", pecan.request.headers, pecan.request.enforcer, {}) - event_time = kwargs['time'] - event_type = kwargs['type'] - details = kwargs['details'] + prom_event_type = None + user_agent = pecan.request.headers.get('User-Agent') + if user_agent and user_agent.startswith("Alertmanager"): + prom_event_type = PROMETHEUS_EVENT_TYPE + + event_time = kwargs.get('time', datetime.utcnow()) + event_type = kwargs.get('type', prom_event_type) + details = kwargs.get('details', kwargs) self.post_event(event_time, event_type, details) diff --git a/vitrage/api_handler/apis/event.py b/vitrage/api_handler/apis/event.py index 8b83357f3..09de8caeb 100644 --- a/vitrage/api_handler/apis/event.py +++ b/vitrage/api_handler/apis/event.py @@ -42,13 +42,12 @@ class EventApis(EntityGraphApisBase): event = {EventProperties.TYPE: event_type, EventProperties.TIME: event_time, EventProperties.DETAILS: details} - if details.get(DoctorDetails.STATUS) == DoctorStatus.UP: notification_type = DoctorProperties.CUSTOM_EVENT_UP elif details.get(DoctorDetails.STATUS) == DoctorStatus.DOWN: notification_type = DoctorProperties.CUSTOM_EVENT_DOWN else: - raise Exception("Unknown status") + notification_type = event_type self.oslo_notifier.info( ctxt={'message_id': uuidutils.generate_uuid(), diff --git a/vitrage/datasources/prometheus/driver.py b/vitrage/datasources/prometheus/driver.py index c9933d701..8d78e9d88 100644 --- a/vitrage/datasources/prometheus/driver.py +++ b/vitrage/datasources/prometheus/driver.py @@ -17,6 +17,7 @@ from oslo_log import log from vitrage.common.constants import DatasourceAction from vitrage.common.constants import DatasourceProperties as DSProps +from vitrage.common.constants import EventProperties as EProps from vitrage.datasources.alarm_driver_base import AlarmDriverBase from vitrage.datasources.prometheus import PROMETHEUS_DATASOURCE from vitrage.datasources.prometheus.properties import get_alarm_update_time @@ -68,23 +69,50 @@ class PrometheusDriver(AlarmDriverBase): :param event: dictionary of this form: { - "status": "firing", - "groupLabels": { - "alertname": "HighInodeUsage" - }, - "groupKey": "{}:{alertname=\"HighInodeUsage\"}", - "commonAnnotations": { - "mount_point": "/%", - "description": "\"Consider ssh\"ing into the instance \"\n", - "title": "High number of inode usage", - "value": "96.81%", - "device": "/dev/vda1%", - "runbook": "troubleshooting/filesystem_alerts_inodes.md" - }, - "alerts": [ + "details": { "status": "firing", - "labels": { + "groupLabels": { + "alertname": "HighInodeUsage" + }, + "groupKey": "{}:{alertname=\"HighInodeUsage\"}", + "commonAnnotations": { + "mount_point": "/%", + "description": "\"Consider ssh\"ing into instance \"\n", + "title": "High number of inode usage", + "value": "96.81%", + "device": "/dev/vda1%", + "runbook": "troubleshooting/filesystem_alerts_inodes.md" + }, + "alerts": [ + { + "status": "firing", + "labels": { + "severity": "critical", + "fstype": "ext4", + "instance": "localhost:9100", + "job": "node", + "alertname": "HighInodeUsage", + "device": "/dev/vda1", + "mountpoint": "/" + }, + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://devstack-4:9090/graph?g0.htm1", + "startsAt": "2018-05-03T12:25:38.231388525Z", + "annotations": { + "mount_point": "/%", + "description": "\"Consider ssh\"ing into instance\"\n", + "title": "High number of inode usage", + "value": "96.81%", + "device": "/dev/vda1%", + "runbook": "filesystem_alerts_inodes.md" + } + } + ], + "version": "4", + "receiver": "vitrage", + "externalURL": "http://devstack-rocky-4:9093", + "commonLabels": { "severity": "critical", "fstype": "ext4", "instance": "localhost:9100", @@ -92,32 +120,8 @@ class PrometheusDriver(AlarmDriverBase): "alertname": "HighInodeUsage", "device": "/dev/vda1", "mountpoint": "/" - }, - "endsAt": "0001-01-01T00:00:00Z", - "generatorURL": "http://devstack-rocky-4:9090/graph?g0.htm1", - "startsAt": "2018-05-03T12:25:38.231388525Z", - "annotations": { - "mount_point": "/%", - "description": "\"Consider ssh\"ing into the instance\"\n", - "title": "High number of inode usage", - "value": "96.81%", - "device": "/dev/vda1%", - "runbook": "troubleshooting/filesystem_alerts_inodes.md" } } - ], - "version": "4", - "receiver": "vitrage", - "externalURL": "http://devstack-rocky-4:9093", - "commonLabels": { - "severity": "critical", - "fstype": "ext4", - "instance": "localhost:9100", - "job": "node", - "alertname": "HighInodeUsage", - "device": "/dev/vda1", - "mountpoint": "/" - } } :param event_type: The type of the event. Always 'prometheus.alarm'. @@ -128,18 +132,21 @@ class PrometheusDriver(AlarmDriverBase): LOG.debug('Going to enrich event: %s', str(event)) alarms = [] + details = event.get(EProps.DETAILS) + if details: + for alarm in details.get(PProps.ALERTS, []): + alarm[DSProps.EVENT_TYPE] = event_type + alarm[PProps.STATUS] = details[PProps.STATUS] - for alarm in event.get(PProps.ALERTS, []): - alarm[DSProps.EVENT_TYPE] = event_type - alarm[PProps.STATUS] = event[PProps.STATUS] - - old_alarm = self._old_alarm(alarm) - alarm = self._filter_and_cache_alarm(alarm, old_alarm, + old_alarm = self._old_alarm(alarm) + alarm = \ + self._filter_and_cache_alarm(alarm, + old_alarm, self._filter_get_erroneous, get_alarm_update_time(alarm)) - if alarm: - alarms.append(alarm) + if alarm: + alarms.append(alarm) LOG.debug('Enriched event. Created alarm events: %s', str(alarms)) diff --git a/vitrage/middleware/basic_and_keystone_auth.py b/vitrage/middleware/basic_and_keystone_auth.py new file mode 100644 index 000000000..ffc65a28e --- /dev/null +++ b/vitrage/middleware/basic_and_keystone_auth.py @@ -0,0 +1,138 @@ +# Copyright 2018 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import werkzeug.http + +from keystoneauth1.identity.generic import password +from keystoneauth1 import loading +from keystoneauth1 import session +from keystonemiddleware.auth_token import AuthProtocol +from oslo_config import cfg +from oslo_log import log as logging +from oslo_serialization import jsonutils +from oslo_utils import encodeutils +from six.moves import http_client as httplib +from webob import exc + + +LOG = logging.getLogger(__name__) +CFG_GROUP = "service_credentials" + + +class BasicAndKeystoneAuth(AuthProtocol): + + def __init__(self, application, conf): + super(BasicAndKeystoneAuth, self).__init__(application, conf) + + self.application = application + + self.oslo_conf = cfg.ConfigOpts() + self.oslo_conf([], + project='vitrage', + validate_default_values=True) + password_option = loading.get_auth_plugin_conf_options('password') + self.oslo_conf.register_opts(password_option, group=CFG_GROUP) + self.auth_url = self.oslo_conf.service_credentials.auth_url + + @property + def reject_auth_headers(self): + header_val = 'Keystone uri=\'%s\'' % self.auth_url + return [('WWW-Authenticate', header_val)] + + def process_request(self, req): + """Process request. + + Evaluate the headers in a request and attempt to authenticate the + request according to authentication mode. + If the request comes through /v1/event api path then it can be + authenticate either with basic auth by providing username and + password or with keystone authentication. + If authenticated then additional headers are added to the + request for use by applications. If not authenticated the request + will be rejected or marked unauthenticated depending on + configuration. + """ + if req.path == '/v1/event': + basic_auth_info = self._get_basic_authenticator(req) + if basic_auth_info: + self._basic_authenticate(basic_auth_info, req) + + else: + super(BasicAndKeystoneAuth, self).process_request(req) + else: + super(BasicAndKeystoneAuth, self).process_request(req) + + def _basic_authenticate(self, auth_info, req): + try: + project_domain_id, project_name, user_domain_id = \ + self._get_auth_params() + auth = password.Password(auth_url=self.auth_url, + username=auth_info.username, + password=auth_info.password, + user_domain_id=user_domain_id, + project_domain_id=project_domain_id, + project_name=project_name) + sess = session.Session(auth=auth) + token = sess.get_token() + project_id = str(auth.get_project_id(sess)) + roles = str(auth.get_auth_ref(sess).role_names[0]) + self._set_req_headers(req, token, project_id, roles) + except Exception as e: + to_unicode = encodeutils.exception_to_unicode(e) + message = 'Authorization exception: %s' % to_unicode + self._unauthorized(message) + + def _get_auth_params(self): + user_domain_id = \ + self.oslo_conf.service_credentials.user_domain_id + project_domain_id = \ + self.oslo_conf.service_credentials.project_domain_id + project_name = self.oslo_conf.service_credentials.project_name + return project_domain_id, project_name, user_domain_id + + def _unauthorized(self, message): + body = {'error': { + 'code': httplib.UNAUTHORIZED, + 'title': httplib.responses.get(httplib.UNAUTHORIZED), + 'message': message, + }} + + raise exc.HTTPUnauthorized(body=jsonutils.dumps(body), + headers=self.reject_auth_headers, + charset='UTF-8', + content_type='application/json') + + @staticmethod + def _get_basic_authenticator(req): + auth = werkzeug.http.parse_authorization_header( + req.headers.get("Authorization")) + return auth + + @staticmethod + def _set_req_headers(req, token, project_id, roles): + req.headers['X-Auth-Token'] = token + req.headers['X-Identity-Status'] = 'Confirmed' + req.headers['X-Project-Id'] = project_id + req.headers['X-Roles'] = roles + + +def filter_factory(global_conf, **local_conf): + """Return a WSGI filter app for use with paste.deploy.""" + conf = global_conf.copy() + conf.update(local_conf) + + def auth_filter(app): + return BasicAndKeystoneAuth(app, conf) + + return auth_filter diff --git a/vitrage/service.py b/vitrage/service.py index e44dee48c..f91dcf90b 100644 --- a/vitrage/service.py +++ b/vitrage/service.py @@ -27,6 +27,7 @@ LOG = log.getLogger(__name__) def prepare_service(args=None, conf=None, config_files=None): + set_defaults() if conf is None: conf = cfg.ConfigOpts() log.register_options(conf) @@ -59,3 +60,16 @@ def prepare_service(args=None, conf=None, config_files=None): messaging.setup() return conf + + +def set_defaults(): + from oslo_middleware import cors + cfg.set_defaults(cors.CORS_OPTS, + allow_headers=[ + 'Authorization', + 'X-Auth-Token', + 'X-Subject-Token', + 'X-User-Id', + 'X-Domain-Id', + 'X-Project-Id', + 'X-Roles']) diff --git a/vitrage/tests/functional/api/v1/test_basic.py b/vitrage/tests/functional/api/v1/test_basic.py new file mode 100644 index 000000000..b757a2fc1 --- /dev/null +++ b/vitrage/tests/functional/api/v1/test_basic.py @@ -0,0 +1,189 @@ +# Copyright 2018 - Nokia Corporation +# Copyright 2014 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# noinspection PyPackageRequirements +import uuid + +from datetime import datetime +from mock import mock +from six.moves import http_client as httplib +from vitrage.tests.functional.api.v1 import FunctionalTest + +HEADERS = { + 'Authorization': 'Basic dml0cmFnZTpwYXNzd29yZA==', + 'User-Agent': 'Alertmanager/0.15.0', + 'Host': '127.0.0.1:8999', + 'Content-Type': 'application/json' +} + +EVENT_TYPE = 'prometheus.alarm' + +VALID_TOKEN = uuid.uuid4().hex + +PROJECT_ID = 'best_project' + + +class Role(object): + pass + + +ROLES = Role() + +ROLES.role_names = ['admin'] + +EVENT_DETAILS = { + "status": "firing", + "groupLabels": { + "alertname": "HighInodeUsage" + }, + "groupKey": "{}:{alertname=\"HighInodeUsage\"}", + "commonAnnotations": { + "mount_point": "/%", + "description": "\"Consider ssh\"ing into the instance \"\n", + "title": "High number of inode usage", + "value": "96.81%", + "device": "/dev/vda1%", + "runbook": "troubleshooting/filesystem_alerts_inodes.md" + }, + "alerts": [ + { + "status": "firing", + "labels": { + "severity": "critical", + "fstype": "ext4", + "instance": "localhost:9100", + "job": "node", + "alertname": "HighInodeUsage", + "device": "/dev/vda1", + "mountpoint": "/" + }, + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://devstack-rocky-4:9090/graph?g0.htm1", + "startsAt": "2018-05-03T12:25:38.231388525Z", + "annotations": { + "mount_point": "/%", + "description": "\"Consider ssh\"ing into the instance\"\n", + "title": "High number of inode usage", + "value": "96.81%", + "device": "/dev/vda1%", + "runbook": "troubleshooting/filesystem_alerts_inodes.md" + } + } + ], + "version": "4", + "receiver": "vitrage", + "externalURL": "http://devstack-rocky-4:9093", + "commonLabels": { + "severity": "critical", + "fstype": "ext4", + "instance": "localhost:9100", + "job": "node", + "alertname": "HighInodeUsage", + "device": "/dev/vda1", + "mountpoint": "/" + } +} + +ERR_MSG_MISSING_AUTH = 'The request you have made requires authentication.' + +ERR_MSG_MISSING_VERSIONED_IDENTITY_ENDPOINTS = 'Authorization exception: ' \ + 'Could not find versioned ' \ + 'identity endpoints when ' \ + 'attempting to authenticate. ' \ + 'Please check that your ' \ + 'auth_url is correct.' + +ERR_MISSING_AUTH = {'error': { + 'code': httplib.UNAUTHORIZED, + 'title': httplib.responses[httplib.UNAUTHORIZED], + 'message': ERR_MSG_MISSING_AUTH, +}} + +ERR_MISSING_VERSIONED_IDENTITY_ENDPOINTS = {'error': { + 'code': httplib.UNAUTHORIZED, + 'title': httplib.responses[httplib.UNAUTHORIZED], + 'message': ERR_MSG_MISSING_VERSIONED_IDENTITY_ENDPOINTS, +}} + + +class BasicAuthTest(FunctionalTest): + + def __init__(self, *args, **kwds): + super(BasicAuthTest, self).__init__(*args, **kwds) + self.auth = 'keystone' + + keystoneauth__identity = 'keystoneauth1.identity' + + @mock.patch('keystoneauth1.session.Session.get_token', + return_value=VALID_TOKEN) + @mock.patch('%s.base.BaseIdentityPlugin.get_project_id' % + keystoneauth__identity, + return_value=PROJECT_ID) + @mock.patch('%s.generic.base.BaseGenericPlugin.get_auth_ref' % + keystoneauth__identity, + return_value=ROLES) + @mock.patch('pecan.request') + def test_header_parsing(self, req_mock, *args): + resp = self.post_json('/event', + params={ + 'time': datetime.now().isoformat(), + 'type': EVENT_TYPE, + 'details': EVENT_DETAILS + }, + headers=HEADERS) + req = resp.request + self.assertEqual('Confirmed', req.headers['X-Identity-Status']) + self.assertEqual(ROLES.role_names[0], req.headers['X-Roles']) + self.assertEqual(PROJECT_ID, req.headers['X-Project-Id']) + self.assertEqual(VALID_TOKEN, req.headers['X-Auth-Token']) + self.assertEqual(1, req_mock.client.call.call_count) + + @mock.patch('keystoneauth1.session.Session.request') + def test_basic_mode_auth_wrong_authorization(self, *args): + wrong_headers = HEADERS.copy() + wrong_headers['Authorization'] = 'Basic dml0cmFnZTpwdml0cmFnZT==' + resp = self.post_json('/event', + params={ + 'time': datetime.now().isoformat(), + 'type': EVENT_TYPE, + 'details': EVENT_DETAILS + }, + headers=wrong_headers, + expect_errors=True) + self.assertEqual(httplib.UNAUTHORIZED, resp.status_code) + self.assertDictEqual(ERR_MISSING_VERSIONED_IDENTITY_ENDPOINTS, + resp.json) + + def test_in_basic_mode_auth_no_header(self): + resp = self.post_json('/event', expect_errors=True) + + self.assertEqual(httplib.UNAUTHORIZED, resp.status_code) + self.assertDictEqual(ERR_MISSING_AUTH, resp.json) + + @mock.patch('keystoneauth1.identity.generic.password.Password') + @mock.patch('keystoneauth1.session.Session') + @mock.patch('pecan.request') + def test_in_basic_mode_auth_success(self, req_mock, *args): + resp = self.post_json('/event', + params={ + 'time': datetime.now().isoformat(), + 'type': EVENT_TYPE, + 'details': EVENT_DETAILS + }, + headers=HEADERS) + + self.assertEqual(1, req_mock.client.call.call_count) + self.assertEqual(httplib.OK, resp.status_code) diff --git a/vitrage/tests/resources/mock_configurations/driver/driver_prometheus_update_dynamic.json b/vitrage/tests/resources/mock_configurations/driver/driver_prometheus_update_dynamic.json index d60b44a72..0fd31fe54 100644 --- a/vitrage/tests/resources/mock_configurations/driver/driver_prometheus_update_dynamic.json +++ b/vitrage/tests/resources/mock_configurations/driver/driver_prometheus_update_dynamic.json @@ -1,21 +1,48 @@ { - "status": "firing", - "groupLabels": { - "alertname": "HighInodeUsage" - }, - "groupKey": "{}:{alertname=\"HighInodeUsage\"}", - "commonAnnotations": { - "mount_point": "/%", - "description": "\"Consider ssh\"ing into the instance and removing files or clean\ntemp files\"\n", - "title": "High number of inode usage", - "value": "96.81%", - "device": "/dev/vda1%", - "runbook": "troubleshooting/filesystem_alerts_inodes.md" - }, - "alerts": [ + "details": { "status": "firing", - "labels": { + "groupLabels": { + "alertname": "HighInodeUsage" + }, + "groupKey": "{}:{alertname=\"HighInodeUsage\"}", + "commonAnnotations": { + "mount_point": "/%", + "description": "\"Consider ssh\"ing into the instance and removing files or clean\ntemp files\"\n", + "title": "High number of inode usage", + "value": "96.81%", + "device": "/dev/vda1%", + "runbook": "troubleshooting/filesystem_alerts_inodes.md" + }, + "alerts": [ + { + "status": "firing", + "labels": { + "severity": "critical", + "fstype": "ext4", + "instance": "localhost:9100", + "job": "node", + "alertname": "HighInodeUsage", + "device": "/dev/vda1", + "mountpoint": "/" + }, + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://devstack-rocky-4:9090/graph?g0.expr=node_filesystem_files_free%7Bfstype%3D~%22%28ext.%7Cxfs%29%22%2Cjob%3D%22node%22%7D+%2F+node_filesystem_files%7Bfstype%3D~%22%28ext.%7Cxfs%29%22%2Cjob%3D%22node%22%7D+%2A+100+%3C%3D+100&g0.tab=1", + "startsAt": "2018-05-03T12:25:38.231388525Z", + "annotations": { + "mount_point": "/%", + "description": "\"Consider ssh\"ing into the instance and removing files or clean\ntemp files\"\n", + "title": "High number of inode usage", + "value": "96.81%", + "device": "/dev/vda1%", + "runbook": "troubleshooting/filesystem_alerts_inodes.md" + } + } + ], + "version": "4", + "receiver": "vitrage", + "externalURL": "http://devstack-rocky-4:9093", + "commonLabels": { "severity": "critical", "fstype": "ext4", "instance": "localhost:9100", @@ -23,30 +50,6 @@ "alertname": "HighInodeUsage", "device": "/dev/vda1", "mountpoint": "/" - }, - "endsAt": "0001-01-01T00:00:00Z", - "generatorURL": "http://devstack-rocky-4:9090/graph?g0.expr=node_filesystem_files_free%7Bfstype%3D~%22%28ext.%7Cxfs%29%22%2Cjob%3D%22node%22%7D+%2F+node_filesystem_files%7Bfstype%3D~%22%28ext.%7Cxfs%29%22%2Cjob%3D%22node%22%7D+%2A+100+%3C%3D+100&g0.tab=1", - "startsAt": "2018-05-03T12:25:38.231388525Z", - "annotations": { - "mount_point": "/%", - "description": "\"Consider ssh\"ing into the instance and removing files or clean\ntemp files\"\n", - "title": "High number of inode usage", - "value": "96.81%", - "device": "/dev/vda1%", - "runbook": "troubleshooting/filesystem_alerts_inodes.md" } } - ], - "version": "4", - "receiver": "vitrage", - "externalURL": "http://devstack-rocky-4:9093", - "commonLabels": { - "severity": "critical", - "fstype": "ext4", - "instance": "localhost:9100", - "job": "node", - "alertname": "HighInodeUsage", - "device": "/dev/vda1", - "mountpoint": "/" - } }