2023-08-03 15:30:19 +02:00
|
|
|
# Copyright 2023 Red Hat, Inc.
|
|
|
|
#
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
|
|
# not use this file except in compliance with the License. You may obtain
|
|
|
|
# a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
|
|
# License for the specific language governing permissions and limitations
|
|
|
|
# under the License.
|
|
|
|
|
|
|
|
import logging
|
|
|
|
import os
|
2023-09-05 14:54:33 +02:00
|
|
|
|
2023-08-03 15:30:19 +02:00
|
|
|
import yaml
|
|
|
|
|
|
|
|
from observabilityclient.prometheus_client import PrometheusAPIClient
|
|
|
|
|
2023-09-05 14:54:33 +02:00
|
|
|
|
2023-08-03 15:30:19 +02:00
|
|
|
DEFAULT_CONFIG_LOCATIONS = [os.environ["HOME"] + "/.config/openstack/",
|
|
|
|
"/etc/openstack/"]
|
|
|
|
CONFIG_FILE_NAME = "prometheus.yaml"
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
|
|
|
|
class ConfigurationError(Exception):
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
def get_config_file():
|
|
|
|
if os.path.exists(CONFIG_FILE_NAME):
|
2023-09-05 14:54:33 +02:00
|
|
|
LOG.debug("Using %s as prometheus configuration", CONFIG_FILE_NAME)
|
2023-08-03 15:30:19 +02:00
|
|
|
return open(CONFIG_FILE_NAME, "r")
|
|
|
|
for path in DEFAULT_CONFIG_LOCATIONS:
|
|
|
|
full_filename = path + CONFIG_FILE_NAME
|
|
|
|
if os.path.exists(full_filename):
|
2023-09-05 14:54:33 +02:00
|
|
|
LOG.debug("Using %s as prometheus configuration", full_filename)
|
2023-08-03 15:30:19 +02:00
|
|
|
return open(full_filename, "r")
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
|
|
def get_prometheus_client():
|
|
|
|
host = None
|
|
|
|
port = None
|
2024-02-26 08:15:15 -05:00
|
|
|
ca_cert = None
|
2023-08-03 15:30:19 +02:00
|
|
|
conf_file = get_config_file()
|
|
|
|
if conf_file is not None:
|
|
|
|
conf = yaml.safe_load(conf_file)
|
|
|
|
if 'host' in conf:
|
|
|
|
host = conf['host']
|
|
|
|
if 'port' in conf:
|
|
|
|
port = conf['port']
|
2024-02-26 08:15:15 -05:00
|
|
|
if 'ca_cert' in conf:
|
|
|
|
ca_cert = conf['ca_cert']
|
2023-08-03 15:30:19 +02:00
|
|
|
conf_file.close()
|
|
|
|
|
|
|
|
# NOTE(jwysogla): We allow to overide the prometheus.yaml by
|
|
|
|
# the environment variables
|
|
|
|
if 'PROMETHEUS_HOST' in os.environ:
|
|
|
|
host = os.environ['PROMETHEUS_HOST']
|
|
|
|
if 'PROMETHEUS_PORT' in os.environ:
|
|
|
|
port = os.environ['PROMETHEUS_PORT']
|
2024-02-26 08:15:15 -05:00
|
|
|
if 'PROMETHEUS_CA_CERT' in os.environ:
|
|
|
|
ca_cert = os.environ['PROMETHEUS_CA_CERT']
|
2023-08-03 15:30:19 +02:00
|
|
|
if host is None or port is None:
|
|
|
|
raise ConfigurationError("Can't find prometheus host and "
|
|
|
|
"port configuration.")
|
2024-02-26 08:15:15 -05:00
|
|
|
client = PrometheusAPIClient(f"{host}:{port}")
|
|
|
|
if ca_cert is not None:
|
|
|
|
client.set_ca_cert(ca_cert)
|
|
|
|
return client
|
2023-08-03 15:30:19 +02:00
|
|
|
|
|
|
|
|
|
|
|
def get_client(obj):
|
|
|
|
return obj.app.client_manager.observabilityclient
|
|
|
|
|
|
|
|
|
|
|
|
def format_labels(d: dict) -> str:
|
|
|
|
def replace_doubled_quotes(string):
|
|
|
|
if "''" in string:
|
|
|
|
string = string.replace("''", "'")
|
|
|
|
if '""' in string:
|
|
|
|
string = string.replace('""', '"')
|
|
|
|
return string
|
|
|
|
|
|
|
|
ret = ""
|
|
|
|
for key, value in d.items():
|
|
|
|
ret += "{}='{}', ".format(key, value)
|
|
|
|
ret = ret[0:-2]
|
|
|
|
old = ""
|
|
|
|
while ret != old:
|
|
|
|
old = ret
|
|
|
|
ret = replace_doubled_quotes(ret)
|
|
|
|
return ret
|
|
|
|
|
|
|
|
|
|
|
|
def metrics2cols(m):
|
Fix table formatter
It's possible for Prometheus to return multiple metrics, where
each of them has different labels. Before this the client
would fail with error when using the default table formatter,
because of for example:
Row has incorrect number of values, (actual) 8!=9 (expected)
It was also possible to mismatch the label keys and values
for some of the rows.
This patch makes sure the label values are matched to the correct
keys. It also makes sure it uses all labels and it fills
missing values with empty strings for metrics, which don't have
that label defined.
Example table output:
+------------+----------+------------+----------------+-------+
| group | __name__ | job | instance | value |
+------------+----------+------------+----------------+-------+
| | up | sg-core | localhost:3000 | 1 |
| production | up | prometheus | localhost:9090 | 1 |
+------------+----------+------------+----------------+-------+
Example json output:
[
{
"__name__": "up",
"group": "",
"job": "sg-core",
"instance": "localhost:3000",
"value": "1"
},
{
"__name__": "up",
"group": "production",
"job": "prometheus",
"instance": "localhost:9090",
"value": "1"
}
]
Change-Id: Id0dfabf52fe0a21194c498f4aefa1bff1d3eeea9
2024-03-21 04:56:36 -04:00
|
|
|
# get all label keys
|
|
|
|
cols = list(set().union(*(d.labels.keys() for d in m)))
|
2024-04-10 02:13:13 -04:00
|
|
|
cols.sort()
|
Fix table formatter
It's possible for Prometheus to return multiple metrics, where
each of them has different labels. Before this the client
would fail with error when using the default table formatter,
because of for example:
Row has incorrect number of values, (actual) 8!=9 (expected)
It was also possible to mismatch the label keys and values
for some of the rows.
This patch makes sure the label values are matched to the correct
keys. It also makes sure it uses all labels and it fills
missing values with empty strings for metrics, which don't have
that label defined.
Example table output:
+------------+----------+------------+----------------+-------+
| group | __name__ | job | instance | value |
+------------+----------+------------+----------------+-------+
| | up | sg-core | localhost:3000 | 1 |
| production | up | prometheus | localhost:9090 | 1 |
+------------+----------+------------+----------------+-------+
Example json output:
[
{
"__name__": "up",
"group": "",
"job": "sg-core",
"instance": "localhost:3000",
"value": "1"
},
{
"__name__": "up",
"group": "production",
"job": "prometheus",
"instance": "localhost:9090",
"value": "1"
}
]
Change-Id: Id0dfabf52fe0a21194c498f4aefa1bff1d3eeea9
2024-03-21 04:56:36 -04:00
|
|
|
cols.append("value")
|
2023-08-03 15:30:19 +02:00
|
|
|
fields = []
|
|
|
|
for metric in m:
|
Fix table formatter
It's possible for Prometheus to return multiple metrics, where
each of them has different labels. Before this the client
would fail with error when using the default table formatter,
because of for example:
Row has incorrect number of values, (actual) 8!=9 (expected)
It was also possible to mismatch the label keys and values
for some of the rows.
This patch makes sure the label values are matched to the correct
keys. It also makes sure it uses all labels and it fills
missing values with empty strings for metrics, which don't have
that label defined.
Example table output:
+------------+----------+------------+----------------+-------+
| group | __name__ | job | instance | value |
+------------+----------+------------+----------------+-------+
| | up | sg-core | localhost:3000 | 1 |
| production | up | prometheus | localhost:9090 | 1 |
+------------+----------+------------+----------------+-------+
Example json output:
[
{
"__name__": "up",
"group": "",
"job": "sg-core",
"instance": "localhost:3000",
"value": "1"
},
{
"__name__": "up",
"group": "production",
"job": "prometheus",
"instance": "localhost:9090",
"value": "1"
}
]
Change-Id: Id0dfabf52fe0a21194c498f4aefa1bff1d3eeea9
2024-03-21 04:56:36 -04:00
|
|
|
row = [""] * len(cols)
|
2023-08-03 15:30:19 +02:00
|
|
|
for key, value in metric.labels.items():
|
Fix table formatter
It's possible for Prometheus to return multiple metrics, where
each of them has different labels. Before this the client
would fail with error when using the default table formatter,
because of for example:
Row has incorrect number of values, (actual) 8!=9 (expected)
It was also possible to mismatch the label keys and values
for some of the rows.
This patch makes sure the label values are matched to the correct
keys. It also makes sure it uses all labels and it fills
missing values with empty strings for metrics, which don't have
that label defined.
Example table output:
+------------+----------+------------+----------------+-------+
| group | __name__ | job | instance | value |
+------------+----------+------------+----------------+-------+
| | up | sg-core | localhost:3000 | 1 |
| production | up | prometheus | localhost:9090 | 1 |
+------------+----------+------------+----------------+-------+
Example json output:
[
{
"__name__": "up",
"group": "",
"job": "sg-core",
"instance": "localhost:3000",
"value": "1"
},
{
"__name__": "up",
"group": "production",
"job": "prometheus",
"instance": "localhost:9090",
"value": "1"
}
]
Change-Id: Id0dfabf52fe0a21194c498f4aefa1bff1d3eeea9
2024-03-21 04:56:36 -04:00
|
|
|
row[cols.index(key)] = value
|
|
|
|
row[cols.index("value")] = metric.value
|
2023-08-03 15:30:19 +02:00
|
|
|
fields.append(row)
|
|
|
|
return cols, fields
|