diff --git a/README.md b/README.md index d408794..b1b1f71 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ # python-observabilityclient observabilityclient is an OpenStackClient (OSC) plugin implementation that -implements commands for management of OpenStack observability components such -as Prometheus, collectd and Ceilometer. +implements commands for management of Prometheus. ## Development @@ -17,58 +16,37 @@ su - stack git clone https://github.com/infrawatch/python-observabilityclient cd python-observabilityclient sudo python setup.py install --prefix=/usr - -# clone and install observability playbooks and roles -git clone https://github.com/infrawatch/osp-observability-ansible -sudo mkdir /usr/share/osp-observability -sudo ln -s `pwd`/osp-observability-ansible/playbooks /usr/share/osp-observability/playbooks -sudo ln -s `pwd`/osp-observability-ansible/roles/spawn_container /usr/share/ansible/roles/spawn_container -sudo ln -s `pwd`/osp-observability-ansible/roles/osp_observability /usr/share/ansible/roles/osp_observability ``` -### Enable collectd write_prometheus -Create a THT environment file to enable the write_prometheus plugin for the collectd service. Then redeploy your overcloud and include this new file: +## Usage +Use `openstack metric query somequery` to query for metrics in prometheus. + +To use the python api do the following: ``` -mkdir -p ~/templates/observability -cat > templates/observability/collectd-write-prometheus.yaml -resource_registry: - OS::TripleO::Services::Collectd: /usr/share/openstack-tripleo-heat-templates/deployment/metrics/collectd-container-puppet.yaml +from observabilityclient import client -# TEST -# parameter_merge_strategies: -# CollectdExtraPlugins: merge - -parameter_defaults: - CollectdExtraPlugins: - - write_prometheus -EOF +c = client.Client( + '1', keystone_client.get_session(conf), + adapter_options={ + 'interface': conf.service_credentials.interface, + 'region_name': conf.service_credentials.region_name}) +c.query.query("somequery") ``` -### Discover endpoints -After deployment of your cloud you can discover endpoints available for scraping: +## List of commands -``` -source stackrc -openstack observability discover --stack-name=standalone -``` +openstack metric list - lists all metrics +openstack metric show - shows current values of a metric +openstack metric query - queries prometheus and outputs the result +openstack metric delete - deletes some metrics +openstack metric snapshot - takes a snapshot of the current data +openstack metric clean-tombstones - cleans the tsdb tombstones -### Deploy prometheus: -Create a config file and run the setup command - -``` -$ cat test_params.yaml -prometheus_remote_write: - stf: - url: https://default-prometheus-proxy-service-telemetry.apps.FAKE.ocp.cluster/api/v1/write - basic_user: internal - basic_pass: Pl4iNt3xTp4a55 - ca_cert: | - -----BEGIN CERTIFICATE----- - ABCDEFGHIJKLMNOPQRSTUVWXYZ - -----END CERTIFICATE----- - not-stf: - url: http://prometheus-rw.example.com/api/v1/write - -$ openstack observability setup prometheus_agent --config ./test_params.yaml -``` +## List of functions provided by the python library +c.query.list - lists all metrics +c.query.show - shows current values of a metric +c.query.query - queries prometheus and outputs the result +c.query.delete - deletes some metrics +c.query.snapshot - takes a snapshot of the current data +c.query.clean-tombstones - cleans the tsdb tombstones diff --git a/observabilityclient/client.py b/observabilityclient/client.py new file mode 100644 index 0000000..4bdd244 --- /dev/null +++ b/observabilityclient/client.py @@ -0,0 +1,22 @@ +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys + + +def Client(version, *args, **kwargs): + module = 'observabilityclient.v%s.client' % version + __import__(module) + client_class = getattr(sys.modules[module], 'Client') + return client_class(*args, **kwargs) diff --git a/observabilityclient/plugin.py b/observabilityclient/plugin.py index c3a01f6..e51f712 100644 --- a/observabilityclient/plugin.py +++ b/observabilityclient/plugin.py @@ -1,3 +1,16 @@ +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. """OpenStackClient Plugin interface""" @@ -8,7 +21,7 @@ DEFAULT_API_VERSION = '1' API_NAME = 'observabilityclient' API_VERSION_OPTION = 'os_observabilityclient_api_version' API_VERSIONS = { - '1': 'observabilityclient.plugin', + '1': 'observabilityclient.v1.client.Client', } @@ -20,12 +33,16 @@ def make_client(instance): :param ClientManager instance: The ClientManager that owns the new client """ - plugin_client = utils.get_client_class( + observability_client = utils.get_client_class( API_NAME, instance._api_version[API_NAME], API_VERSIONS) - client = plugin_client() + client = observability_client(session=instance.session, + adapter_options={ + 'interface': instance.interface, + 'region_name': instance.region_name + }) return client diff --git a/observabilityclient/prometheus_client.py b/observabilityclient/prometheus_client.py new file mode 100644 index 0000000..bba5b26 --- /dev/null +++ b/observabilityclient/prometheus_client.py @@ -0,0 +1,200 @@ +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import requests + + +LOG = logging.getLogger(__name__) + + +class PrometheusAPIClientError(Exception): + def __init__(self, response): + self.resp = response + + def __str__(self) -> str: + if self.resp.status_code != requests.codes.ok: + if self.resp.status_code != 204: + decoded = self.resp.json() + if 'error' in decoded: + return f'[{self.resp.status_code}] {decoded["error"]}' + return f'[{self.resp.status_code}] {self.resp.reason}' + else: + decoded = self.resp.json() + return f'[{decoded.status}]' + + def __repr__(self) -> str: + if self.resp.status_code != requests.codes.ok: + if self.resp.status_code != 204: + decoded = self.resp.json() + if 'error' in decoded: + return f'[{self.resp.status_code}] {decoded["error"]}' + return f'[{self.resp.status_code}] {self.resp.reason}' + else: + decoded = self.resp.json() + return f'[{decoded.status}]' + + +class PrometheusMetric: + def __init__(self, input): + self.timestamp = input['value'][0] + self.labels = input['metric'] + self.value = input['value'][1] + + +class PrometheusAPIClient: + def __init__(self, host): + self._host = host + self._session = requests.Session() + self._session.verify = False + + def set_ca_cert(self, ca_cert): + self._session.verify = ca_cert + + def set_client_cert(self, client_cert, client_key): + self._session.cert = client_cert + self._session.key = client_key + + def set_basic_auth(self, auth_user, auth_password): + self._session.auth = (auth_user, auth_password) + + def _get(self, endpoint, params=None): + url = (f"{'https' if self._session.verify else 'http'}://" + f"{self._host}/api/v1/{endpoint}") + resp = self._session.get(url, params=params, + headers={'Accept': 'application/json'}) + if resp.status_code != requests.codes.ok: + raise PrometheusAPIClientError(resp) + decoded = resp.json() + if decoded['status'] != 'success': + raise PrometheusAPIClientError(resp) + + return decoded + + def _post(self, endpoint, params=None): + url = (f"{'https' if self._session.verify else 'http'}://" + f"{self._host}/api/v1/{endpoint}") + resp = self._session.post(url, params=params, + headers={'Accept': 'application/json'}) + if resp.status_code != requests.codes.ok: + raise PrometheusAPIClientError(resp) + decoded = resp.json() + if 'status' in decoded and decoded['status'] != 'success': + raise PrometheusAPIClientError(resp) + return decoded + + def query(self, query): + """Sends custom queries to Prometheus + + :param query: the query to send + :type query: str + """ + + LOG.debug(f"Querying prometheus with query: {query}") + decoded = self._get("query", dict(query=query)) + + if decoded['data']['resultType'] == 'vector': + result = [PrometheusMetric(i) for i in decoded['data']['result']] + else: + result = [PrometheusMetric(decoded)] + return result + + def series(self, matches): + """Queries the /series/ endpoint of prometheus + + :param matches: List of matches to send as parameters + :type matches: [str] + """ + + LOG.debug(f"Querying prometheus for series with matches: {matches}") + decoded = self._get("series", {"match[]": matches}) + + return decoded['data'] + + def labels(self): + """Queries the /labels/ endpoint of prometheus, returns list of labels + + There isn't a way to tell prometheus to restrict + which labels to return. It's not possible to enforce + rbac with this for example. + """ + + LOG.debug("Querying prometheus for labels") + decoded = self._get("labels") + + return decoded['data'] + + def label_values(self, label): + """Queries prometheus for values of a specified label. + + :param label: Name of label for which to return values + :type label: str + """ + + LOG.debug(f"Querying prometheus for the values of label: {label}") + decoded = self._get(f"label/{label}/values") + + return decoded['data'] + + # --------- + # admin api + # --------- + + def delete(self, matches, start=None, end=None): + """Deletes some metrics from prometheus + + :param matches: List of matches, that specify which metrics to delete + :type matches [str] + :param start: Timestamp from which to start deleting. + None for as early as possible. + :type start: timestamp + :param end: Timestamp until which to delete. + None for as late as possible. + :type end: timestamp + """ + # NOTE Prometheus doesn't seem to return anything except + # of 204 status code. There doesn't seem to be a + # way to know if anything got actually deleted. + # It does however return 500 code and error msg + # if the admin APIs are disabled. + + LOG.debug(f"Deleting metrics from prometheus matching: {matches}") + try: + self._post("admin/tsdb/delete_series", {"match[]": matches, + "start": start, + "end": end}) + except PrometheusAPIClientError as exc: + # The 204 is allowed here. 204 is "No Content", + # which is expected on a successful call + if exc.resp.status_code != 204: + raise exc + + def clean_tombstones(self): + """Asks prometheus to clean tombstones""" + + LOG.debug("Cleaning tombstones from prometheus") + try: + self._post("admin/tsdb/clean_tombstones") + except PrometheusAPIClientError as exc: + # The 204 is allowed here. 204 is "No Content", + # which is expected on a successful call + if exc.resp.status_code != 204: + raise exc + + def snapshot(self): + """Creates a snapshot and returns the file name containing the data""" + + LOG.debug("Taking prometheus data snapshot") + ret = self._post("admin/tsdb/snapshot") + return ret["data"]["name"] diff --git a/observabilityclient/utils/metric_utils.py b/observabilityclient/utils/metric_utils.py new file mode 100644 index 0000000..632cef4 --- /dev/null +++ b/observabilityclient/utils/metric_utils.py @@ -0,0 +1,110 @@ +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os +import yaml + +from observabilityclient.prometheus_client import PrometheusAPIClient + +DEFAULT_CONFIG_LOCATIONS = [os.environ["HOME"] + "/.config/openstack/", + "/etc/openstack/"] +CONFIG_FILE_NAME = "prometheus.yaml" +LOG = logging.getLogger(__name__) + + +class ConfigurationError(Exception): + pass + + +def get_config_file(): + if os.path.exists(CONFIG_FILE_NAME): + LOG.debug(f"Using {CONFIG_FILE_NAME} as prometheus configuration") + return open(CONFIG_FILE_NAME, "r") + for path in DEFAULT_CONFIG_LOCATIONS: + full_filename = path + CONFIG_FILE_NAME + if os.path.exists(full_filename): + LOG.debug(f"Using {full_filename} as prometheus configuration") + return open(full_filename, "r") + return None + + +def get_prometheus_client(): + host = None + port = None + conf_file = get_config_file() + if conf_file is not None: + conf = yaml.safe_load(conf_file) + if 'host' in conf: + host = conf['host'] + if 'port' in conf: + port = conf['port'] + conf_file.close() + + # NOTE(jwysogla): We allow to overide the prometheus.yaml by + # the environment variables + if 'PROMETHEUS_HOST' in os.environ: + host = os.environ['PROMETHEUS_HOST'] + if 'PROMETHEUS_PORT' in os.environ: + port = os.environ['PROMETHEUS_PORT'] + if host is None or port is None: + raise ConfigurationError("Can't find prometheus host and " + "port configuration.") + return PrometheusAPIClient(f"{host}:{port}") + + +def get_client(obj): + return obj.app.client_manager.observabilityclient + + +def list2cols(cols, objs): + return cols, [tuple([o[k] for k in cols]) + for o in objs] + + +def format_labels(d: dict) -> str: + def replace_doubled_quotes(string): + if "''" in string: + string = string.replace("''", "'") + if '""' in string: + string = string.replace('""', '"') + return string + + ret = "" + for key, value in d.items(): + ret += "{}='{}', ".format(key, value) + ret = ret[0:-2] + old = "" + while ret != old: + old = ret + ret = replace_doubled_quotes(ret) + return ret + + +def metrics2cols(m): + cols = [] + fields = [] + first = True + for metric in m: + row = [] + for key, value in metric.labels.items(): + if first: + cols.append(key) + row.append(value) + if first: + cols.append("value") + row.append(metric.value) + fields.append(row) + first = False + return cols, fields diff --git a/observabilityclient/utils/runner.py b/observabilityclient/utils/runner.py deleted file mode 100644 index f049340..0000000 --- a/observabilityclient/utils/runner.py +++ /dev/null @@ -1,201 +0,0 @@ -# Copyright 2022 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - -import ansible_runner -import configparser -import os -import shutil - -from ansible.inventory.manager import InventoryManager -from ansible.parsing.dataloader import DataLoader -from ansible.vars.manager import VariableManager - -from observabilityclient.utils import shell - - -class AnsibleRunnerException(Exception): - """Base exception class for runner exceptions""" - - -class AnsibleRunnerFailed(AnsibleRunnerException): - """Raised when ansible run failed""" - - def __init__(self, status, rc, stderr): - super(AnsibleRunnerFailed).__init__() - self.status = status - self.rc = rc - self.stderr = stderr - - def __str__(self): - return ('Ansible run failed with status {}' - ' (return code {}):\n{}').format(self.status, self.rc, - self.stderr) - - -def parse_inventory_hosts(inventory): - """Returns list of dictionaries. Each dictionary contains info about - single node from inventory. - """ - dl = DataLoader() - if isinstance(inventory, str): - inventory = [inventory] - im = InventoryManager(loader=dl, sources=inventory) - vm = VariableManager(loader=dl, inventory=im) - - out = [] - for host in im.get_hosts(): - data = vm.get_vars(host=host) - out.append( - dict(host=data.get('inventory_hostname', str(host)), - ip=data.get('ctlplane_ip', data.get('ansible_host')), - hostname=data.get('canonical_hostname')) - ) - return out - - -class AnsibleRunner: - """Simple wrapper for ansible-playbook.""" - - def __init__(self, workdir: str, moduledir: str = None, - ssh_user: str = 'root', ssh_key: str = None, - ansible_cfg: str = None): - """ - :param workdir: Location of the working directory. - :type workdir: String - - :param ssh_user: User for the ssh connection. - :type ssh_user: String - - :param ssh_key: Private key to use for the ssh connection. - :type ssh_key: String - - :param moduledir: Location of the ansible module and library. - :type moduledir: String - - :param ansible_cfg: Path to an ansible configuration file. - :type ansible_cfg: String - """ - self.workdir = shell.file_check(workdir, ftype='directory') - - if moduledir is None: - moduledir = '' - ansible_cfg = ansible_cfg or os.path.join(workdir, 'ansible.cfg') - if not os.path.exists(ansible_cfg): - conf = dict( - ssh_connection=dict( - ssh_args=( - '-o UserKnownHostsFile={} ' - '-o StrictHostKeyChecking=no ' - '-o ControlMaster=auto ' - '-o ControlPersist=30m ' - '-o ServerAliveInterval=64 ' - '-o ServerAliveCountMax=1024 ' - '-o Compression=no ' - '-o TCPKeepAlive=yes ' - '-o VerifyHostKeyDNS=no ' - '-o ForwardX11=no ' - '-o ForwardAgent=yes ' - '-o PreferredAuthentications=publickey ' - '-T' - ).format(os.devnull), - retries=3, - timeout=30, - scp_if_ssh=True, - pipelining=True - ), - defaults=dict( - deprecation_warnings=False, - remote_user=ssh_user, - private_key_file=ssh_key, - library=os.path.expanduser( - '~/.ansible/plugins/modules:{workdir}/modules:' - '{userdir}:{ansible}/plugins/modules:' - '{ansible}-modules'.format( - userdir=moduledir, workdir=workdir, - ansible='/usr/share/ansible' - ) - ), - lookup_plugins=os.path.expanduser( - '~/.ansible/plugins/lookup:{workdir}/lookup:' - '{ansible}/plugins/lookup:'.format( - workdir=workdir, ansible='/usr/share/ansible' - ) - ), - gathering='smart', - log_path=shell.file_check( - os.path.join(workdir, 'ansible.log'), - clear=True - ) - ), - ) - parser = configparser.ConfigParser() - parser.read_dict(conf) - with open(ansible_cfg, 'w') as conffile: - parser.write(conffile) - os.environ['ANSIBLE_CONFIG'] = ansible_cfg - - def run(self, playbook, tags: str = None, skip_tags: str = None, - timeout: int = 30, quiet: bool = False, debug: bool = False): - """Run given Ansible playbook. - - :param playbook: Playbook filename. - :type playbook: String - - :param tags: Run specific tags. - :type tags: String - - :param skip_tags: Skip specific tags. - :type skip_tags: String - - :param timeout: Timeout to finish playbook execution (minutes). - :type timeout: int - - :param quiet: Disable all output (Defaults to False) - :type quiet: Boolean - - :param debug: Enable debug output (Defaults to False) - :type quiet: Boolean - """ - kwargs = { - 'private_data_dir': self.workdir, - 'verbosity': 3 if debug else 0, - } - locs = locals() - for arg in ['playbook', 'tags', 'skip_tags', 'quiet']: - if locs[arg] is not None: - kwargs[arg] = locs[arg] - run_conf = ansible_runner.runner_config.RunnerConfig(**kwargs) - run_conf.prepare() - run = ansible_runner.Runner(config=run_conf) - try: - status, rc = run.run() - finally: - if status in ['failed', 'timeout', 'canceled'] or rc != 0: - err = getattr(run, 'stderr', getattr(run, 'stdout', None)) - if err: - error = err.read() - else: - error = "Ansible failed with status %s" % status - raise AnsibleRunnerFailed(status, rc, error) - - def destroy(self, clear: bool = False): - """Cleans environment after Ansible run. - - :param clear: Clear also workdir - :type clear: Boolean - """ - del os.environ['ANSIBLE_CONFIG'] - if clear: - shutil.rmtree(self.workdir, ignore_errors=True) diff --git a/observabilityclient/utils/shell.py b/observabilityclient/utils/shell.py deleted file mode 100644 index 558d1af..0000000 --- a/observabilityclient/utils/shell.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2022 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - -import os -import pipes -import shutil -import subprocess -import tempfile - - -from contextlib import contextmanager -from observabilityclient.utils import strings - - -@contextmanager -def tempdir(base: str, prefix: str = None, clear: bool = True) -> str: - path = tempfile.mkdtemp(prefix=prefix, dir=base) - try: - yield path - finally: - if clear: - shutil.rmtree(path, ignore_errors=True) - - -def file_check(path: str, ftype: str = 'file', clear: bool = False) -> str: - """Check if given path exists and create it in case required.""" - if not os.path.exists(path) or clear: - if ftype == 'directory': - if clear: - shutil.rmtree(path, ignore_errors=True) - os.makedirs(path, mode=0o700, exist_ok=True) - elif ftype == 'file': - with open(path, 'w') as f: - f.close() - return path - - -def execute(cmd, workdir: str = None, can_fail: bool = True, - mask_list: list = None, use_shell: bool = False): - """ - Runs given shell command. Returns return code and content of stdout. - - :param workdir: Location of the working directory. - :type workdir: String - - :param can_fail: If is set to True RuntimeError is raised in case - of command returned non-zero return code. - :type can_fail: Boolean - """ - mask_list = mask_list or [] - - if not isinstance(cmd, str): - masked = ' '.join((pipes.quote(i) for i in cmd)) - else: - masked = cmd - masked = strings.mask_string(masked, mask_list) - - proc = subprocess.Popen(cmd, cwd=workdir, shell=use_shell, close_fds=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = proc.communicate() - if proc.returncode and can_fail: - raise RuntimeError('Failed to execute command: %s' % masked) - return proc.returncode, out, err diff --git a/observabilityclient/utils/strings.py b/observabilityclient/utils/strings.py deleted file mode 100644 index 3fde7f6..0000000 --- a/observabilityclient/utils/strings.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2022 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - - -STR_MASK = '*' * 8 -COLORS = {'nocolor': "\033[0m", - 'red': "\033[0;31m", - 'green': "\033[32m", - 'blue': "\033[34m", - 'yellow': "\033[33m"} - - -def color_text(text, color): - """Returns given text string with appropriate color tag. Allowed value - for color parameter is 'red', 'blue', 'green' and 'yellow'. - """ - return '%s%s%s' % (COLORS[color], text, COLORS['nocolor']) - - -def mask_string(unmasked, mask_list=None): - """Replaces words from mask_list with MASK in unmasked string.""" - mask_list = mask_list or [] - - masked = unmasked - for word in mask_list: - if not word: - continue - masked = masked.replace(word, STR_MASK) - return masked diff --git a/observabilityclient/v1/base.py b/observabilityclient/v1/base.py index 746a44c..473d5ef 100644 --- a/observabilityclient/v1/base.py +++ b/observabilityclient/v1/base.py @@ -13,24 +13,12 @@ # under the License. # -import os -import shutil - from osc_lib.command import command from osc_lib.i18n import _ -from observabilityclient.utils import runner -from observabilityclient.utils import shell - - -OBSLIBDIR = shell.file_check('/usr/share/osp-observability', 'directory') -OBSWRKDIR = shell.file_check( - os.path.expanduser('~/.osp-observability'), 'directory' -) - class ObservabilityBaseCommand(command.Command): - """Base class for observability commands.""" + """Base class for metric commands.""" def get_parser(self, prog_name): parser = super().get_parser(prog_name) @@ -44,68 +32,50 @@ class ObservabilityBaseCommand(command.Command): action='store_true', help=_("Disable cleanup of temporary files.") ) + + # TODO(jwysogla): Should this be restricted somehow? parser.add_argument( - '--workdir', - default=OBSWRKDIR, - help=_("Working directory for observability commands.") - ) - parser.add_argument( - '--moduledir', - default=None, - help=_("Directory with additional Ansible modules.") - ) - parser.add_argument( - '--ssh-user', - default='heat-admin', - help=_("Username to be used for SSH connection.") - ) - parser.add_argument( - '--ssh-key', - default='/home/stack/.ssh/id_rsa', - help=_("SSH private key to be used for SSH connection.") - ) - parser.add_argument( - '--ansible-cfg', - default=os.path.join(OBSWRKDIR, 'ansible.cfg'), - help=_("Path to Ansible configuration.") - ) - parser.add_argument( - '--config', - default=None, - help=_("Path to playbook configuration file.") + '--disable-rbac', + action='store_true', + help=_("Disable rbac injection") ) return parser - def _run_playbook(self, playbook, inventory, parsed_args): - """Run Ansible raw playbook""" - playbook = os.path.join(OBSLIBDIR, 'playbooks', playbook) - with shell.tempdir(parsed_args.workdir, - prefix=os.path.splitext(playbook)[0], - clear=not parsed_args.messy) as tmpdir: - # copy extravars file for the playbook run - if parsed_args.config: - envdir = shell.file_check(os.path.join(tmpdir, 'env'), - 'directory') - shutil.copy(parsed_args.config, - os.path.join(envdir, 'extravars')) - # copy inventory file for the playbook run - shutil.copy(inventory, os.path.join(tmpdir, 'inventory')) - # run playbook - rnr = runner.AnsibleRunner(tmpdir, - moduledir=parsed_args.moduledir, - ssh_user=parsed_args.ssh_user, - ssh_key=parsed_args.ssh_key, - ansible_cfg=parsed_args.ansible_cfg) - if parsed_args.messy: - print("Running playbook %s" % playbook) - rnr.run(playbook, debug=parsed_args.dev) - rnr.destroy(clear=not parsed_args.messy) - def _execute(self, command, parsed_args): - """Execute local command""" - with shell.tempdir(parsed_args.workdir, prefix='exec', - clear=not parsed_args.messy) as tmpdir: - rc, out, err = shell.execute(command, workdir=tmpdir, - can_fail=parsed_args.dev, - use_shell=True) - return rc, out, err +class Manager(object): + """Base class for the python api.""" + DEFAULT_HEADERS = { + "Accept": "application/json", + } + + def __init__(self, client): + self.client = client + self.prom = client.prometheus_client + + def _set_default_headers(self, kwargs): + headers = kwargs.get('headers', {}) + for k, v in self.DEFAULT_HEADERS.items(): + if k not in headers: + headers[k] = v + kwargs['headers'] = headers + return kwargs + + def _get(self, *args, **kwargs): + self._set_default_headers(kwargs) + return self.client.api.get(*args, **kwargs) + + def _post(self, *args, **kwargs): + self._set_default_headers(kwargs) + return self.client.api.post(*args, **kwargs) + + def _put(self, *args, **kwargs): + self._set_default_headers(kwargs) + return self.client.api.put(*args, **kwargs) + + def _patch(self, *args, **kwargs): + self._set_default_headers(kwargs) + return self.client.api.patch(*args, **kwargs) + + def _delete(self, *args, **kwargs): + self._set_default_headers(kwargs) + return self.client.api.delete(*args, **kwargs) diff --git a/observabilityclient/v1/cli.py b/observabilityclient/v1/cli.py new file mode 100644 index 0000000..66fde53 --- /dev/null +++ b/observabilityclient/v1/cli.py @@ -0,0 +1,109 @@ +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from observabilityclient.utils import metric_utils +from observabilityclient.v1 import base +from osc_lib.i18n import _ + +from cliff import lister + + +class List(base.ObservabilityBaseCommand, lister.Lister): + """Query prometheus for list of all metrics""" + + def take_action(self, parsed_args): + client = metric_utils.get_client(self) + metrics = client.query.list(disable_rbac=parsed_args.disable_rbac) + return ["metric_name"], [[m] for m in metrics] + + +class Show(base.ObservabilityBaseCommand, lister.Lister): + """Query prometheus for the current value of metric""" + + def get_parser(self, prog_name): + parser = super().get_parser(prog_name) + parser.add_argument( + 'name', + help=_("Name of the metric to show")) + return parser + + def take_action(self, parsed_args): + client = metric_utils.get_client(self) + metric = client.query.show(parsed_args.name, + disable_rbac=parsed_args.disable_rbac) + return metric_utils.metrics2cols(metric) + + +class Query(base.ObservabilityBaseCommand, lister.Lister): + """Query prometheus with a custom query string""" + + def get_parser(self, prog_name): + parser = super().get_parser(prog_name) + parser.add_argument( + 'query', + help=_("Custom PromQL query")) + return parser + + def take_action(self, parsed_args): + client = metric_utils.get_client(self) + metric = client.query.query(parsed_args.query, + disable_rbac=parsed_args.disable_rbac) + ret = metric_utils.metrics2cols(metric) + return ret + + +class Delete(base.ObservabilityBaseCommand): + """Delete data for a selected series and time range""" + def get_parser(self, prog_name): + parser = super().get_parser(prog_name) + parser.add_argument( + 'matches', + action="append", + nargs='+', + help=_("Series selector, that selects the series to delete. " + "Specify multiple selectors delimited by space to " + "delete multiple series.")) + parser.add_argument( + '--start', + help=_("Start timestamp in rfc3339 or unix timestamp. " + "Defaults to minimum possible timestamp.")) + parser.add_argument( + '--end', + help=_("End timestamp in rfc3339 or unix timestamp. " + "Defaults to maximum possible timestamp.")) + return parser + + def take_action(self, parsed_args): + client = metric_utils.get_client(self) + return client.query.delete(parsed_args.matches, + parsed_args.start, + parsed_args.end) + + +class CleanTombstones(base.ObservabilityBaseCommand): + """Remove deleted data from disk and clean up the existing tombstones""" + def get_parser(self, prog_name): + parser = super().get_parser(prog_name) + return parser + + def take_action(self, parsed_args): + client = metric_utils.get_client(self) + return client.query.clean_tombstones() + + +class Snapshot(base.ObservabilityBaseCommand, lister.Lister): + def take_action(self, parsed_args): + client = metric_utils.get_client(self) + ret = client.query.snapshot() + return ["Snapshot file name"], [[ret]] diff --git a/observabilityclient/v1/client.py b/observabilityclient/v1/client.py new file mode 100644 index 0000000..9c12722 --- /dev/null +++ b/observabilityclient/v1/client.py @@ -0,0 +1,43 @@ +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import keystoneauth1.session + +from observabilityclient.utils.metric_utils import get_prometheus_client +from observabilityclient.v1 import python_api +from observabilityclient.v1 import rbac + + +class Client(object): + """Client for the observabilityclient api""" + + def __init__(self, session=None, adapter_options=None, + session_options=None, disable_rbac=False): + """Initialize a new client for the Observabilityclient v1 API.""" + session_options = session_options or {} + adapter_options = adapter_options or {} + + adapter_options.setdefault('service_type', "metric") + + if session is None: + session = keystoneauth1.session.Session(**session_options) + else: + if session_options: + raise ValueError("session and session_options are exclusive") + + self.session = session + + self.prometheus_client = get_prometheus_client() + self.query = python_api.QueryManager(self) + self.rbac = rbac.Rbac(self, self.session, disable_rbac) diff --git a/observabilityclient/v1/deploy.py b/observabilityclient/v1/deploy.py deleted file mode 100644 index c550848..0000000 --- a/observabilityclient/v1/deploy.py +++ /dev/null @@ -1,180 +0,0 @@ -# Copyright 2022 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - -import os -import requests -import shutil -import sys -import yaml - -from osc_lib.i18n import _ - -from observabilityclient.v1 import base -from observabilityclient.utils import runner - - -class InventoryError(Exception): - def __init__(self, err, out): - self.err = err - self.out = out - - def __str__(self): - return ('Failed to generate or locate Ansible ' - 'inventory file:\n%s\n%s' % (self.err or '', self.out)) - - -INVENTORY = os.path.join(base.OBSWRKDIR, 'openstack-inventory.yaml') -INV_FALLBACKS = [ - '~/tripleo-deploy/{stack}/openstack-inventory.yaml', - '~/tripleo-deploy/{stack}/tripleo-ansible-inventory.yaml', - './overcloud-deploy/{stack}/openstack-inventory.yaml', - './overcloud-deploy/{stack}/tripleo-ansible-inventory.yaml', -] -ENDPOINTS = os.path.join(base.OBSWRKDIR, 'scrape-endpoints.yaml') -STACKRC = os.path.join(base.OBSWRKDIR, 'stackrc') - - -def _curl(host: dict, port: int, timeout: int = 1) -> str: - """Returns scraping endpoint URL if it is reachable - otherwise returns None.""" - url = f'http://{host["ip"]}:{port}/metrics' - try: - r = requests.get(url, timeout=1) - if r.status_code != 200: - url = None - r.close() - except requests.exceptions.ConnectionError: - url = None - return url - - -class Discover(base.ObservabilityBaseCommand): - """Generate Ansible inventory file and scrapable enpoints list file.""" - - def get_parser(self, prog_name): - parser = super().get_parser(prog_name) - parser.add_argument( - '--scrape', - action='append', - default=['collectd/9103'], - help=_("Service/Port of scrape endpoint to check on nodes") - ) - parser.add_argument( - '--stack-name', - default='overcloud', - help=_("Overcloud stack name for which inventory file should " - "be generated") - ) - parser.add_argument( - '--inventory', - help=_("Use this argument in case you have inventory file " - "generated or moved to non-standard place. Value has to be " - "path to inventory file including the file name.") - ) - return parser - - def take_action(self, parsed_args): - # discover undercloud and overcloud nodes - try: - rc, out, err = self._execute( - 'tripleo-ansible-inventory ' - '--static-yaml-inventory {} ' - '--stack {}'.format(INVENTORY, parsed_args.stack_name), - parsed_args - ) - if rc: - raise InventoryError(err, out) - - # OSP versions with deprecated tripleo-ansible-inventory fallbacks - # to static inventory file generated at one of the fallback path - if not os.path.exists(INVENTORY): - if parsed_args.inventory: - INV_FALLBACKS.insert(0, parsed_args.inventory) - for i in INV_FALLBACKS: - absi = i.format(stack=parsed_args.stack_name) - absi = os.path.abspath(os.path.expanduser(absi)) - if os.path.exists(absi): - shutil.copyfile(absi, INVENTORY) - break - else: - raise InventoryError('None of the fallback inventory files' - ' exists: %s' % INV_FALLBACKS, '') - except InventoryError as ex: - print(str(ex)) - sys.exit(1) - - # discover scrape endpoints - endpoints = dict() - hosts = runner.parse_inventory_hosts(INVENTORY) - for scrape in parsed_args.scrape: - service, port = scrape.split('/') - for host in hosts: - if parsed_args.dev: - name = host["hostname"] if host["hostname"] else host["ip"] - print(f'Trying to fetch {service} metrics on host ' - f'{name} at port {port}', end='') - node = _curl(host, port, timeout=1) - if node: - endpoints.setdefault(service.strip(), []).append(node) - if parsed_args.dev: - print(' [success]' if node else ' [failure]') - data = yaml.safe_dump(endpoints, default_flow_style=False) - with open(ENDPOINTS, 'w') as f: - f.write(data) - print("Discovered following scraping endpoints:\n%s" % data) - - -class Setup(base.ObservabilityBaseCommand): - """Install and configure given Observability component(s)""" - - auth_required = False - - def get_parser(self, prog_name): - parser = super().get_parser(prog_name) - parser.add_argument( - 'components', - nargs='+', - choices=[ - 'prometheus_agent', - # TODO: in future will contain option for all stack components - ] - ) - parser.add_argument( - '--inventory', - help=_("Use this argument in case you don't want to use for " - "whatever reason the inventory file generated by discovery " - "command") - ) - return parser - - def take_action(self, parsed_args): - inventory = INVENTORY - if parsed_args.inventory: - inventory = parsed_args.inventory - for compnt in parsed_args.components: - playbook = '%s.yml' % compnt - try: - self._run_playbook(playbook, inventory, - parsed_args=parsed_args) - except OSError as ex: - print('Failed to load playbook file: %s' % ex) - sys.exit(1) - except yaml.YAMLError as ex: - print('Failed to parse playbook configuration: %s' % ex) - sys.exit(1) - except runner.AnsibleRunnerFailed as ex: - print('Ansible run %s (rc %d)' % (ex.status, ex.rc)) - if parsed_args.dev: - print(ex.stderr) diff --git a/observabilityclient/v1/python_api.py b/observabilityclient/v1/python_api.py new file mode 100644 index 0000000..5bba4c8 --- /dev/null +++ b/observabilityclient/v1/python_api.py @@ -0,0 +1,96 @@ +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from observabilityclient.utils.metric_utils import format_labels +from observabilityclient.v1 import base + + +class QueryManager(base.Manager): + def list(self, disable_rbac=False): + """Lists metric names + + :param disable_rbac: Disables rbac injection if set to True + :type disable_rbac: boolean + """ + if disable_rbac or self.client.rbac.disable_rbac: + metric_names = self.prom.label_values("__name__") + return metric_names + else: + match = f"{{{format_labels(self.client.rbac.default_labels)}}}" + metrics = self.prom.series(match) + if metrics == []: + return [] + unique_metric_names = list(set([m['__name__'] for m in metrics])) + return sorted(unique_metric_names) + + def show(self, name, disable_rbac=False): + """Shows current values for metrics of a specified name + + :param disable_rbac: Disables rbac injection if set to True + :type disable_rbac: boolean + """ + enriched = self.client.rbac.append_rbac(name, + disable_rbac=disable_rbac) + last_metric_query = f"last_over_time({enriched}[5m])" + return self.prom.query(last_metric_query) + + def query(self, query, disable_rbac=False): + """Sends a query to prometheus + + The query can be any PromQL query. Labels for enforcing + rbac will be added to all of the metric name inside the query. + Having labels as part of a query is allowed. + + A call like this: + query("sum(name1) - sum(name2{label1='value'})") + will result in a query string like this: + "sum(name1{rbac='rbac_value'}) - + sum(name2{label1='value', rbac='rbac_value'})" + + :param query: Custom query string + :type query: str + :param disable_rbac: Disables rbac injection if set to True + :type disable_rbac: boolean + """ + query = self.client.rbac.enrich_query(query, disable_rbac) + return self.prom.query(query) + + def delete(self, matches, start=None, end=None): + """Deletes metrics from Prometheus + + The metrics aren't deleted immediately. Do a call to clean_tombstones() + to speed up the deletion. If start and end isn't specified, then + minimum and maximum timestamps are used. + + :param matches: List of matches to match which metrics to delete + :type matches: [str] + :param start: timestamp from which to start deleting + :type start: rfc3339 or unix_timestamp + :param end: timestamp until which to delete + :type end: rfc3339 or unix_timestamp + """ + # TODO(jwysogla) Do we want to restrict access to the admin api + # endpoints? We could either try to inject + # the project label like in query. We could also + # do some check right here, before + # it gets to prometheus. + return self.prom.delete(matches, start, end) + + def clean_tombstones(self): + """Instructs prometheus to clean tombstones""" + return self.prom.clean_tombstones() + + def snapshot(self): + "Creates a snapshot of the current data" + return self.prom.snapshot() diff --git a/observabilityclient/v1/rbac.py b/observabilityclient/v1/rbac.py new file mode 100644 index 0000000..db17c01 --- /dev/null +++ b/observabilityclient/v1/rbac.py @@ -0,0 +1,139 @@ +# Copyright 2023 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from keystoneauth1.exceptions.auth_plugins import MissingAuthPlugin +from observabilityclient.utils.metric_utils import format_labels +import re + + +class ObservabilityRbacError(Exception): + pass + + +class Rbac(): + def __init__(self, client, session, disable_rbac=False): + self.client = client + self.session = session + self.disable_rbac = disable_rbac + try: + self.project_id = self.session.get_project_id() + self.default_labels = { + "project": self.project_id + } + self.rbac_init_successful = True + except MissingAuthPlugin: + self.project_id = None + self.default_labels = { + "project": "no-project" + } + self.rbac_init_successful = False + + def _find_label_value_end(self, query, start, quote_char): + end = start + while (end == start or + query[end - 1] == '\\'): + # Looking for first unescaped quotes + end = query.find(quote_char, end + 1) + # returns the quote position or -1 + return end + + def _find_label_pair_end(self, query, start): + eq_sign_pos = query.find('=', start) + quote_char = "'" + quote_start_pos = query.find(quote_char, eq_sign_pos) + if quote_start_pos == -1: + quote_char = '"' + quote_start_pos = query.find(quote_char, eq_sign_pos) + end = self._find_label_value_end(query, quote_start_pos, quote_char) + # returns the pair end or -1 + return end + + def _find_label_section_end(self, query, start): + nearest_curly_brace_pos = None + while nearest_curly_brace_pos != -1: + pair_end = self._find_label_pair_end(query, start) + nearest_curly_brace_pos = query.find("}", pair_end) + nearest_eq_sign_pos = query.find("=", pair_end) + if (nearest_curly_brace_pos < nearest_eq_sign_pos or + nearest_eq_sign_pos == -1): + # If we have "}" before the nearest "=", + # then we must be at the end of the label section + # and the "=" is a part of the next section. + return nearest_curly_brace_pos + start = pair_end + return -1 + + def enrich_query(self, query, disable_rbac=False): + """Used to add rbac labels to queries + + :param query: The query to enrich + :type query: str + :param disable_rbac: Disables rbac injection if set to True + :type disable_rbac: boolean + """ + # TODO(jwysogla): This should be properly tested + if disable_rbac: + return query + labels = self.default_labels + + # We need to get all metric names, no matter the rbac + metric_names = self.client.query.list(disable_rbac=False) + + # We need to detect the locations of metric names + # inside the query + # NOTE the locations are the locations within the original query + name_end_locations = [] + for name in metric_names: + # Regex for a metric name is: [a-zA-Z_:][a-zA-Z0-9_:]* + # We need to make sure, that "name" isn't just a part + # of a longer word, so we try to expand it by "name_regex" + name_regex = "[a-zA-Z_:]?[a-zA-Z0-9_:]*" + name + "[a-zA-Z0-9_:]*" + potential_names = re.finditer(name_regex, query) + for potential_name in potential_names: + if potential_name.group(0) == name: + name_end_locations.append(potential_name.end()) + + name_end_locations = sorted(name_end_locations, reverse=True) + for name_end_location in name_end_locations: + if (name_end_location < len(query) and + query[name_end_location] == "{"): + # There already are some labels + labels_end = self._find_label_section_end(query, + name_end_location) + query = (f"{query[:labels_end]}, " + f"{format_labels(labels)}" + f"{query[labels_end:]}") + else: + query = (f"{query[:name_end_location]}" + f"{{{format_labels(labels)}}}" + f"{query[name_end_location:]}") + return query + + def append_rbac(self, query, disable_rbac=False): + """Used to append rbac labels to queries + + It's a simplified and faster version of enrich_query(). This just + appends the labels at the end of the query string. For proper handling + of complex queries, where metric names might occure elsewhere than + just at the end, please use the enrich_query() function. + + :param query: The query to append to + :type query: str + :param disable_rbac: Disables rbac injection if set to True + :type disable_rbac: boolean + """ + labels = self.default_labels + if disable_rbac: + return query + return f"{query}{{{format_labels(labels)}}}" diff --git a/setup.cfg b/setup.cfg index a092d5e..0e50b18 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ name = python-observabilityclient summary = OpenStack Observability Client description_file = - README.rst + README.md license = Apache License, Version 2.0 author = OpenStack author_email = openstack-discuss@lists.openstack.org @@ -35,12 +35,12 @@ openstack.cli.extension = observabilityclient = observabilityclient.plugin openstack.observabilityclient.v1 = - observability_discover = observabilityclient.v1.deploy:Discover - observability_setup = observabilityclient.v1.deploy:Setup -# observability_upgrade = observabilityclient.v1.deploy:Upgrade - -# metrics_list = observabilityclient.v1.metrics:List -# metrics_get = observabilityclient.v1.metrics:Get + metric_list = observabilityclient.v1.cli:List + metric_show = observabilityclient.v1.cli:Show + metric_query = observabilityclient.v1.cli:Query + metric_delete = observabilityclient.v1.cli:Delete + metric_clean-tombstones = observabilityclient.v1.cli:CleanTombstones + metric_snapshot = observabilityclient.v1.cli:Snapshot [flake8]