Initial functionality (#1)

This patch adds basic functionality of the plugin. It successfully registers as openstackclient plugin and contains two basic observability commands:

   - discover
      - prepares ansible inventory file with overcloud and undercloud nodes and gather data for prometheus agent according to which nodes are scrapable
   - setup
      - starts proper ansible playbook based on component (currently only prometheus_agent is available)

Co-authored-by: Marihan Girgis mgirgisf@redhat.com
Partially-Implements: OSP-14664
Related: infrawatch/osp-observability-ansible#11

observabilityclient/plugin.py

@@ -0,0 +1,48 @@
+
+"""OpenStackClient Plugin interface"""
+
+from osc_lib import utils
+
+
+DEFAULT_API_VERSION = '1'
+API_NAME = 'observabilityclient'
+API_VERSION_OPTION = 'os_observabilityclient_api_version'
+API_VERSIONS = {
+    '1': 'observabilityclient.plugin',
+}
+
+
+def make_client(instance):
+    """Returns a client to the ClientManager
+
+    Called to instantiate the requested client version.  instance has
+    any available auth info that may be required to prepare the client.
+
+    :param ClientManager instance: The ClientManager that owns the new client
+    """
+    plugin_client = utils.get_client_class(
+        API_NAME,
+        instance._api_version[API_NAME],
+        API_VERSIONS)
+
+    client = plugin_client()
+    return client
+
+
+def build_option_parser(parser):
+    """Hook to add global options
+
+    Called from openstackclient.shell.OpenStackShell.__init__()
+    after the builtin parser has been initialized.  This is
+    where a plugin can add global options such as an API version setting.
+
+    :param argparse.ArgumentParser parser: The parser object that has been
+        initialized by OpenStackShell.
+    """
+    parser.add_argument(
+        '--os-observability-api-version',
+        metavar='<observability-api-version>',
+        help='Observability Plugin API version, default='
+             + DEFAULT_API_VERSION
+             + ' (Env: OS_OSCPLUGIN_API_VERSION)')
+    return parser

observabilityclient/utils/runner.py

@@ -0,0 +1,201 @@
+#   Copyright 2022 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+import ansible_runner
+import configparser
+import os
+import shutil
+
+from ansible.inventory.manager import InventoryManager
+from ansible.parsing.dataloader import DataLoader
+from ansible.vars.manager import VariableManager
+
+from observabilityclient.utils import shell
+
+
+class AnsibleRunnerException(Exception):
+    """Base exception class for runner exceptions"""
+
+
+class AnsibleRunnerFailed(AnsibleRunnerException):
+    """Raised when ansible run failed"""
+
+    def __init__(self, status, rc, stderr):
+        super(AnsibleRunnerFailed).__init__()
+        self.status = status
+        self.rc = rc
+        self.stderr = stderr
+
+    def __str__(self):
+        return ('Ansible run failed with status {}'
+                ' (return code {}):\n{}').format(self.status, self.rc,
+                                                 self.stderr)
+
+
+def parse_inventory_hosts(inventory):
+    """Returns list of dictionaries. Each dictionary contains info about
+    single node from inventory.
+    """
+    dl = DataLoader()
+    if isinstance(inventory, str):
+        inventory = [inventory]
+    im = InventoryManager(loader=dl, sources=inventory)
+    vm = VariableManager(loader=dl, inventory=im)
+
+    out = []
+    for host in im.get_hosts():
+        data = vm.get_vars(host=host)
+        out.append(
+            dict(host=data.get('inventory_hostname', str(host)),
+                 ip=data.get('ctlplane_ip', data.get('ansible_host')),
+                 hostname=data.get('canonical_hostname'))
+        )
+    return out
+
+
+class AnsibleRunner:
+    """Simple wrapper for ansible-playbook."""
+
+    def __init__(self, workdir: str, moduledir: str = None,
+                 ssh_user: str = 'root', ssh_key: str = None,
+                 ansible_cfg: str = None):
+        """
+        :param workdir: Location of the working directory.
+        :type workdir: String
+
+        :param ssh_user: User for the ssh connection.
+        :type ssh_user: String
+
+        :param ssh_key: Private key to use for the ssh connection.
+        :type ssh_key: String
+
+        :param moduledir: Location of the ansible module and library.
+        :type moduledir: String
+
+        :param ansible_cfg: Path to an ansible configuration file.
+        :type ansible_cfg: String
+        """
+        self.workdir = shell.file_check(workdir, ftype='directory')
+
+        if moduledir is None:
+            moduledir = ''
+        ansible_cfg = ansible_cfg or os.path.join(workdir, 'ansible.cfg')
+        if not os.path.exists(ansible_cfg):
+            conf = dict(
+                ssh_connection=dict(
+                    ssh_args=(
+                        '-o UserKnownHostsFile={} '
+                        '-o StrictHostKeyChecking=no '
+                        '-o ControlMaster=auto '
+                        '-o ControlPersist=30m '
+                        '-o ServerAliveInterval=64 '
+                        '-o ServerAliveCountMax=1024 '
+                        '-o Compression=no '
+                        '-o TCPKeepAlive=yes '
+                        '-o VerifyHostKeyDNS=no '
+                        '-o ForwardX11=no '
+                        '-o ForwardAgent=yes '
+                        '-o PreferredAuthentications=publickey '
+                        '-T'
+                    ).format(os.devnull),
+                    retries=3,
+                    timeout=30,
+                    scp_if_ssh=True,
+                    pipelining=True
+                ),
+                defaults=dict(
+                    deprecation_warnings=False,
+                    remote_user=ssh_user,
+                    private_key_file=ssh_key,
+                    library=os.path.expanduser(
+                        '~/.ansible/plugins/modules:{workdir}/modules:'
+                        '{userdir}:{ansible}/plugins/modules:'
+                        '{ansible}-modules'.format(
+                            userdir=moduledir, workdir=workdir,
+                            ansible='/usr/share/ansible'
+                        )
+                    ),
+                    lookup_plugins=os.path.expanduser(
+                        '~/.ansible/plugins/lookup:{workdir}/lookup:'
+                        '{ansible}/plugins/lookup:'.format(
+                            workdir=workdir, ansible='/usr/share/ansible'
+                        )
+                    ),
+                    gathering='smart',
+                    log_path=shell.file_check(
+                        os.path.join(workdir, 'ansible.log'),
+                        clear=True
+                    )
+                ),
+            )
+            parser = configparser.ConfigParser()
+            parser.read_dict(conf)
+            with open(ansible_cfg, 'w') as conffile:
+                parser.write(conffile)
+        os.environ['ANSIBLE_CONFIG'] = ansible_cfg
+
+    def run(self, playbook, tags: str = None, skip_tags: str = None,
+            timeout: int = 30, quiet: bool = False, debug: bool = False):
+        """Run given Ansible playbook.
+
+        :param playbook: Playbook filename.
+        :type playbook: String
+
+        :param tags: Run specific tags.
+        :type tags: String
+
+        :param skip_tags: Skip specific tags.
+        :type skip_tags: String
+
+        :param timeout: Timeout to finish playbook execution (minutes).
+        :type timeout: int
+
+        :param quiet: Disable all output (Defaults to False)
+        :type quiet: Boolean
+
+        :param debug: Enable debug output (Defaults to False)
+        :type quiet: Boolean
+        """
+        kwargs = {
+            'private_data_dir': self.workdir,
+            'verbosity': 3 if debug else 0,
+        }
+        locs = locals()
+        for arg in ['playbook', 'tags', 'skip_tags', 'quiet']:
+            if locs[arg] is not None:
+                kwargs[arg] = locs[arg]
+        run_conf = ansible_runner.runner_config.RunnerConfig(**kwargs)
+        run_conf.prepare()
+        run = ansible_runner.Runner(config=run_conf)
+        try:
+            status, rc = run.run()
+        finally:
+            if status in ['failed', 'timeout', 'canceled'] or rc != 0:
+                err = getattr(run, 'stderr', getattr(run, 'stdout', None))
+                if err:
+                    error = err.read()
+                else:
+                    error = "Ansible failed with status %s" % status
+                raise AnsibleRunnerFailed(status, rc, error)
+
+    def destroy(self, clear: bool = False):
+        """Cleans environment after Ansible run.
+
+        :param clear: Clear also workdir
+        :type clear: Boolean
+        """
+        del os.environ['ANSIBLE_CONFIG']
+        if clear:
+            shutil.rmtree(self.workdir, ignore_errors=True)

observabilityclient/utils/shell.py

@@ -0,0 +1,75 @@
+#   Copyright 2022 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+import os
+import pipes
+import shutil
+import subprocess
+import tempfile
+
+
+from contextlib import contextmanager
+from observabilityclient.utils import strings
+
+
+@contextmanager
+def tempdir(base: str, prefix: str = None, clear: bool = True) -> str:
+    path = tempfile.mkdtemp(prefix=prefix, dir=base)
+    try:
+        yield path
+    finally:
+        if clear:
+            shutil.rmtree(path, ignore_errors=True)
+
+
+def file_check(path: str, ftype: str = 'file', clear: bool = False) -> str:
+    """Check if given path exists and create it in case required."""
+    if not os.path.exists(path) or clear:
+        if ftype == 'directory':
+            if clear:
+                shutil.rmtree(path, ignore_errors=True)
+            os.makedirs(path, mode=0o700, exist_ok=True)
+        elif ftype == 'file':
+            with open(path, 'w') as f:
+                f.close()
+    return path
+
+
+def execute(cmd, workdir: str = None, can_fail: bool = True,
+            mask_list: list = None, use_shell: bool = False):
+    """
+    Runs given shell command. Returns return code and content of stdout.
+
+    :param workdir: Location of the working directory.
+    :type workdir: String
+
+    :param can_fail: If is set to True RuntimeError is raised in case
+                     of command returned non-zero return code.
+    :type can_fail: Boolean
+    """
+    mask_list = mask_list or []
+
+    if not isinstance(cmd, str):
+        masked = ' '.join((pipes.quote(i) for i in cmd))
+    else:
+        masked = cmd
+    masked = strings.mask_string(masked, mask_list)
+
+    proc = subprocess.Popen(cmd, cwd=workdir, shell=use_shell, close_fds=True,
+                            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    out, err = proc.communicate()
+    if proc.returncode and can_fail:
+        raise RuntimeError('Failed to execute command: %s' % masked)
+    return proc.returncode, out, err

observabilityclient/utils/strings.py

@@ -0,0 +1,41 @@
+#   Copyright 2022 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+
+STR_MASK = '*' * 8
+COLORS = {'nocolor': "\033[0m",
+          'red': "\033[0;31m",
+          'green': "\033[32m",
+          'blue': "\033[34m",
+          'yellow': "\033[33m"}
+
+
+def color_text(text, color):
+    """Returns given text string with appropriate color tag. Allowed value
+    for color parameter is 'red', 'blue', 'green' and 'yellow'.
+    """
+    return '%s%s%s' % (COLORS[color], text, COLORS['nocolor'])
+
+
+def mask_string(unmasked, mask_list=None):
+    """Replaces words from mask_list with MASK in unmasked string."""
+    mask_list = mask_list or []
+
+    masked = unmasked
+    for word in mask_list:
+        if not word:
+            continue
+        masked = masked.replace(word, STR_MASK)
+    return masked

observabilityclient/v1/base.py

@@ -0,0 +1,111 @@
+#   Copyright 2022 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+import os
+import shutil
+
+from osc_lib.command import command
+from osc_lib.i18n import _
+
+from observabilityclient.utils import runner
+from observabilityclient.utils import shell
+
+
+OBSLIBDIR = shell.file_check('/usr/share/osp-observability', 'directory')
+OBSWRKDIR = shell.file_check(
+    os.path.expanduser('~/.osp-observability'), 'directory'
+)
+
+
+class ObservabilityBaseCommand(command.Command):
+    """Base class for observability commands."""
+
+    def get_parser(self, prog_name):
+        parser = super().get_parser(prog_name)
+        parser.add_argument(
+            '--dev',
+            action='store_true',
+            help=_("Enable development output.")
+        )
+        parser.add_argument(
+            '--messy',
+            action='store_true',
+            help=_("Disable cleanup of temporary files.")
+        )
+        parser.add_argument(
+            '--workdir',
+            default=OBSWRKDIR,
+            help=_("Working directory for observability commands.")
+        )
+        parser.add_argument(
+            '--moduledir',
+            default=None,
+            help=_("Directory with additional Ansible modules.")
+        )
+        parser.add_argument(
+            '--ssh-user',
+            default='heat-admin',
+            help=_("Username to be used for SSH connection.")
+        )
+        parser.add_argument(
+            '--ssh-key',
+            default='/home/stack/.ssh/id_rsa',
+            help=_("SSH private key to be used for SSH connection.")
+        )
+        parser.add_argument(
+            '--ansible-cfg',
+            default=os.path.join(OBSWRKDIR, 'ansible.cfg'),
+            help=_("Path to Ansible configuration.")
+        )
+        parser.add_argument(
+            '--config',
+            default=None,
+            help=_("Path to playbook configuration file.")
+        )
+        return parser
+
+    def _run_playbook(self, playbook, inventory, parsed_args):
+        """Run Ansible raw playbook"""
+        playbook = os.path.join(OBSLIBDIR, 'playbooks', playbook)
+        with shell.tempdir(parsed_args.workdir,
+                           prefix=os.path.splitext(playbook)[0],
+                           clear=not parsed_args.messy) as tmpdir:
+            # copy extravars file for the playbook run
+            if parsed_args.config:
+                envdir = shell.file_check(os.path.join(tmpdir, 'env'),
+                                          'directory')
+                shutil.copy(parsed_args.config,
+                            os.path.join(envdir, 'extravars'))
+            # copy inventory file for the playbook run
+            shutil.copy(inventory, os.path.join(tmpdir, 'inventory'))
+            # run playbook
+            rnr = runner.AnsibleRunner(tmpdir,
+                                       moduledir=parsed_args.moduledir,
+                                       ssh_user=parsed_args.ssh_user,
+                                       ssh_key=parsed_args.ssh_key,
+                                       ansible_cfg=parsed_args.ansible_cfg)
+            if parsed_args.messy:
+                print("Running playbook %s" % playbook)
+            rnr.run(playbook, debug=parsed_args.dev)
+            rnr.destroy(clear=not parsed_args.messy)
+
+    def _execute(self, command, parsed_args):
+        """Execute local command"""
+        with shell.tempdir(parsed_args.workdir, prefix='exec',
+                           clear=not parsed_args.messy) as tmpdir:
+            rc, out, err = shell.execute(command, workdir=tmpdir,
+                                         can_fail=parsed_args.dev,
+                                         use_shell=True)
+        return rc, out, err

observabilityclient/v1/deploy.py

@@ -0,0 +1,161 @@
+#   Copyright 2022 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+import os
+import requests
+import shutil
+import sys
+import yaml
+
+from osc_lib.i18n import _
+
+from observabilityclient.v1 import base
+from observabilityclient.utils import runner
+
+
+class InventoryError(Exception):
+    def __init__(self, err, out):
+        self.err = err
+        self.out = out
+
+    def __str__(self):
+        return ('Failed to generate or locate Ansible '
+                'inventory file:\n%s\n%s' % (self.err or '', self.out))
+
+
+INVENTORY = os.path.join(base.OBSWRKDIR, 'openstack-inventory.yaml')
+INV_FALLBACKS = [
+    '~/tripleo-deploy/{stack}/openstack-inventory.yaml',
+    './overcloud-deploy/{stack}/openstack-inventory.yaml'
+]
+ENDPOINTS = os.path.join(base.OBSWRKDIR, 'scrape-endpoints.yaml')
+STACKRC = os.path.join(base.OBSWRKDIR, 'stackrc')
+
+
+def _curl(host: dict, port: int, timeout: int = 1) -> str:
+    """Returns scraping endpoint URL if it is reachable
+    otherwise returns None."""
+    url = f'http://{host["ip"]}:{port}/metrics'
+    try:
+        r = requests.get(url, timeout=1)
+        if r.status_code != 200:
+            url = None
+        r.close()
+    except requests.exceptions.ConnectionError:
+        url = None
+    return url
+
+
+class Discover(base.ObservabilityBaseCommand):
+    """Generate Ansible inventory file and scrapable enpoints list file."""
+
+    def get_parser(self, prog_name):
+        parser = super().get_parser(prog_name)
+        parser.add_argument(
+            '--scrape',
+            action='append',
+            default=['collectd/9103'],
+            help=_("Service/Port of scrape endpoint to check on nodes")
+        )
+        parser.add_argument(
+            '--stack-name',
+            default='overcloud',
+            help=_("Overcloud stack name for which inventory file should "
+                   "be generated")
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        # discover undercloud and overcloud nodes
+        try:
+            rc, out, err = self._execute(
+                'tripleo-ansible-inventory '
+                '--static-yaml-inventory {} '
+                '--stack {}'.format(INVENTORY, parsed_args.stack_name),
+                parsed_args
+            )
+            if rc:
+                raise InventoryError(err, out)
+
+            # OSP versions with deprecated tripleo-ansible-inventory fallbacks
+            # to static inventory file generated at one of the fallback path
+            if not os.path.exists(INVENTORY):
+                for i in INV_FALLBACKS:
+                    absi = i.format(stack=parsed_args.stack_name)
+                    absi = os.path.abspath(os.path.expanduser(absi))
+                    if os.path.exists(absi):
+                        shutil.copyfile(absi, INVENTORY)
+                        break
+                else:
+                    raise InventoryError('None of the fallback inventory files'
+                                         ' exists: %s' % INV_FALLBACKS, '')
+        except InventoryError as ex:
+            print(str(ex))
+            sys.exit(1)
+
+        # discover scrape endpoints
+        endpoints = dict()
+        hosts = runner.parse_inventory_hosts(INVENTORY)
+        for scrape in parsed_args.scrape:
+            service, port = scrape.split('/')
+            for host in hosts:
+                if parsed_args.dev:
+                    name = host["hostname"] if host["hostname"] else host["ip"]
+                    print(f'Trying to fetch {service} metrics on host '
+                          f'{name} at port {port}', end='')
+                node = _curl(host, port, timeout=1)
+                if node:
+                    endpoints.setdefault(service.strip(), []).append(node)
+                if parsed_args.dev:
+                    print(' [success]' if node else ' [failure]')
+        data = yaml.safe_dump(endpoints, default_flow_style=False)
+        with open(ENDPOINTS, 'w') as f:
+            f.write(data)
+        print("Discovered following scraping endpoints:\n%s" % data)
+
+
+class Setup(base.ObservabilityBaseCommand):
+    """Install and configure given Observability component(s)"""
+
+    auth_required = False
+
+    def get_parser(self, prog_name):
+        parser = super().get_parser(prog_name)
+        parser.add_argument(
+            'components',
+            nargs='+',
+            choices=[
+                'prometheus_agent',
+                # TODO: in future will contain option for all stack components
+            ]
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        for compnt in parsed_args.components:
+            playbook = '%s.yml' % compnt
+            try:
+                self._run_playbook(playbook, INVENTORY,
+                                   parsed_args=parsed_args)
+            except OSError as ex:
+                print('Failed to load playbook file: %s' % ex)
+                sys.exit(1)
+            except yaml.YAMLError as ex:
+                print('Failed to parse playbook configuration: %s' % ex)
+                sys.exit(1)
+            except runner.AnsibleRunnerFailed as ex:
+                print('Ansible run %s (rc %d)' % (ex.status, ex.rc))
+                if parsed_args.dev:
+                    print(ex.stderr)