Add functional tests

This adds new functional tests, which are supposed to be run
on devstack with a running instance of prometheus.
It tests all of the cli commands as well as all the functions
exposed in the python client.

These tests could be included into the telemetry-dsvm-integration
jobs in the future to use the same devstack vm.

Change-Id: Ibd6deec559465bf3cb7480681b816f55bdf9010e

.zuul.yaml

@@ -1,3 +1,24 @@
+- job:
+    # TODO(jwysogla): Include these tests in the
+    # telemetry-dsvm-integration jobs
+    name: observabilityclient-dsvm-functional
+    parent: devstack-tox-functional
+    description: |
+      Devstack-based functional tests for observabilityclient.
+    required-projects:
+      - openstack/python-observabilityclient
+      - openstack/ceilometer
+      - infrawatch/sg-core
+    timeout: 4200
+    vars:
+      devstack_localrc:
+        USE_PYTHON3: True
+        PROMETHEUS_SERVICE_SCRAPE_TARGETS: prometheus,sg-core
+        CEILOMETER_BACKEND: sg-core
+      devstack_plugins:
+        sg-core: https://github.com/infrawatch/sg-core
+        ceilometer: https://opendev.org/openstack/ceilometer
+
 - project:
     queue: telemetry
     templates:
@@ -25,6 +46,8 @@
         - telemetry-dsvm-integration-centos-9s-fips:
             irrelevant-files: *pobsc-irrelevant-files
             voting: false
+        - observabilityclient-dsvm-functional:
+            irrelevant-files: *pobsc-irrelevant-files
     gate:
       jobs:
         - telemetry-dsvm-integration:
@@ -39,3 +62,5 @@
         - telemetry-dsvm-integration-centos-9s-fips:
             irrelevant-files: *pobsc-irrelevant-files
             voting: false
+        - observabilityclient-dsvm-functional:
+            irrelevant-files: *pobsc-irrelevant-files

AUTHORS

@@ -1,7 +1,9 @@
 Chris Sibbitt <csibbitt@redhat.com>
 Erno Kuvaja <jokke@usr.fi>
+Jaromir Wysoglad <jwysogla@redhat.com>
 Jaromír Wysoglad <jwysogla@redhat.com>
 Leif Madsen <leif@leifmadsen.com>
 Leif Madsen <lmadsen@redhat.com>
 Marihan Girgis mgirgisf@redhat.com
+Martin Magr <mmagr@redhat.com>
 Martin Mágr <mmagr@redhat.com>

observabilityclient/tests/functional/base.py

@@ -0,0 +1,188 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# base.py file taken and modified from the openstackclient functional tests
+
+import json
+import logging
+import os
+import shlex
+import subprocess
+
+from observabilityclient import client
+
+from keystoneauth1 import loading
+from keystoneauth1 import session
+import os_client_config
+from tempest.lib.cli import output_parser
+from tempest.lib import exceptions
+import testtools
+
+ADMIN_CLOUD = os.environ.get('OS_ADMIN_CLOUD', 'devstack-admin')
+LOG = logging.getLogger(__name__)
+
+
+class PythonAPITestCase(testtools.TestCase):
+    def _getKeystoneSession(self):
+        conf = os_client_config.OpenStackConfig()
+        creds = conf.get_one_cloud(cloud=ADMIN_CLOUD).get_auth_args()
+        ks_creds = dict(
+            auth_url=creds["auth_url"],
+            username=creds["username"],
+            password=creds["password"],
+            project_name=creds["project_name"],
+            user_domain_id=creds["user_domain_id"],
+            project_domain_id=creds["project_domain_id"])
+        loader = loading.get_plugin_loader("password")
+        auth = loader.load_from_options(**ks_creds)
+        return session.Session(auth=auth)
+
+    def setUp(self):
+        super(PythonAPITestCase, self).setUp()
+        self.client = client.Client(
+            1,
+            self._getKeystoneSession()
+        )
+
+
+def execute(cmd, fail_ok=False, merge_stderr=False):
+    """Execute specified command for the given action."""
+    LOG.debug('Executing: %s', cmd)
+    cmdlist = shlex.split(cmd)
+    stdout = subprocess.PIPE
+    stderr = subprocess.STDOUT if merge_stderr else subprocess.PIPE
+
+    proc = subprocess.Popen(cmdlist, stdout=stdout, stderr=stderr)
+
+    result_out, result_err = proc.communicate()
+    result_out = result_out.decode('utf-8')
+    LOG.debug('stdout: %s', result_out)
+    LOG.debug('stderr: %s', result_err)
+
+    if not fail_ok and proc.returncode != 0:
+        raise exceptions.CommandFailed(
+            proc.returncode,
+            cmd,
+            result_out,
+            result_err,
+        )
+
+    return result_out
+
+
+class CliTestCase(testtools.TestCase):
+    @classmethod
+    def openstack(
+        cls,
+        cmd,
+        *,
+        cloud=ADMIN_CLOUD,
+        fail_ok=False,
+        parse_output=False,
+    ):
+        """Execute observabilityclient command for the given action.
+
+        :param cmd: A string representation of the command to execute.
+        :param cloud: The cloud to execute against. This can be a string, empty
+            string, or None. A string results in '--os-auth-type $cloud', an
+            empty string results in the '--os-auth-type' option being
+            omitted, and None resuts in '--os-auth-type none' for legacy
+            reasons.
+        :param fail_ok: If failure is permitted. If False (default), a command
+            failure will result in `~tempest.lib.exceptions.CommandFailed`
+            being raised.
+        :param parse_output: If true, pass the '-f json' parameter and decode
+            the output.
+        :returns: The output from the command.
+        :raises: `~tempest.lib.exceptions.CommandFailed` if the command failed
+            and ``fail_ok`` was ``False``.
+        """
+        auth_args = []
+        if cloud is None:
+            # Execute command with no auth
+            auth_args.append('--os-auth-type none')
+        elif cloud != '':
+            # Execute command with an explicit cloud specified
+            auth_args.append(f'--os-cloud {cloud}')
+
+        format_args = []
+        if parse_output:
+            format_args.append('-f json')
+
+        output = execute(
+            ' '.join(['openstack'] + auth_args + [cmd] + format_args),
+            fail_ok=fail_ok,
+        )
+
+        if parse_output:
+            return json.loads(output)
+        else:
+            return output
+
+    @classmethod
+    def assertOutput(cls, expected, actual):
+        if expected != actual:
+            raise Exception(expected + ' != ' + actual)
+
+    @classmethod
+    def assertInOutput(cls, expected, actual):
+        if expected not in actual:
+            raise Exception(expected + ' not in ' + actual)
+
+    @classmethod
+    def assertNotInOutput(cls, expected, actual):
+        if expected in actual:
+            raise Exception(expected + ' in ' + actual)
+
+    @classmethod
+    def assertsOutputNotNone(cls, observed):
+        if observed is None:
+            raise Exception('No output observed')
+
+    def assert_table_structure(self, items, field_names):
+        """Verify that all items have keys listed in field_names."""
+        for item in items:
+            for field in field_names:
+                self.assertIn(field, item)
+
+    def assert_show_fields(self, show_output, field_names):
+        """Verify that all items have keys listed in field_names."""
+        # field_names = ['name', 'description']
+        # show_output = [{'name': 'fc2b98d8faed4126b9e371eda045ade2'},
+        #          {'description': 'description-821397086'}]
+        # this next line creates a flattened list of all 'keys' (like 'name',
+        # and 'description' out of the output
+        all_headers = [item for sublist in show_output for item in sublist]
+        for field_name in field_names:
+            self.assertIn(field_name, all_headers)
+
+    def parse_show_as_object(self, raw_output):
+        """Return a dict with values parsed from cli output."""
+        items = self.parse_show(raw_output)
+        o = {}
+        for item in items:
+            o.update(item)
+        return o
+
+    def parse_show(self, raw_output):
+        """Return list of dicts with item values parsed from cli output."""
+        items = []
+        table_ = output_parser.table(raw_output)
+        for row in table_['values']:
+            item = {}
+            item[row[0]] = row[1]
+            items.append(item)
+        return items
+
+    def parse_listing(self, raw_output):
+        """Return list of dicts with basic item parsed from cli output."""
+        return output_parser.listing(raw_output)

observabilityclient/tests/functional/test_cli.py

@@ -0,0 +1,161 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from observabilityclient.tests.functional import base
+import time
+
+
+class CliTestFunctionalRBACDisabled(base.CliTestCase):
+    """Functional tests for cli commands with disabled RBAC."""
+
+    def test_list(self):
+        cmd_output = self.openstack(
+            'metric list --disable-rbac',
+            parse_output=True,
+        )
+        name_list = [item.get('metric_name') for item in cmd_output]
+        self.assertIn(
+            'up',
+            name_list
+        )
+
+    def test_show(self):
+        cmd_output = self.openstack(
+            'metric show up --disable-rbac',
+            parse_output=True,
+        )
+        for metric in cmd_output:
+            self.assertEqual(
+                "up",
+                metric["__name__"]
+            )
+            self.assertEqual(
+                "1",
+                metric["value"]
+            )
+
+    def test_query(self):
+        cmd_output = self.openstack(
+            'metric query up --disable-rbac',
+            parse_output=True,
+        )
+        for metric in cmd_output:
+            self.assertEqual(
+                "up",
+                metric["__name__"]
+            )
+            self.assertEqual(
+                "1",
+                metric["value"]
+            )
+
+
+class CliTestFunctionalRBACEnabled(base.CliTestCase):
+    """Functional tests for cli commands with enabled RBAC."""
+
+    def test_list(self):
+        cmd_output = self.openstack(
+            'metric list',
+            parse_output=True,
+        )
+        name_list = [item.get('metric_name') for item in cmd_output]
+        self.assertIn(
+            'ceilometer_image_size',
+            name_list
+        )
+        self.assertNotIn(
+            'up',
+            name_list
+        )
+
+    def test_show(self):
+        cmd_output = self.openstack(
+            'metric show ceilometer_image_size',
+            parse_output=True,
+        )
+        for metric in cmd_output:
+            self.assertEqual(
+                "ceilometer_image_size",
+                metric["__name__"]
+            )
+            self.assertEqual(
+                "sg-core",
+                metric["job"]
+            )
+
+    def test_query(self):
+        cmd_output = self.openstack(
+            'metric query ceilometer_image_size',
+            parse_output=True,
+        )
+        for metric in cmd_output:
+            self.assertEqual(
+                "ceilometer_image_size",
+                metric["__name__"]
+            )
+            self.assertEqual(
+                "sg-core",
+                metric["job"]
+            )
+
+
+class CliTestFunctionalAdminCommands(base.CliTestCase):
+    """Functional tests for cli admin commands."""
+
+    def test_delete(self):
+        test_start_time = int(time.time())
+        query_before = self.openstack(
+            f'metric query prometheus_ready@{test_start_time} --disable-rbac',
+            parse_output=True,
+        )
+
+        values = [item.get("__name__") for item in query_before]
+        # Check, that the metric is present before the deletion
+        self.assertIn(
+            "prometheus_ready",
+            values
+        )
+
+        self.openstack(
+            'metric delete prometheus_ready --disable-rbac',
+            parse_output=False,
+        )
+
+        query_after = self.openstack(
+            f'metric query prometheus_ready@{test_start_time} --disable-rbac',
+            parse_output=True,
+        )
+        values = [item.get("__name__") for item in query_after]
+        # Check, that the metric is not present after the deletion
+        self.assertNotIn(
+            "prometheus_ready",
+            values
+        )
+
+    def test_clean_tombstones(self):
+        # NOTE(jwysogla) There is not much to check here
+        # except for the fact, that the command doesn't
+        # raise an exception. Prometheus doesn't send any
+        # data back and we don't have a reliable way to query
+        # prometheus that this command did something.
+        self.openstack('metric clean-tombstones')
+
+    def test_snapshot(self):
+        cmd_output = self.openstack(
+            'metric snapshot',
+            parse_output=True,
+        )
+        for name in cmd_output:
+            self.assertInOutput(
+                time.strftime('%Y%m%d'),
+                name.get("Snapshot file name")
+            )

observabilityclient/tests/functional/test_python_api.py

@@ -0,0 +1,94 @@
+#   Copyright 2023 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+
+from observabilityclient.tests.functional import base
+
+import time
+
+
+class PythonAPITestFunctionalRBACDisabled(base.PythonAPITestCase):
+    def test_list(self):
+        ret = self.client.query.list(disable_rbac=True)
+
+        self.assertIn("up", ret)
+
+    def test_show(self):
+        ret = self.client.query.show("up", disable_rbac=True)
+
+        for metric in ret:
+            self.assertEqual("up", metric.labels["__name__"])
+            self.assertEqual("1", metric.value)
+
+    def test_query(self):
+        ret = self.client.query.query("up", disable_rbac=True)
+
+        for metric in ret:
+            self.assertEqual("up", metric.labels["__name__"])
+            self.assertEqual("1", metric.value)
+
+
+class PythonAPITestFunctionalRBACEnabled(base.PythonAPITestCase):
+    def test_list(self):
+        ret = self.client.query.list(disable_rbac=False)
+
+        self.assertIn("ceilometer_image_size", ret)
+        self.assertNotIn("up", ret)
+
+    def test_show(self):
+        ret = self.client.query.show("ceilometer_image_size",
+                                     disable_rbac=False)
+
+        for metric in ret:
+            self.assertEqual("ceilometer_image_size",
+                             metric.labels["__name__"])
+            self.assertEqual("sg-core",
+                             metric.labels["job"])
+
+    def test_query(self):
+        ret = self.client.query.query("ceilometer_image_size",
+                                      disable_rbac=False)
+
+        for metric in ret:
+            self.assertEqual("ceilometer_image_size",
+                             metric.labels["__name__"])
+            self.assertEqual("sg-core", metric.labels["job"])
+
+
+class PythonAPITestFunctionalAdminCommands(base.PythonAPITestCase):
+    def test_delete(self):
+        now = time.time()
+        metric_name = "prometheus_build_info"
+        query = f"{metric_name}@{now}"
+
+        query_before = self.client.query.query(query, disable_rbac=True)
+
+        for metric in query_before:
+            self.assertEqual(metric_name, metric.labels["__name__"])
+
+        self.client.query.delete(metric_name)
+
+        query_after = self.client.query.query(query, disable_rbac=True)
+        self.assertEqual([], query_after)
+
+    def test_clean_tombstones(self):
+        # NOTE(jwysogla) There is not much to check here
+        # except for the fact, that the command doesn't
+        # raise an exception. Prometheus doesn't send any
+        # data back and we don't have a reliable way to query
+        # prometheus that this command did something.
+        self.client.query.clean_tombstones()
+
+    def test_snapshot(self):
+        ret = self.client.query.snapshot()
+        self.assertIn(time.strftime("%Y%m%d"), ret)

test-requirements.txt

@@ -0,0 +1,2 @@
+python-openstackclient>=6.3.0 # Apache-2.0
+os-client-config>=1.28.0 # Apache-2.0

tools/fix_ca_bundle.sh

@@ -0,0 +1,41 @@
+# File taken from python-glanceclient
+
+# When the functional tests are run in a devstack environment, we
+# need to make sure that the python-requests module installed by
+# tox in the test environment can find the distro-specific CA store
+# where the devstack certs have been installed.
+#
+# assumptions:
+# - devstack is running
+# - the devstack tls-proxy service is running
+# - the environment var OS_TESTENV_NAME is set in tox.ini (defaults
+#   to 'functional'
+#
+# This code based on a function in devstack lib/tls
+function set_ca_bundle {
+    local python_cmd=".tox/${OS_TESTENV_NAME:-functional}/bin/python"
+    local capath=$($python_cmd -c $'try:\n from requests import certs\n print (certs.where())\nexcept ImportError: pass')
+    # of course, each distro keeps the CA store in a different location
+    local fedora_CA='/etc/pki/tls/certs/ca-bundle.crt'
+    local ubuntu_CA='/etc/ssl/certs/ca-certificates.crt'
+    local suse_CA='/etc/ssl/ca-bundle.pem'
+
+    # the distro CA is rooted in /etc, so if ours isn't, we need to
+    # change it
+    if [[ ! $capath == "" && ! $capath =~ ^/etc/.* && ! -L $capath ]]; then
+        if [[ -e $fedora_CA ]]; then
+            rm -f $capath
+            ln -s $fedora_CA $capath
+        elif [[ -e $ubuntu_CA ]]; then
+            rm -f $capath
+            ln -s $ubuntu_CA $capath
+        elif [[ -e $suse_CA ]]; then
+            rm -f $capath
+            ln -s $suse_CA $capath
+        else
+            echo "can't set CA bundle, expect tests to fail"
+        fi
+    fi
+}
+
+set_ca_bundle

tox.ini

@@ -9,12 +9,14 @@ usedevelop = True
 setenv =
    VIRTUAL_ENV={envdir}
    OBSERVABILITY_CLIENT_EXEC_DIR={envdir}/bin
+   OS_TEST_PATH = ./observabilityclient/tests/unit
 passenv =
 	PROMETHEUS_*
 	OBSERVABILITY_*
 deps = .[test]
        pytest
-commands = pytest {posargs:observabilityclient/tests}
+
+commands = pytest {posargs} {env:OS_TEST_PATH}
 
 [testenv:pep8]
 basepython = python3
@@ -28,7 +30,22 @@ commands = {posargs}
 [testenv:cover]
 deps = {[testenv]deps}
        pytest-cov
-commands = observabilityclient {posargs:observabilityclient/tests}
+commands = observabilityclient {posargs} {env:$OS_TEST_PATH}
+
+[testenv:functional]
+setenv =
+       OS_TEST_PATH = ./observabilityclient/tests/functional
+       OS_TESTENV_NAME = {envname}
+allowlist_externals =
+       bash
+deps = .[test]
+       pytest
+       -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+commands =
+       bash tools/fix_ca_bundle.sh
+       pytest {posargs} {env:OS_TEST_PATH}
+
 
 [flake8]
 show-source = True