Set service configuration in all OSP types

Method 'set_service_setting' to apply OSP component configuration
changes on devstack/podified setups.

Also made 'get_osp_cmd_prefix' to run openstack commands for
devstack/podified, meant to ease implementation of CLI whitebox testing
(Follow-up patch will include CLI whitebox testing,
security group logging).

Migrated and adjusted 'validate_command' and 'run_group_cmd'
for next gen and neutron whitebox plugin.

Depends-On: https://review.opendev.org/c/x/whitebox-neutron-tempest-plugin/+/914937

Change-Id: I8a344ae3a3c6648a3118f40ce779b3ad8476b3cb

whitebox_neutron_tempest_plugin/tests/scenario/base.py

@@ -13,6 +13,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 import base64
+from functools import partial
+from multiprocessing import Process
 import os
 import random
 import re
@@ -27,6 +29,7 @@ from neutron_lib import constants
 from neutron_tempest_plugin.common import shell
 from neutron_tempest_plugin.common import ssh
 from neutron_tempest_plugin.common import utils as common_utils
+from neutron_tempest_plugin import exceptions
 from neutron_tempest_plugin.scenario import base
 from oslo_log import log
 from tempest.common import utils
@@ -80,8 +83,8 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
             cls.master_cont_cmd_executor = cls.proxy_host_client
         else:
             LOG.warning(("Unrecognized deployer tool '{}', plugin supports "
-                         "openstack_type as devstack/podified."
+                         "openstack_type as devstack/podified.".format(
-                         .format(WB_CONF.openstack_type)))
+                             WB_CONF.openstack_type)))
 
     @classmethod
     def run_on_master_controller(cls, cmd):
@@ -273,8 +276,50 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
         if service == 'neutron':
             pod = cls.get_pod_of_service(service)
             return cls.proxy_host_client.exec_command(
-                'oc rsh {} find {} -type f'.format(pod, os.path.split(
+                'oc rsh -n openstack {} find {} -type f'.format(
-                    WB_CONF.neutron_config)[0])).strip().split('\n')
+                    pod, os.path.split(
+                        WB_CONF.neutron_config)[0])).strip().split('\n')
+
+    # TODO(mblue): next gen computes configuration set should be done too,
+    # 'oc patch' for data plane would need more steps and triggers deployment
+    @classmethod
+    def set_service_setting(cls, node_type='controller',
+                            file='', service='neutron',
+                            section='DEFAULT', param='', value=''):
+        """Set configuration for service
+
+        :param node_type(str): Node type for change, ex: controller/compute
+        (currently only controllers).
+        :param file(str): File for configuration change (except in podified).
+        :param service(str): Podified service name (only podified).
+        :param section(str): Section in the config file.
+        :param param(str): Parameter in section to change.
+        :param value(str): Value to set.
+        """
+        assert param, "'param' must be supplied"
+        if WB_CONF.openstack_type == 'podified':
+            if node_type == 'compute':
+                raise cls.skipException(
+                    "Setting computes configuration not supported yet on "
+                    "podified setups (TODO).")
+            patch_buffer = '''
+                spec:
+                  {}:
+                    template:
+                      customServiceConfig: |
+                        [{}]
+                        {} = {}'''.format(
+                           service, section, param, value)
+            cmd = "oc patch $(oc get oscp -o name) --type=merge --patch '" + \
+                patch_buffer + "'"
+            LOG.debug("Set configuration command:\n%s", cmd)
+            output = cls.proxy_host_client.exec_command(cmd)
+            LOG.debug("Output:\n%s", output)
+        else:
+            cls.run_group_cmd(
+                'sudo crudini --set {} {} {} {} && sudo sync'.format(
+                    file, section, param, value),
+                node_type)
 
     @classmethod
     def check_service_setting(
@@ -283,13 +328,14 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
             msg='Required config value is missing', skip_if_fails=True):
         """Check if a service on a node has a setting with a value in config
 
-        :param node(dict): Dictionary with host-related parameters,
+        :param host(dict): Dictionary with host-related parameters,
-                           host['client'] is a required parameter
+                           host['client'] is a required parameter.
         :param service(str): Name of the containerized service.
         :param config_files(list): List with paths to config files. List makes
                                    sense on podified where e.g. neutron has
                                    2 config files with same sections.
         :param section(str): Section in the config file.
+        :param param(str): Parameter in section to check.
         :param value(str): Expected value.
         :param msg(str): Message to print in case of expected value not found
         :param skip_if_fails(bool): skip if the check fails - if it fails and
@@ -298,7 +344,7 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
         """
 
         if WB_CONF.openstack_type == 'podified':
-            service_prefix = "oc rsh {}".format(
+            service_prefix = "oc rsh -n openstack {}".format(
                 cls.get_pod_of_service(service))
         else:
             service_prefix = ""
@@ -502,6 +548,150 @@ class BaseTempestWhiteboxTestCase(base.BaseTempestTestCase):
             port_type=port_type)
         return sender, receiver
 
+    @classmethod
+    def get_osp_cmd_prefix(cls, admin=True):
+        # TODO(mblue): figure how admin used in podified setup when needed
+        if WB_CONF.openstack_type == 'podified':
+            prefix = 'oc rsh -n openstack openstackclient '
+        elif WB_CONF.openstack_type == 'devstack':
+            prefix = '. /opt/stack/devstack/openrc{} && '.format(
+                ' admin' if admin else '')
+        else:
+            prefix = '. ~/overcloudrc && '
+        return prefix
+
+    @staticmethod
+    def validate_command(cmd, pattern='', timeout=60,
+                         ssh_client=None,
+                         ret_bool_status=False, ret_bool_pattern=False):
+        """Run a command on a given host.
+        Optional: validation of output by regex, and exit status.
+
+        :param cmd: Command to execute on given host.
+        :type cmd: str
+
+        :param pattern: Optional regex pattern to validate each
+                        commands output.
+        :type pattern: str, optional
+
+        :param timeout: Timeout for command to finish.
+        :type timeout: int, optional
+
+        :param ssh_client: Ssh client to execute command, default UC.
+        :type ssh_client: SSHClient, optional
+
+        :param ret_bool_pattern: Return boolean instead of error raise
+                            in pattern verification (Default False).
+                            Without any boolean option, returns all output.
+        :type ret_bool_pattern: bool, optional
+
+        :param ret_bool_status: Return boolean instead of error raise
+                            in exit status verification (Default False).
+                            Without any boolean option, returns all output.
+        :type ret_bool_status: bool, optional
+
+        :returns: all output of command as str, or boolean if either of
+         return boolean options is True (ret_bool_pattern or ret_bool_status).
+        """
+        # execute on tester node or other, according to CI configuration
+        if ssh_client is None and not WB_CONF.exec_on_tester:
+            ssh_client = ssh.Client(
+                host=WB_CONF.tester_ip,
+                username=WB_CONF.tester_user,
+                password=WB_CONF.tester_pass,
+                key_filename=WB_CONF.tester_key_file)
+        # verify command success using exception
+        try:
+            result = shell.execute(
+                cmd, timeout=timeout, check=(not ret_bool_status),
+                ssh_client=ssh_client)
+        except exceptions.ShellCommandFailed:
+            LOG.exception(
+                'Tested command failed (raising error) -> "{}":'.format(cmd))
+        # verify command success using boolean
+        if ret_bool_status and result.exit_status != 0:
+            LOG.debug(
+                'Tested command failed (returning False) -> "{}":'.format(cmd))
+            return False
+        # verify desired output using exception/boolean
+        all_output = (result.stderr if result.stderr else '') + \
+            (result.stdout if result.stdout else '')
+        if pattern:
+            fail_msg = 'Pattern "{}" not found in output of "{}" command.'
+            try:
+                if not re.search(pattern, all_output):
+                    raise AssertionError(fail_msg.format(pattern, cmd))
+            except AssertionError as err:
+                if ret_bool_pattern:
+                    return False
+                raise err
+        if ret_bool_status or ret_bool_pattern:
+            return True
+        return all_output
+
+    @classmethod
+    def run_group_cmd(cls, cmd, group='', pattern='', timeout=60,
+                      check_status=True, parallel=True):
+        """Run a command on a group of overcloud nodes,
+        either in parallel/sequential.
+        Optional: validation of output by regex, and exit status.
+
+        :param cmd: Command to execute on nodes group.
+        :type cmd: str
+
+        :param group: Initial name to fit group, ex: controller
+                      (Default all overcloud nodes).
+        :type group: str, optional
+
+        :param pattern: Optional regex pattern to validate each
+                        commands output.
+        :type pattern: str, optional
+
+        :param timeout: Timeout for all commands to finish.
+        :type timeout: int, optional
+
+        :param check_status: Whether to verify commands exit status.
+        :type check_status: bool, optional
+
+        :param parallel: Run commands in parallel or sequential.
+        :type parallel: bool, optional
+        """
+        tasks = []
+        group = group.lower()
+        group_name = group if group else 'all'
+        for node in cls.nodes:
+            if node['is_' + group]:
+                LOG.info('Running command in %s "%s" on "%s" from group "%s"',
+                         'parallel' if parallel else 'sequence',
+                         cmd, node['name'], group_name)
+                # functools.partial instead of lambda for "freezed" arguments
+                # (figure values in definition time rather than execution time)
+                call = partial(cls.validate_command,
+                               cmd, pattern, timeout, node['client'],
+                               not check_status)
+                tasks.append(Process(target=call)) if parallel else call()
+        if parallel and tasks:
+            for task in tasks:
+                task.start()
+            for task in tasks:
+                task.join()
+            # NOTE(mblue): guarantee wait for exit, or proper error
+            common_utils.wait_until_true(
+                lambda: None not in [t.exitcode for t in tasks],
+                timeout=timeout,
+                sleep=min(5, timeout),
+                exception=RuntimeError(
+                    ('Timed out: waiting for command "{}" on "{}" nodes\n'
+                     '({}/{} processes not finished)').format(
+                         cmd, group_name,
+                         len([t for t in tasks if t.exitcode is None]),
+                         len(tasks))))
+            if check_status:
+                if sum([t.exitcode for t in tasks]) != 0:
+                    raise AssertionError(
+                        'Command failure "{}" on "{}" nodes.'.format(
+                            cmd, group_name))
+
 
 class BaseTempestTestCaseAdvanced(BaseTempestWhiteboxTestCase):
     """Base class skips test suites unless advanced image is available,
@@ -713,7 +903,7 @@ class BaseTempestTestCaseOvn(BaseTempestWhiteboxTestCase):
         if WB_CONF.openstack_type == 'podified':
             sb_pod = cls.proxy_host_client.exec_command(
                 "oc get pods | grep ovsdbserver-sb | cut -f1 -d' '").strip()
-            sb_prefix = 'oc rsh {}'.format(sb_pod)
+            sb_prefix = 'oc rsh -n openstack {}'.format(sb_pod)
             nb_prefix = sb_prefix.replace('sb', 'nb')
             cmd = "{} ovn-{}ctl"
             return [cmd.format(nb_prefix, 'nb'), cmd.format(sb_prefix, 'sb')]