Add argument which allows users to add extensions

--append argument appends a value or values to the specified
section.key pair. It may be helpful in cases when a user wants
to add custom extensions to tempest.conf in an automated job.

Change-Id: I116f4456823913f21b5f8f01ff2b14d42ec67dc2
Story: 2004429
Task: 28088

config_tempest/main.py

@@ -1,4 +1,4 @@
-# Copyright 2016 Red Hat, Inc.
+# Copyright 2016, 2018 Red Hat, Inc.
 # All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -286,15 +286,29 @@ def get_arg_parser():
     parser.add_argument('--network-id',
                         help="""Specify which network with external connectivity
                                 should be used by the tests.""")
+    parser.add_argument('--append', action='append', default=[],
+                        metavar="SECTION.KEY=VALUE[,VALUE]",
+                        help="""Append values to tempest.conf
+                                Key value pair to be appended to the
+                                configuration file.
+                                NOTE: Multiple values are supposed to be
+                                divided by a COLON only, WITHOUT spaces.
+                                For example:
+                                 $ discover-tempest-config \\
+                                  --append features.ext=tag[,tag-ext] \\
+                                  --append section.ext=ext[,another-ext]
+                             """)
     parser.add_argument('--remove', action='append', default=[],
                         metavar="SECTION.KEY=VALUE[,VALUE]",
                         help="""Remove values from tempest.conf
                                 Key value pair to be removed from the
                                 configuration file.
+                                NOTE: Multiple values are supposed to be
+                                divided by a COLON only, WITHOUT spaces.
                                 For example:
                                  $ discover-tempest-config \\
                                   --remove identity.username=myname \\
-                                  --remove feature-enabled.api_ext=http,https
+                                  --remove feature-enabled.api_ext=http[,https]
                              """)
     return parser
 
@@ -327,9 +341,9 @@ def parse_values_to_remove(options):
         if len(argument.split('=')) == 2:
             section, values = argument.split('=')
             if len(section.split('.')) != 2:
-                raise Exception("Missing dot. The option --remove has to"
+                raise Exception("Missing dot. The option --remove has to "
-                                "come in the format 'section.key=value,"
+                                "come in the format 'section.key=value[,value"
-                                " but got '%s'." % (argument))
+                                "]', but got '%s'." % argument)
             parsed[section] = values.split(',')
         else:
             # missing equal sign, all values in section.key will be deleted
@@ -337,6 +351,34 @@ def parse_values_to_remove(options):
     return parsed
 
 
+def parse_values_to_append(options):
+    """Manual parsing of --append arguments.
+
+    :param options: list of arguments following --append argument.
+    :return: dictionary containing key paths with values to be added
+    :rtype: dict
+    """
+    parsed = {}
+    for argument in options:
+        if len(argument.split('=')) == 2:
+            section, values = argument.split('=')
+            if len(section.split('.')) != 2:
+                raise Exception("Missing dot. The option --append has to "
+                                "come in the format 'section.key=value[,value"
+                                "]', but got '%s'." % argument)
+            if values == '':
+                raise Exception("No values to append specified. The option "
+                                "--append has to come in the format "
+                                "'section.key=value[, value]', but got "
+                                "'%s'" % values)
+            parsed[section] = values.split(',')
+        else:
+            # missing equal sign, no values to add were specified, if a user
+            # wants to just create a section, it can be done so via overrides
+            raise Exception("Missing equal sign or more than just one found.")
+    return parsed
+
+
 def parse_overrides(overrides):
     """Manual parsing of positional arguments.
 
@@ -426,6 +468,7 @@ def get_cloud_creds(args_namespace):
 def config_tempest(**kwargs):
     # convert a list of remove values to a dict
     remove = parse_values_to_remove(kwargs.get('remove', []))
+    add = parse_values_to_append(kwargs.get('append', []))
     set_logging(kwargs.get('debug', False), kwargs.get('verbose', False))
 
     accounts_path = kwargs.get('test_accounts')
@@ -474,6 +517,9 @@ def config_tempest(**kwargs):
     if remove != {}:
         LOG.info("Removing configuration: %s", str(remove))
         conf.remove_values(remove)
+    if add != {}:
+        LOG.info("Adding configuration: %s", str(add))
+        conf.append_values(add)
     out_path = kwargs.get('out', 'etc/tempest.conf')
     conf.write(out_path)
 
@@ -482,6 +528,7 @@ def main():
     args = parse_arguments()
     cloud_creds = get_cloud_creds(args)
     config_tempest(
+        append=args.append,
         cloud_creds=cloud_creds,
         create=args.create,
         create_accounts_file=args.create_accounts_file,

config_tempest/tempest_conf.py

@@ -1,4 +1,4 @@
-# Copyright 2016, 2017 Red Hat, Inc.
+# Copyright 2016, 2017, 2018 Red Hat, Inc.
 # All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -144,3 +144,23 @@ class TempestConf(configparser.SafeConfigParser):
             except configparser.NoSectionError:
                 # only inform a user, section specified by him doesn't exist
                 C.LOG.error(sys.exc_info()[1])
+
+    def append_values(self, to_append):
+        """Appends values to configuration file specified in arguments.
+
+        :param to_append: {'section.key': [values_to_be_added], ...}
+        :type to_append: dict
+        """
+        for key_path in to_append:
+            section, key = key_path.split('.')
+            try:
+                conf_val = self.get(section, key).split(',')
+                # omit duplicates if found any
+                conf_val += list(set(to_append[key_path]) - set(conf_val))
+                self.set(section, key, ",".join(conf_val))
+            except configparser.NoOptionError:
+                # only inform a user, option specified by him doesn't exist
+                C.LOG.error(sys.exc_info()[1])
+            except configparser.NoSectionError:
+                # only inform a user, section specified by him doesn't exist
+                C.LOG.error(sys.exc_info()[1])

config_tempest/tests/test_tempest_conf.py

@@ -101,7 +101,7 @@ class TestTempestConf(BaseConfigTempestTest):
                 self.assertTrue(ext in conf_exts)
 
     def test_remove_values_having_hyphen(self):
-        api_exts = "dvr, l3-flavors, rbac-policies, project-id"
+        api_exts = "dvr,l3-flavors,rbac-policies,project-id"
         remove_exts = ["dvr", "project-id"]
         remove = {
             "network-feature-enabled.api_extensions": remove_exts
@@ -125,3 +125,41 @@ class TestTempestConf(BaseConfigTempestTest):
         self.conf.remove_values({"section.notExistKey": []})
         # check if LOG.error was called
         self.assertTrue(mock_logging.error.called)
+
+    def test_append_values(self):
+        api_exts = "dvr,l3-flavors,rbac-policies"
+        add_exts = ["dvr", "project-id"]
+        add = {
+            "compute-feature-enabled.api_extensions": add_exts
+        }
+        self.conf = self._get_conf("v2.0", "v3")
+        self.conf.set("compute-feature-enabled", "api_extensions", api_exts)
+        self.conf.append_values(add)
+        conf_exts = self.conf.get("compute-feature-enabled", "api_extensions")
+        conf_exts = conf_exts.split(',')
+        self.assertEqual(len(conf_exts), 4)
+        self.assertTrue("project-id" in conf_exts)
+
+    def test_append_values_with_overrides(self):
+        # Test if --add option can override an option which was
+        # passed to python-tempestconf as an override, it shouldn't
+        api_exts = "dvr,l3-flavors,rbac-policies"
+        add_exts = ["dvr", "project-id"]
+        add = {
+            "compute-feature-enabled.api_extensions": add_exts
+        }
+        self.conf = self._get_conf("v2.0", "v3")
+        # let's simulate a situation when the following apis were set
+        # via overrides => they are set with the priority
+        self.conf.set("compute-feature-enabled", "api_extensions",
+                      api_exts, priority=True)
+        self.conf.append_values(add)
+        conf_exts = self.conf.get("compute-feature-enabled", "api_extensions")
+        conf_exts = conf_exts.split(',')
+        # if there are still 3 extensions, no new was added
+        self.assertEqual(len(conf_exts), 3)
+        # option added via --add shouldn't be there
+        self.assertFalse("project-id" in conf_exts)
+        self.assertTrue("dvr" in conf_exts)
+        self.assertTrue("l3-flavors" in conf_exts)
+        self.assertTrue("rbac-policies" in conf_exts)

doc/source/user/usage.rst

@@ -103,13 +103,6 @@ The generated ``tempest.conf`` will look like:
     <omitted some content>
 
 
-.. note::
-
-    -\\-`remove`_ option will remove even values set as overrides
-
-    .. _remove: ./usage.html#prevent-some-key-value-pairs-to-be-set-in-tempest-conf
-
-
 Prevent some key-value pairs to be set in tempest.conf
 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
@@ -158,6 +151,34 @@ removed.
 
     .. _overrides: ./usage.html#override-values
 
+.. note::
+
+    This argument's functionality is opposite to ``--append`` one, see
+    `Append values to tempest.conf`_
+
+
+Append values to tempest.conf
++++++++++++++++++++++++++++++
+
+In a case when ``python-tempestconf`` is not able to discover some wanted
+api_extensions, you can make ``python-tempestconf`` append any extensions
+by using ``--append`` argument.
+
+The following will make ``python-tempestconf`` append my_ext extension to
+compute-feature-enabled.api_extensions and tag and tag-ext extensions to
+network-feature-enabled.api_extensions.
+
+.. code-block:: shell-session
+
+    $ discover-tempest-config \
+        --append compute-feature-enabled.api_extensions=my_ext \
+        --append network-feature-enabled.api_extensions=tag,tag-ext
+
+.. note::
+
+    This argument's functionality is opposite to ``--remove`` one, see
+    `Prevent some key-value pairs to be set in tempest.conf`_
+
 
 Usage with tempest accounts file
 ++++++++++++++++++++++++++++++++

releasenotes/notes/Add-argument-which-allows-users-to-add-extensions-f7b82af27d603c18.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    --append argument appends a value or values to the specified
+    section.key pair. It may be helpful in cases when a user wants to add
+    custom extensions to tempest.conf in an automated job.
+    Argument format for adding values:
+    [--append SECTION.KEY=VALUE[,VALUE]]
+
+    If a section or an option specified in CLI does not exist, tempestconf will
+    inform a user about that in logging output.