Add ability to remove any option from tempest.conf

A user will be able to specify which values should not be included in
tempest configuration file through --remove argument.
For ommiting value(s):
--remove section.key=value[,value2[...]]

For ommiting all values in section.key:
--remove section.key

Patch removes network.remove-extension option, because the patch is more
generic solution of the same issue.

Change-Id: I0375f2bbfa3bb7db4f9b81ea1518e86d725c30a3

config_tempest/config_tempest.py

@@ -38,6 +38,7 @@ import ConfigParser
 import logging
 import os
 import shutil
+import sys
 import tempest.config
 import urllib2
 
@@ -176,6 +177,11 @@ def main():
     configure_discovered_services(conf, services)
     configure_boto(conf, services)
     configure_horizon(conf)
+
+    # remove all unwanted values if were specified
+    if args.remove != {}:
+        LOG.info("Removing configuration: %s", str(args.remove))
+        conf.remove_values(args)
     LOG.info("Creating configuration file %s", os.path.abspath(args.out))
     with open(args.out, 'w') as f:
         conf.write(f)
@@ -221,6 +227,12 @@ def parse_arguments():
     parser.add_argument('--network-id',
                         help="""The ID of an existing network in our openstack
                                 instance with external connectivity""")
+    parser.add_argument('--remove', action='append', default=[],
+                        metavar="SECTION.KEY=VALUE[,VALUE]",
+                        help="""key value pair to be removed from
+                                configuration file.
+                                For example: --remove identity.username=myname
+                                --remove feature-enabled.api_ext=http,https""")
 
     args = parser.parse_args()
 
@@ -229,9 +241,33 @@ def parse_arguments():
                         " together, since creating" " resources requires"
                         " admin rights")
     args.overrides = parse_overrides(args.overrides)
+    args.remove = parse_values_to_remove(args.remove)
     return args
 
 
+def parse_values_to_remove(options):
+    """Manual parsing of remove arguments.
+
+    :options list of arguments following --remove argument
+    :returns dict containing key paths with values to be removed
+    EXAMPLE: {'identity.username': [myname],
+              'identity-feature-enabled.api_extensions': [http, https]}
+    """
+    parsed = {}
+    for argument in options:
+        if len(argument.split('=')) == 2:
+            section, values = argument.split('=')
+            if len(section.split('.')) != 2:
+                raise Exception("Missing dot. The option --remove has to"
+                                "come in the format 'section.key=value,"
+                                " but got '%s'." % (argument))
+            parsed[section] = values.split(',')
+        else:
+            # missing equal sign, all values in section.key will be deleted
+            parsed[argument] = []
+    return parsed
+
+
 def parse_overrides(overrides):
     """Manual parsing of positional arguments.
 
@@ -464,6 +500,37 @@ class TempestConf(ConfigParser.SafeConfigParser):
         ConfigParser.SafeConfigParser.set(self, section, key, value)
         return True
 
+    def remove_values(self, args):
+        """Remove values from configuration file specified in arguments.
+
+        :args - arguments object
+        """
+        for key_path in args.remove:
+            section, key = key_path.split('.')
+            try:
+                conf_values = self.get(section, key).split(',')
+                remove = args.remove[key_path]
+                if len(remove) == 0:
+                    # delete all values in section.key
+                    self.remove_option(section, key)
+                elif len(conf_values) == 1:
+                    # make sure only the value specified by user
+                    # will be deleted if in the key is other value
+                    # than expected, ignore it
+                    if conf_values[0] in args.remove[key_path]:
+                        self.remove_option(section, key)
+                else:
+                    # exclude all unwanted values from the list
+                    # and preserve the original order of items
+                    conf_values = [v for v in conf_values if v not in remove]
+                    self.set(section, key, ",".join(conf_values))
+            except ConfigParser.NoOptionError:
+                # only inform a user, option specified by him doesn't exist
+                LOG.error(sys.exc_info()[1])
+            except ConfigParser.NoSectionError:
+                # only inform a user, section specified by him doesn't exist
+                LOG.error(sys.exc_info()[1])
+
 
 def create_tempest_users(tenants_client, roles_client, users_client, conf,
                          services):
@@ -783,14 +850,6 @@ def configure_discovered_services(conf, services):
 
     # set service extensions
     keystone_v3_support = conf.get('identity-feature-enabled', 'api_v3')
-    # Currently neutron ext-list provides available api-extension but
-    # does not provide enabled extension due to bug in dvr.
-    # So we are removing dvr from neutron api-extension list.
-    # We can remove dvr from extension list using network.remove-extension dvr
-    # https://bugs.launchpad.net/neutron/+bug/1450067
-    if not conf.has_option('network', 'remove-extension'):
-        conf.set('network', 'remove-extension', '')
-    network_extension = conf.get('network', 'remove-extension')
     for service, ext_key in SERVICE_EXTENSION_KEY.iteritems():
         if service in services:
             extensions = ','.join(services[service].get('extensions', ""))
@@ -803,11 +862,6 @@ def configure_discovered_services(conf, services):
                     conf.get("identity", "uri_v3"))
                 extensions = list(set(extensions.split(',') + identity_v3_ext))
                 extensions = ','.join(extensions)
-            elif service == 'network' and network_extension:
-                extensions = set(str(extensions).split(','))
-                remove_ext = set(network_extension.split(','))
-                extensions = list(extensions.difference(remove_ext))
-                extensions = ','.join(extensions)
             conf.set(service + '-feature-enabled', ext_key, extensions)
 
 

config_tempest/tests/test_config_tempest.py

@@ -15,6 +15,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from argparse import Namespace
 from config_tempest import config_tempest as tool
 from config_tempest.tests.base import BaseConfigTempestTest
 from fixtures import MonkeyPatch
@@ -151,6 +152,43 @@ class TestTempestConf(BaseConfigTempestTest):
         self.assertFalse(self.conf.get_bool_value("False"))
         self.assertRaises(ValueError, self.conf.get_bool_value, "no")
 
+    def test_remove_values(self):
+        api_exts = "router_availability_zone,rbac-policies,pagination,sorting,"
+        api_exts += "standard-attr-description,router,binding,metering,"
+        api_exts += "allowed-address-pairs,project-id,dvr,l3-flavors,tag-ext"
+        remove_exts = ["router", "project-id", "dvr"]
+        args = Namespace(
+            remove={
+                "identity.username": ["demo"],
+                "identity.tenant_name": ["tenant"],
+                "compute.image_ssh_user": ["rhel", "cirros"],
+                "network-feature-enabled.api_extensions": remove_exts
+            }
+        )
+        self.conf = self._get_conf("v2.0", "v3")
+        self.conf.set("compute", "image_ssh_user", "cirros")
+        self.conf.set("network-feature-enabled", "api_extensions", api_exts)
+        self.conf.remove_values(args)
+        self.assertFalse(self.conf.has_option("identity", "username"))
+        self.assertTrue(self.conf.has_option("identity", "tenant_name"))
+        self.assertFalse(self.conf.has_option("compute", "image_ssh_user"))
+        conf_exts = self.conf.get("network-feature-enabled", "api_extensions")
+        conf_exts = conf_exts.split(',')
+        for ext in api_exts.split(','):
+            if ext in remove_exts:
+                self.assertFalse(ext in conf_exts)
+            else:
+                self.assertTrue(ext in conf_exts)
+
+    @mock.patch('config_tempest.config_tempest.LOG')
+    def test_remove_not_defined_values(self, mock_logging):
+        self.conf.remove_values(Namespace(remove={"notExistSection.key": []}))
+        # check if LOG.error was called
+        self.assertTrue(mock_logging.error.called)
+        self.conf.remove_values(Namespace(remove={"section.notExistKey": []}))
+        # check if LOG.error was called
+        self.assertTrue(mock_logging.error.called)
+
 
 class TestConfigTempest(BaseConfigTempestTest):
 

releasenotes/notes/remove-configuration-values-from-cli-c5c8eb496e1ed3d7.yaml

@@ -0,0 +1,15 @@
+---
+prelude: >
+    Add a new ability to remove any configuration values from tempest.conf
+features:
+  - |
+    Add a new parameter --remove to specify which configuration values should
+    not be included in tempest configuration file.
+    Parameter format for removing values:
+    [--remove SECTION.KEY=VALUE[,VALUE]]
+
+    Parameter format for removing all values in key.section:
+    [--remove SECTION.KEY]
+
+    If a section or an option specified in CLI does not exist, tempestconf will
+    inform a user about that in logging output.