Enhance Shaker to support custom user options

Issue: Some of our more complex tests require injecting "custom" data
into the heat stacks in order to run correctly. For example in order to
run certain Contrail based tests we need to set "contrail-asn:(some
asn number)". The asn number varies between cloud deployments, so we
dynamically set this field in a Heat environment file. However this is
a very specific field that other Shaker users might not need or
understand it, so this commit allows for a more generic approach.

This commit aims to let the user specify "custom" user defined options
similar to the matrix config parameter. Since this opts is not
directly referenced anywhere in code it's main use is to inject data
into heat environment files. The YAML format allows flexibility for any
object the user might want to add during their scenario
run or it can be left blank without any issues.

Change-Id: I96b6e578eb59813e5e0c8a2fe7a14c5ecc369be7

.gitignore

@@ -13,6 +13,7 @@ doc/build/
 doc/source/api/
 *.egg-info
 *.egg
+*.eggs
 .autogenerated
 .coverage
 .stestr/

doc/source/tools/shaker-all-in-one.txt

@@ -2,7 +2,8 @@ usage: shaker-all-in-one [-h] [--agent-join-timeout AGENT_JOIN_TIMEOUT]
                          [--agent-loss-timeout AGENT_LOSS_TIMEOUT]
                          [--artifacts-dir ARTIFACTS_DIR] [--book BOOK]
                          [--cleanup] [--cleanup-on-error] [--config-dir DIR]
-                         [--config-file PATH] [--debug]
+                         [--config-file PATH]
+                         [--custom_user_opts CUSTOM_USER_OPTS] [--debug]
                          [--dns-nameservers DNS_NAMESERVERS]
                          [--external-net EXTERNAL_NET]
                          [--flavor-disk FLAVOR_DISK]
@@ -69,6 +70,14 @@ optional arguments:
   --config-file PATH    Path to a config file to use. Multiple config files
                         can be specified, with values in later files taking
                         precedence. Defaults to None.
+  --custom_user_opts CUSTOM_USER_OPTS
+                        Set custom user option parameters for the scenario.
+                        The value is specified in YAML, e.g. custom_user_opts
+                        = { key1:value1, key2:value2} The values specified can
+                        be referenced in the usual python way. e.g. {{
+                        CONF.custom_user_opts['key1'] }}. This option is
+                        useful to inject custom values into heat environment
+                        files
   --debug, -d           If set to true, the logging level will be set to DEBUG
                         instead of the default INFO level.
   --dns-nameservers DNS_NAMESERVERS

doc/source/tools/shaker-spot.txt

@@ -1,5 +1,6 @@
 usage: shaker-spot [-h] [--artifacts-dir ARTIFACTS_DIR] [--book BOOK]
-                   [--config-dir DIR] [--config-file PATH] [--debug]
+                   [--config-dir DIR] [--config-file PATH]
+                   [--custom_user_opts CUSTOM_USER_OPTS] [--debug]
                    [--log-config-append PATH] [--log-date-format DATE_FORMAT]
                    [--log-dir LOG_DIR] [--log-file PATH] [--matrix MATRIX]
                    [--no-report-on-error] [--nodebug] [--nono-report-on-error]
@@ -29,6 +30,14 @@ optional arguments:
   --config-file PATH    Path to a config file to use. Multiple config files
                         can be specified, with values in later files taking
                         precedence. Defaults to None.
+  --custom_user_opts CUSTOM_USER_OPTS
+                        Set custom user option parameters for the scenario.
+                        The value is specified in YAML, e.g. custom_user_opts
+                        = { key1:value1, key2:value2} The values specified can
+                        be referenced in the usual python way. e.g. {{
+                        CONF.custom_user_opts['key1'] }}. This option is
+                        useful to inject custom values into heat environment
+                        files
   --debug, -d           If set to true, the logging level will be set to DEBUG
                         instead of the default INFO level.
   --log-config-append PATH, --log-config PATH, --log_config PATH

doc/source/tools/shaker.txt

@@ -2,7 +2,8 @@ usage: shaker [-h] [--agent-join-timeout AGENT_JOIN_TIMEOUT]
               [--agent-loss-timeout AGENT_LOSS_TIMEOUT]
               [--artifacts-dir ARTIFACTS_DIR] [--book BOOK]
               [--cleanup-on-error] [--config-dir DIR] [--config-file PATH]
-              [--debug] [--dns-nameservers DNS_NAMESERVERS]
+              [--custom_user_opts CUSTOM_USER_OPTS] [--debug]
+              [--dns-nameservers DNS_NAMESERVERS]
               [--external-net EXTERNAL_NET] [--flavor-name FLAVOR_NAME]
               [--image-name IMAGE_NAME] [--log-config-append PATH]
               [--log-date-format DATE_FORMAT] [--log-dir LOG_DIR]
@@ -55,6 +56,14 @@ optional arguments:
   --config-file PATH    Path to a config file to use. Multiple config files
                         can be specified, with values in later files taking
                         precedence. Defaults to None.
+  --custom_user_opts CUSTOM_USER_OPTS
+                        Set custom user option parameters for the scenario.
+                        The value is specified in YAML, e.g. custom_user_opts
+                        = { key1:value1, key2:value2} The values specified can
+                        be referenced in the usual python way. e.g. {{
+                        CONF.custom_user_opts['key1'] }}. This option is
+                        useful to inject custom values into heat environment
+                        files
   --debug, -d           If set to true, the logging level will be set to DEBUG
                         instead of the default INFO level.
   --dns-nameservers DNS_NAMESERVERS

etc/shaker.conf

@@ -246,6 +246,13 @@
 # to SCENARIO_COMPUTE_NODES (integer value)
 #scenario_compute_nodes = <None>
 
+# Set custom user option parameters for the scenario. The value is specified in
+# YAML, e.g. custom_user_opts = { key1:value1, key2:value2} The values
+# specified can be referenced in the usual python way. e.g. {{
+# CONF.custom_user_opts['key1'] }}. This option is useful to inject custom
+# values into heat environment files (string value)
+#custom_user_opts = <None>
+
 # Timeout to treat agent as lost in seconds, defaults to
 # env[SHAKER_AGENT_LOSS_TIMEOUT] (integer value)
 #agent_loss_timeout = 60

shaker/engine/config.py

@@ -166,7 +166,6 @@ OPENSTACK_OPTS = [
                 default=(utils.env('SHAKER_CLEANUP_ON_ERROR') or True),
                 help='Clean up the heat-stack upon any error occurred during '
                      'scenario execution.'),
-
 ]
 
 SERVER_AGENT_OPTS = [
@@ -228,6 +227,16 @@ SCENARIO_OPTS = [
                     'override the compute_nodes accomodation setting in the '
                     'scenario test definition. '
                     'Defaults to SCENARIO_COMPUTE_NODES'),
+    cfg.Opt('custom_user_opts',
+            default=utils.env('CUSTOM_USER_OPTS'),
+            type=Yaml(),
+            help='Set custom user option parameters for the scenario. '
+                 'The value is specified in YAML, e.g. '
+                 'custom_user_opts = { key1:value1, key2:value2} '
+                 'The values specified can be referenced in the usual '
+                 'python way. e.g. {{ CONF.custom_user_opts[\'key1\'] }}. '
+                 'This option is useful to inject custom values into heat '
+                 'environment files'),
 ]
 
 SERVER_OPTS = SCENARIO_OPTS + SERVER_AGENT_OPTS