Allow to specify auto_security_group in default templates

The aim of this patch is to provide ability to specify
auto_security_group and security groups in default templates.

Change-Id: I304f4965580a9737c826e2862a70970a53cd5db8
Closes-bug: 1470177

sahara/db/templates/api.py

@@ -55,7 +55,13 @@ node_group_template_opts = [
                help='Flavor id field for a node group template.'),
 
     cfg.StrOpt('floating_ip_pool',
-               help='Floating ip pool field for a node group template.')
+               help='Floating ip pool field for a node group template.'),
+    cfg.BoolOpt('auto_security_group',
+                default=False,
+                help='Auto security group field for node group template.'),
+    cfg.ListOpt('security_groups',
+                default=[],
+                help='Security group field for node group template.')
 ]
 
 # Options that we allow to be replaced in a cluster template
@@ -260,12 +266,7 @@ def substitute_config_values(configs, template, path):
 
     for opt, value in six.iteritems(configs):
         if opt in opt_names and opt in template:
-            if value is None:
+            template[opt] = value
-                template[opt] = None
-            else:
-                # Use args to allow for keyword arguments to format
-                args = {opt: value}
-                template[opt] = template[opt].format(**args)
 
 
 def get_configs(section):

sahara/plugins/default_templates/cdh/version_5_0_0/datanode.json

@@ -7,5 +7,7 @@
     ],
     "name": "cdh-5-default-datanode",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/cdh/version_5_0_0/namenode.json

@@ -10,5 +10,7 @@
     ],
     "name": "cdh-5-default-namenode",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/cdh/version_5_0_0/secondary-namenode.json

@@ -8,5 +8,7 @@
     ],
     "name": "cdh-5-default-secondary-namenode",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/cdh/version_5_3_0/datanode.json

@@ -7,5 +7,7 @@
     ],
     "name": "cdh-530-default-datanode",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/cdh/version_5_3_0/namenode.json

@@ -10,5 +10,7 @@
     ],
     "name": "cdh-530-default-namenode",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/cdh/version_5_3_0/secondary-namenode.json

@@ -8,5 +8,7 @@
     ],
     "name": "cdh-530-default-secondary-namenode",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/cdh/version_5_4_0/datanode.json

@@ -7,5 +7,7 @@
     ],
     "name": "cdh-540-default-datanode",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/cdh/version_5_4_0/namenode.json

@@ -10,5 +10,7 @@
     ],
     "name": "cdh-540-default-namenode",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/cdh/version_5_4_0/secondary-namenode.json

@@ -8,5 +8,7 @@
     ],
     "name": "cdh-540-default-secondary-namenode",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/hdp/version_2_0_6/master.json

@@ -10,5 +10,7 @@
     ],
     "name": "hdp-206-default-master",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/hdp/version_2_0_6/secondary-master.json

@@ -8,5 +8,7 @@
     ],
     "name": "hdp-206-default-secondary-master",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/hdp/version_2_0_6/worker.json

@@ -7,5 +7,7 @@
     ],
     "name": "hdp-206-default-worker",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/mapr/3_1_1/master.json

@@ -11,5 +11,7 @@
     ],
     "name": "mapr-311-default-master",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/mapr/3_1_1/worker.json

@@ -7,5 +7,7 @@
     ],
     "name": "mapr-311-default-worker",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/mapr/4_0_1_mrv1/master.json

@@ -11,5 +11,7 @@
     ],
     "name": "mapr-401mr1-default-master",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/mapr/4_0_1_mrv1/worker.json

@@ -7,5 +7,7 @@
     ],
     "name": "mapr-401mr1-default-worker",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/mapr/4_0_1_mrv2/master.json

@@ -12,5 +12,7 @@
     ],
     "name": "mapr-401mr2-default-master",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/mapr/4_0_1_mrv2/worker.json

@@ -7,5 +7,7 @@
     ],
     "name": "mapr-401mr2-default-worker",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/mapr/4_0_2_mrv2/master.json

@@ -12,5 +12,7 @@
     ],
     "name": "mapr-402mr2-default-master",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/mapr/4_0_2_mrv2/worker.json

@@ -7,5 +7,7 @@
     ],
     "name": "mapr-402mr2-default-worker",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/mapr/spark/master.json

@@ -11,5 +11,7 @@
     ],
     "name": "mapr-spark-default-master",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/mapr/spark/worker.json

@@ -7,5 +7,7 @@
     ],
     "name": "mapr-spark-default-worker",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/spark/v1_0_0/master.json

@@ -7,5 +7,7 @@
     ],
     "name": "spark-100-default-master",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/spark/v1_0_0/slave.json

@@ -7,5 +7,7 @@
     ],
     "name": "spark-100-default-slave",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/template.conf

@@ -8,6 +8,7 @@ flavor_id = 2
 # of the master node to 4 which is m1.large in the
 # default flavor set
 flavor_id = 4
+auto_security_group = true
 
 [cdh-530-default-namenode]
 # For the CDH plugin, version 5.3.0, set the flavor_id

sahara/plugins/default_templates/vanilla/v2_6_0/master.json

@@ -8,5 +8,7 @@
     ],
     "name": "vanilla-260-default-master",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/vanilla/v2_6_0/secondary-master.json

@@ -8,5 +8,7 @@
     ],
     "name": "vanilla-260-default-secondary-master",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/plugins/default_templates/vanilla/v2_6_0/worker.json

@@ -7,5 +7,7 @@
     ],
     "name": "vanilla-260-default-worker",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    "security_groups": "{security_groups}"
 }

sahara/tests/unit/db/templates/test_update.py

@@ -57,7 +57,9 @@ master_json = {
     ],
     "name": "vanilla-260-default-master",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    'security_groups': "{security_groups}"
 }
 
 worker_json = {
@@ -69,7 +71,9 @@ worker_json = {
     ],
     "name": "vanilla-260-default-worker",
     "floating_ip_pool": "{floating_ip_pool}",
-    "flavor_id": "{flavor_id}"
+    "flavor_id": "{flavor_id}",
+    "auto_security_group": "{auto_security_group}",
+    'security_groups': "{security_groups}"
 }
 
 
@@ -241,7 +245,8 @@ class TemplateUpdateTestCase(base.ConductorManagerTestCase):
         files = self._write_files(
             tempdir, [cluster_json, master_json, worker_json, some_other_json])
 
-        get_configs.return_value = {"flavor_id": 2}
+        get_configs.return_value = {"flavor_id": '2', 'security_groups': [],
+                                    'auto_security_group': False}
         option_values = {"plugin_name": None,
                          "plugin_version": None}
         template_api.set_conf(Config(option_values))
@@ -286,7 +291,12 @@ class TemplateUpdateTestCase(base.ConductorManagerTestCase):
         files = self._write_files(
             tempdir, [cluster_json, master_json, worker_json])
 
-        get_configs.return_value = {"flavor_id": 2}
+        get_configs.return_value = {
+            "flavor_id": '2',
+            'security_groups': [],
+            'auto_security_group': False
+        }
+
         option_values = {"plugin_name": None,
                          "plugin_version": None}
         template_api.set_conf(Config(option_values))
@@ -312,7 +322,8 @@ class TemplateUpdateTestCase(base.ConductorManagerTestCase):
         files = self._write_files(
             tempdir, [cluster_json, master_json, worker_json])
 
-        get_configs.return_value = {"flavor_id": 2}
+        get_configs.return_value = {"flavor_id": '2', 'security_groups': [],
+                                    'auto_security_group': False}
         option_values = {"plugin_name": None,
                          "plugin_version": None}
         template_api.set_conf(Config(option_values))
@@ -603,7 +614,7 @@ class TemplateUpdateTestCase(base.ConductorManagerTestCase):
 
     @mock.patch("sahara.db.templates.api.get_configs")
     @mock.patch("sahara.db.templates.api.add_config_section_for_template")
-    def test_do_update(self, add_config, get_configs):
+    def test_do_update_trash(self, add_config, get_configs):
         self.logger.clear_log()
         ctx = context.ctx()
 
@@ -611,15 +622,19 @@ class TemplateUpdateTestCase(base.ConductorManagerTestCase):
 
         self._write_files(tempdir, [cluster_json, master_json, worker_json])
 
-        get_configs.return_value = {"flavor_id": 2,
+        get_configs.return_value = {
-                                    "neutron_management_network": uuid.uuid4()}
+            "flavor_id": '2',
+            "neutron_management_network": str(uuid.uuid4()),
+            'auto_security_group': True,
+            'security_groups': [],
+        }
+
         option_values = {"tenant_id": ctx.tenant_id,
                          "directory": tempdir,
                          "norecurse": None,
                          "plugin_name": None,
                          "plugin_version": None}
         template_api.set_conf(Config(option_values))
-
         template_api.do_update()
 
         ngs = self.api.node_group_template_get_all(ctx)
@@ -644,8 +659,11 @@ class TemplateUpdateTestCase(base.ConductorManagerTestCase):
 
         self._write_files(tempdir, [cluster_json, master_json, worker_json])
 
-        get_configs.return_value = {"flavor_id": 2,
+        get_configs.return_value = {
-                                    "neutron_management_network": uuid.uuid4()}
+            "flavor_id": '2',
+            "neutron_management_network": str(uuid.uuid4())
+        }
+
         option_values = {"tenant_id": ctx.tenant_id,
                          "directory": tempdir,
                          "norecurse": None,
@@ -679,8 +697,10 @@ class TemplateUpdateTestCase(base.ConductorManagerTestCase):
 
         self._write_files(tempdir, [cluster_json, master_json, worker_json])
 
-        get_configs.return_value = {"flavor_id": 2,
+        get_configs.return_value = {
-                                    "neutron_management_network": uuid.uuid4()}
+            "flavor_id": '2',
+            "neutron_management_network": str(uuid.uuid4())
+        }
         option_values = {"tenant_id": ctx.tenant_id,
                          "directory": tempdir,
                          "norecurse": None,
@@ -714,8 +734,11 @@ class TemplateUpdateTestCase(base.ConductorManagerTestCase):
 
         self._write_files(tempdir, [cluster_json, master_json, worker_json])
 
-        get_configs.return_value = {"flavor_id": 2,
+        get_configs.return_value = {
-                                    "neutron_management_network": uuid.uuid4()}
+            "flavor_id": '2',
+            "neutron_management_network": str(uuid.uuid4())
+        }
+
         option_values = {"tenant_id": ctx.tenant_id,
                          "directory": tempdir,
                          "norecurse": None,
@@ -755,8 +778,11 @@ class TemplateUpdateTestCase(base.ConductorManagerTestCase):
 
         self._write_files(tempdir, [cluster_json, master_json, worker_json])
 
-        get_configs.return_value = {"flavor_id": 2,
+        get_configs.return_value = {
-                                    "neutron_management_network": uuid.uuid4()}
+            "flavor_id": '2',
+            "neutron_management_network": str(uuid.uuid4())
+        }
+
         option_values = {"tenant_id": ctx.tenant_id,
                          "directory": tempdir,
                          "norecurse": None,