Add list of open ports for HDP plugin

It's needed for correct working auto security groups

Change-Id: Ie555d957da9b6081994b932c74a6e19e00682e66
Closes-bug: #1391518

sahara/plugins/hdp/ambariplugin.py

@@ -362,6 +362,11 @@ class AmbariPlugin(p.ProvisioningPluginBase):
             for server in servers:
                 server.configure_topology(topology_str)
 
+    def get_open_ports(self, node_group):
+        handler = self.version_factory.get_version_handler(
+            node_group.cluster.hadoop_version)
+        return handler.get_open_ports(node_group)
+
 
 class AmbariInfo(object):
     def __init__(self, host, port, user, password):

sahara/plugins/hdp/versions/abstractversionhandler.py

@@ -61,3 +61,7 @@ class AbstractVersionHandler(object):
     @abc.abstractmethod
     def get_edp_engine(self, cluster, job_type):
         return
+
+    @abc.abstractmethod
+    def get_open_ports(self, node_group):
+        return []

sahara/plugins/hdp/versions/version_1_3_2/versionhandler.py

@@ -125,6 +125,34 @@ class VersionHandler(avm.AbstractVersionHandler):
             return edp_engine.EdpOozieEngine(cluster)
         return None
 
+    def get_open_ports(self, node_group):
+        ports = [8660]  # for Ganglia
+
+        ports_map = {
+            'AMBARI_SERVER': [8080, 8440, 8441],
+            'NAMENODE': [50070, 50470, 8020, 9000],
+            'DATANODE': [50075, 50475, 50010, 50020],
+            'SECONDARY_NAMENODE': [50090],
+            'JOBTRACKER': [50030, 8021],
+            'TASKTRACKER': [50060],
+            'HISTORYSERVER': [51111],
+            'HIVE_SERVER': [10000],
+            'HIVE_METASTORE': [9083],
+            'HBASE_MASTER': [60000, 60010],
+            'HBASE_REGIONSERVER': [60020, 60030],
+            'WEBHCAT_SERVER': [50111],
+            'GANGLIA_SERVER': [8661, 8662, 8663, 8651],
+            'MYSQL_SERVER': [3306],
+            'OOZIE_SERVER': [11000, 11001],
+            'ZOOKEEPER_SERVER': [2181, 2888, 3888],
+            'NAGIOS_SERVER': [80]
+        }
+        for process in node_group.node_processes:
+            if process in ports_map:
+                ports.extend(ports_map[process])
+
+        return ports
+
 
 class AmbariClient(object):
 

sahara/plugins/hdp/versions/version_2_0_6/versionhandler.py

@@ -114,6 +114,34 @@ class VersionHandler(avm.AbstractVersionHandler):
             return edp_engine.EdpOozieEngine(cluster)
         return None
 
+    def get_open_ports(self, node_group):
+        ports = [8660]  # for Ganglia
+
+        ports_map = {
+            'AMBARI_SERVER': [8080, 8440, 8441],
+            'NAMENODE': [50070, 50470, 8020, 9000],
+            'DATANODE': [50075, 50475, 50010, 8010],
+            'SECONDARY_NAMENODE': [50090],
+            'HISTORYSERVER': [19888],
+            'RESOURCEMANAGER': [8025, 8041, 8050],
+            'NODEMANAGER': [45454],
+            'HIVE_SERVER': [10000],
+            'HIVE_METASTORE': [9083],
+            'HBASE_MASTER': [60000, 60010],
+            'HBASE_REGIONSERVER': [60020, 60030],
+            'WEBHCAT_SERVER': [50111],
+            'GANGLIA_SERVER': [8661, 8662, 8663, 8651],
+            'MYSQL_SERVER': [3306],
+            'OOZIE_SERVER': [11000, 11001],
+            'ZOOKEEPER_SERVER': [2181, 2888, 3888],
+            'NAGIOS_SERVER': [80]
+        }
+        for process in node_group.node_processes:
+            if process in ports_map:
+                ports.extend(ports_map[process])
+
+        return ports
+
 
 class AmbariClient(object):
 

sahara/tests/integration/tests/gating/test_hdp2_gating.py

@@ -57,6 +57,9 @@ class HDP2GatingTest(swift.SwiftTest, scaling.ScalingTest,
             'description': 'test node group template for HDP plugin',
             'node_processes': self.hdp2_config.MASTER_NODE_PROCESSES,
             'floating_ip_pool': self.floating_ip_pool,
+            # TODO(sreshetniak): Enable auto security group when #1392738 is
+            # resolved
+            'auto_security_group': False,
             'node_configs': {}
         }
         self.ng_tmpl_rm_nn_id = self.create_node_group_template(**template)
@@ -70,6 +73,9 @@ class HDP2GatingTest(swift.SwiftTest, scaling.ScalingTest,
             'description': 'test node group template for HDP plugin',
             'node_processes': self.hdp2_config.WORKER_NODE_PROCESSES,
             'floating_ip_pool': self.floating_ip_pool,
+            # TODO(sreshetniak): Enable auto security group when #1392738 is
+            # resolved
+            'auto_security_group': False,
             'node_configs': {}
         }
         self.ng_tmpl_nm_dn_id = self.create_node_group_template(**template)

sahara/tests/integration/tests/gating/test_hdp_gating.py

@@ -73,7 +73,8 @@ class HDPGatingTest(cinder.CinderVolumeTest, edp.EDPTest,
                 volumes_size=volumes_size,
                 node_processes=self.hdp_config.WORKER_NODE_PROCESSES,
                 node_configs={},
-                floating_ip_pool=floating_ip_pool
+                floating_ip_pool=floating_ip_pool,
+                auto_security_group=True
             )
             node_group_template_id_list.append(node_group_template_tt_dn_id)
 
@@ -98,7 +99,9 @@ class HDPGatingTest(cinder.CinderVolumeTest, edp.EDPTest,
                         node_processes=self.hdp_config.MASTER_NODE_PROCESSES,
                         node_configs={},
                         floating_ip_pool=floating_ip_pool,
-                        count=1),
+                        count=1,
+                        auto_security_group=True
+                    ),
                     dict(
                         name='worker-node-tt-dn',
                         node_group_template_id=node_group_template_tt_dn_id,