From f1f8602da9fe3868744aa1f77ebe150e1b226d46 Mon Sep 17 00:00:00 2001 From: Vitaly Gridnev Date: Fri, 11 Mar 2016 16:06:45 +0300 Subject: [PATCH] Fix a mess in config helpers Some configs are loaded to wrong targets, that cause possible issues with pushing configs. Also, added empty services for each process in versionhandler. Change-Id: I8abb429e8ab5bf07b8ea5fb2165987a5dbb666da Partial-bug: 1556064 --- sahara/plugins/cdh/v5_4_0/config_helper.py | 28 +++++++++++++++------ sahara/plugins/cdh/v5_4_0/versionhandler.py | 22 +++++++++++++++- sahara/plugins/cdh/v5_5_0/config_helper.py | 28 +++++++++++++++------ sahara/plugins/cdh/v5_5_0/versionhandler.py | 22 +++++++++++++++- 4 files changed, 84 insertions(+), 16 deletions(-) diff --git a/sahara/plugins/cdh/v5_4_0/config_helper.py b/sahara/plugins/cdh/v5_4_0/config_helper.py index 3b26a015..8111e1dd 100644 --- a/sahara/plugins/cdh/v5_4_0/config_helper.py +++ b/sahara/plugins/cdh/v5_4_0/config_helper.py @@ -149,20 +149,23 @@ class ConfigHelperV540(c_h.ConfigHelper): 'hdfs-gateway.json', 'HDFS_GATEWAY', 'node') self.journalnode_confs = self._load_and_init_configs( 'hdfs-journalnode.json', 'JOURNALNODE', 'node') + self.yarn_confs = self._load_and_init_configs( 'yarn-service.json', 'YARN', 'cluster') self.resourcemanager_confs = self._load_and_init_configs( - 'yarn-resourcemanager.json', 'YARN_GATEWAY', 'node') + 'yarn-resourcemanager.json', 'RESOURCEMANAGER', 'node') self.nodemanager_confs = self._load_and_init_configs( - 'yarn-nodemanager.json', 'RESOURCEMANAGER', 'node') + 'yarn-nodemanager.json', 'NODEMANAGER', 'node') self.jobhistory_confs = self._load_and_init_configs( - 'yarn-jobhistory.json', 'NODEMANAGER', 'node') + 'yarn-jobhistory.json', 'JOBHISTORY', 'node') self.yarn_gateway_conf = self._load_and_init_configs( - 'yarn-gateway.json', 'JOBHISTORY', 'node') + 'yarn-gateway.json', 'YARN_GATEWAY', 'node') + self.oozie_service_confs = self._load_and_init_configs( 'oozie-service.json', 'OOZIE', 'cluster') self.oozie_role_confs = self._load_and_init_configs( 'oozie-oozie_server.json', 'OOZIE', 'node') + self.hive_service_confs = self._load_and_init_configs( 'hive-service.json', 'HIVE', 'cluster') self.hive_metastore_confs = self._load_and_init_configs( @@ -171,44 +174,54 @@ class ConfigHelperV540(c_h.ConfigHelper): 'hive-hiveserver2.json', 'HIVESERVER', 'node') self.hive_webhcat_confs = self._load_and_init_configs( 'hive-webhcat.json', 'WEBHCAT', 'node') + self.hue_service_confs = self._load_and_init_configs( 'hue-service.json', 'HUE', 'cluster') self.hue_role_confs = self._load_and_init_configs( 'hue-hue_server.json', 'HUE', 'node') + self.spark_service_confs = self._load_and_init_configs( 'spark-service.json', 'SPARK_ON_YARN', 'cluster') self.spark_role_confs = self._load_and_init_configs( 'spark-spark_yarn_history_server.json', 'SPARK_ON_YARN', 'node') + self.zookeeper_server_confs = self._load_and_init_configs( - 'zookeeper-server.json', 'ZOOKEEPER', 'cluster') + 'zookeeper-service.json', 'ZOOKEEPER', 'cluster') self.zookeeper_service_confs = self._load_and_init_configs( - 'zookeeper-service.json', 'ZOOKEEPER', 'node') + 'zookeeper-server.json', 'ZOOKEEPER', 'node') + self.hbase_confs = self._load_and_init_configs( 'hbase-service.json', 'HBASE', 'cluster') self.master_confs = self._load_and_init_configs( 'hbase-master.json', 'MASTER', 'node') self.regionserver_confs = self._load_and_init_configs( 'hbase-regionserver.json', 'REGIONSERVER', 'node') + self.flume_service_confs = self._load_and_init_configs( - 'flume-service.json', 'HDFS', 'cluster') + 'flume-service.json', 'FLUME', 'cluster') self.flume_agent_confs = self._load_and_init_configs( 'flume-agent.json', 'FLUME', 'node') + self.sentry_service_confs = self._load_and_init_configs( 'sentry-service.json', 'SENTRY', 'cluster') self.sentry_server_confs = self._load_and_init_configs( 'sentry-sentry_server.json', 'SENTRY', 'node') + self.solr_service_confs = self._load_and_init_configs( 'solr-service.json', 'SOLR', 'cluster') self.solr_server_confs = self._load_and_init_configs( 'solr-solr_server.json', 'SOLR', 'node') + self.sqoop_service_confs = self._load_and_init_configs( 'sqoop-service.json', 'SQOOP', 'cluster') self.sqoop_server_confs = self._load_and_init_configs( 'sqoop-sqoop_server.json', 'SQOOP', 'node') + self.ks_indexer_service_confs = self._load_and_init_configs( 'ks_indexer-service.json', 'KS_INDEXER', 'cluster') self.ks_indexer_role_confs = self._load_and_init_configs( 'ks_indexer-hbase_indexer.json', 'KS_INDEXER', 'node') + self.impala_service_confs = self._load_and_init_configs( 'impala-service.json', 'IMPALA', 'cluster') self.impala_catalogserver_confs = self._load_and_init_configs( @@ -217,6 +230,7 @@ class ConfigHelperV540(c_h.ConfigHelper): 'impala-impalad.json', 'IMPALAD', 'node') self.impala_statestore_confs = self._load_and_init_configs( 'impala-statestore.json', 'STATESTORE', 'node') + self.kms_service_confs = self._load_and_init_configs( 'kms-service.json', 'KMS', 'cluster') self.kms_kms_confs = self._load_and_init_configs( diff --git a/sahara/plugins/cdh/v5_4_0/versionhandler.py b/sahara/plugins/cdh/v5_4_0/versionhandler.py index d02a4cae..27e73525 100644 --- a/sahara/plugins/cdh/v5_4_0/versionhandler.py +++ b/sahara/plugins/cdh/v5_4_0/versionhandler.py @@ -54,8 +54,28 @@ class VersionHandler(avm.BaseVersionHandler): "SQOOP": ['SQOOP_SERVER'], "SENTRY": ['SENTRY_SERVER'], "KMS": ['KMS'], + "YARN_GATEWAY": [], - "HDFS_GATEWAY": [] + "RESOURCEMANAGER": [], + "NODEMANAGER": [], + "JOBHISTORY": [], + + "HDFS_GATEWAY": [], + 'DATANODE': [], + 'NAMENODE': [], + 'SECONDARYNAMENODE': [], + 'JOURNALNODE': [], + + 'REGIONSERVER': [], + 'MASTER': [], + + 'HIVEMETASTORE': [], + 'HIVESERVER': [], + 'WEBCAT': [], + + 'CATALOGSERVER': [], + 'STATESTORE': [], + 'IMPALAD': [], } def get_edp_engine(self, cluster, job_type): diff --git a/sahara/plugins/cdh/v5_5_0/config_helper.py b/sahara/plugins/cdh/v5_5_0/config_helper.py index 75a7c69f..a3218245 100644 --- a/sahara/plugins/cdh/v5_5_0/config_helper.py +++ b/sahara/plugins/cdh/v5_5_0/config_helper.py @@ -149,20 +149,23 @@ class ConfigHelperV550(c_h.ConfigHelper): 'hdfs-gateway.json', 'HDFS_GATEWAY', 'node') self.journalnode_confs = self._load_and_init_configs( 'hdfs-journalnode.json', 'JOURNALNODE', 'node') + self.yarn_confs = self._load_and_init_configs( 'yarn-service.json', 'YARN', 'cluster') self.resourcemanager_confs = self._load_and_init_configs( - 'yarn-resourcemanager.json', 'YARN_GATEWAY', 'node') + 'yarn-resourcemanager.json', 'RESOURCEMANAGER', 'node') self.nodemanager_confs = self._load_and_init_configs( - 'yarn-nodemanager.json', 'RESOURCEMANAGER', 'node') + 'yarn-nodemanager.json', 'NODEMANAGER', 'node') self.jobhistory_confs = self._load_and_init_configs( - 'yarn-jobhistory.json', 'NODEMANAGER', 'node') + 'yarn-jobhistory.json', 'JOBHISTORY', 'node') self.yarn_gateway_conf = self._load_and_init_configs( - 'yarn-gateway.json', 'JOBHISTORY', 'node') + 'yarn-gateway.json', 'YARN_GATEWAY', 'node') + self.oozie_service_confs = self._load_and_init_configs( 'oozie-service.json', 'OOZIE', 'cluster') self.oozie_role_confs = self._load_and_init_configs( 'oozie-oozie_server.json', 'OOZIE', 'node') + self.hive_service_confs = self._load_and_init_configs( 'hive-service.json', 'HIVE', 'cluster') self.hive_metastore_confs = self._load_and_init_configs( @@ -171,44 +174,54 @@ class ConfigHelperV550(c_h.ConfigHelper): 'hive-hiveserver2.json', 'HIVESERVER', 'node') self.hive_webhcat_confs = self._load_and_init_configs( 'hive-webhcat.json', 'WEBHCAT', 'node') + self.hue_service_confs = self._load_and_init_configs( 'hue-service.json', 'HUE', 'cluster') self.hue_role_confs = self._load_and_init_configs( 'hue-hue_server.json', 'HUE', 'node') + self.spark_service_confs = self._load_and_init_configs( 'spark-service.json', 'SPARK_ON_YARN', 'cluster') self.spark_role_confs = self._load_and_init_configs( 'spark-spark_yarn_history_server.json', 'SPARK_ON_YARN', 'node') + self.zookeeper_server_confs = self._load_and_init_configs( - 'zookeeper-server.json', 'ZOOKEEPER', 'cluster') + 'zookeeper-service.json', 'ZOOKEEPER', 'cluster') self.zookeeper_service_confs = self._load_and_init_configs( - 'zookeeper-service.json', 'ZOOKEEPER', 'node') + 'zookeeper-server.json', 'ZOOKEEPER', 'node') + self.hbase_confs = self._load_and_init_configs( 'hbase-service.json', 'HBASE', 'cluster') self.master_confs = self._load_and_init_configs( 'hbase-master.json', 'MASTER', 'node') self.regionserver_confs = self._load_and_init_configs( 'hbase-regionserver.json', 'REGIONSERVER', 'node') + self.flume_service_confs = self._load_and_init_configs( - 'flume-service.json', 'HDFS', 'cluster') + 'flume-service.json', 'FLUME', 'cluster') self.flume_agent_confs = self._load_and_init_configs( 'flume-agent.json', 'FLUME', 'node') + self.sentry_service_confs = self._load_and_init_configs( 'sentry-service.json', 'SENTRY', 'cluster') self.sentry_server_confs = self._load_and_init_configs( 'sentry-sentry_server.json', 'SENTRY', 'node') + self.solr_service_confs = self._load_and_init_configs( 'solr-service.json', 'SOLR', 'cluster') self.solr_server_confs = self._load_and_init_configs( 'solr-solr_server.json', 'SOLR', 'node') + self.sqoop_service_confs = self._load_and_init_configs( 'sqoop-service.json', 'SQOOP', 'cluster') self.sqoop_server_confs = self._load_and_init_configs( 'sqoop-sqoop_server.json', 'SQOOP', 'node') + self.ks_indexer_service_confs = self._load_and_init_configs( 'ks_indexer-service.json', 'KS_INDEXER', 'cluster') self.ks_indexer_role_confs = self._load_and_init_configs( 'ks_indexer-hbase_indexer.json', 'KS_INDEXER', 'node') + self.impala_service_confs = self._load_and_init_configs( 'impala-service.json', 'IMPALA', 'cluster') self.impala_catalogserver_confs = self._load_and_init_configs( @@ -217,6 +230,7 @@ class ConfigHelperV550(c_h.ConfigHelper): 'impala-impalad.json', 'IMPALAD', 'node') self.impala_statestore_confs = self._load_and_init_configs( 'impala-statestore.json', 'STATESTORE', 'node') + self.kms_service_confs = self._load_and_init_configs( 'kms-service.json', 'KMS', 'cluster') self.kms_kms_confs = self._load_and_init_configs( diff --git a/sahara/plugins/cdh/v5_5_0/versionhandler.py b/sahara/plugins/cdh/v5_5_0/versionhandler.py index 9e8c0f42..607145eb 100644 --- a/sahara/plugins/cdh/v5_5_0/versionhandler.py +++ b/sahara/plugins/cdh/v5_5_0/versionhandler.py @@ -54,8 +54,28 @@ class VersionHandler(avm.BaseVersionHandler): "SQOOP": ['SQOOP_SERVER'], "SENTRY": ['SENTRY_SERVER'], "KMS": ['KMS'], + "YARN_GATEWAY": [], - "HDFS_GATEWAY": [] + "RESOURCEMANAGER": [], + "NODEMANAGER": [], + "JOBHISTORY": [], + + "HDFS_GATEWAY": [], + 'DATANODE': [], + 'NAMENODE': [], + 'SECONDARYNAMENODE': [], + 'JOURNALNODE': [], + + 'REGIONSERVER': [], + 'MASTER': [], + + 'HIVEMETASTORE': [], + 'HIVESERVER': [], + 'WEBCAT': [], + + 'CATALOGSERVER': [], + 'STATESTORE': [], + 'IMPALAD': [], } def get_edp_engine(self, cluster, job_type):