From 68999ce9cb54f02b6e594cc9a57b258a4760992c Mon Sep 17 00:00:00 2001 From: Ken Chen Date: Mon, 27 Apr 2015 14:40:02 +0800 Subject: [PATCH] Add CDH5.4 support in sahara We add a directory v_5_4_0 in cdh plugin, so that we have a choice 5.4.0 when cdh plugin is chosen. Implements: blueprint cdh-5-4-support Change-Id: I05d2082b589eb031b0ce1b5a253dc62a75912e1b --- MANIFEST.in | 4 + etc/scenario/sahara-ci/cdh-5.4.0.yaml | 68 ++ sahara/plugins/cdh/v5_4_0/__init__.py | 0 sahara/plugins/cdh/v5_4_0/cloudera_utils.py | 405 ++++++++++ sahara/plugins/cdh/v5_4_0/config_helper.py | 334 ++++++++ sahara/plugins/cdh/v5_4_0/db_helper.py | 41 + sahara/plugins/cdh/v5_4_0/deploy.py | 205 +++++ sahara/plugins/cdh/v5_4_0/edp_engine.py | 71 ++ sahara/plugins/cdh/v5_4_0/plugin_utils.py | 133 ++++ .../cdh/v5_4_0/resources/cdh_config.py | 135 ++++ .../cdh/v5_4_0/resources/cdh_config.sh | 3 + .../cdh/v5_4_0/resources/create_hive_db.sql | 4 + .../cdh/v5_4_0/resources/create_sentry_db.sql | 4 + .../cdh/v5_4_0/resources/flume-agent.json | 284 +++++++ .../cdh/v5_4_0/resources/flume-service.json | 104 +++ .../cdh/v5_4_0/resources/hbase-gateway.json | 56 ++ .../resources/hbase-hbaserestserver.json | 278 +++++++ .../resources/hbase-hbasethriftserver.json | 284 +++++++ .../cdh/v5_4_0/resources/hbase-master.json | 350 ++++++++ .../v5_4_0/resources/hbase-regionserver.json | 584 ++++++++++++++ .../cdh/v5_4_0/resources/hbase-service.json | 524 ++++++++++++ .../cdh/v5_4_0/resources/hdfs-balancer.json | 44 ++ .../cdh/v5_4_0/resources/hdfs-datanode.json | 440 +++++++++++ .../resources/hdfs-failovercontroller.json | 230 ++++++ .../cdh/v5_4_0/resources/hdfs-gateway.json | 68 ++ .../cdh/v5_4_0/resources/hdfs-httpfs.json | 290 +++++++ .../v5_4_0/resources/hdfs-journalnode.json | 308 ++++++++ .../cdh/v5_4_0/resources/hdfs-namenode.json | 548 +++++++++++++ .../cdh/v5_4_0/resources/hdfs-nfsgateway.json | 254 ++++++ .../resources/hdfs-secondarynamenode.json | 308 ++++++++ .../cdh/v5_4_0/resources/hdfs-service.json | 746 ++++++++++++++++++ .../cdh/v5_4_0/resources/hive-gateway.json | 62 ++ .../v5_4_0/resources/hive-hivemetastore.json | 248 ++++++ .../v5_4_0/resources/hive-hiveserver2.json | 500 ++++++++++++ .../cdh/v5_4_0/resources/hive-service.json | 380 +++++++++ .../cdh/v5_4_0/resources/hive-site.xml | 61 ++ .../cdh/v5_4_0/resources/hive-webhcat.json | 230 ++++++ .../cdh/v5_4_0/resources/hue-hue_server.json | 188 +++++ .../cdh/v5_4_0/resources/hue-kt_renewer.json | 134 ++++ .../cdh/v5_4_0/resources/hue-service.json | 350 ++++++++ .../resources/impala-catalogserver.json | 266 +++++++ .../cdh/v5_4_0/resources/impala-impalad.json | 440 +++++++++++ .../cdh/v5_4_0/resources/impala-llama.json | 368 +++++++++ .../cdh/v5_4_0/resources/impala-service.json | 356 +++++++++ .../v5_4_0/resources/impala-statestore.json | 236 ++++++ .../resources/ks_indexer-hbase_indexer.json | 242 ++++++ .../v5_4_0/resources/ks_indexer-service.json | 110 +++ .../cdh/v5_4_0/resources/mapred-site.xml | 157 ++++ .../v5_4_0/resources/oozie-oozie_server.json | 446 +++++++++++ .../cdh/v5_4_0/resources/oozie-service.json | 140 ++++ .../v5_4_0/resources/priority-one-confs.json | 34 + .../resources/sentry-sentry_server.json | 218 +++++ .../cdh/v5_4_0/resources/sentry-service.json | 158 ++++ .../cdh/v5_4_0/resources/solr-gateway.json | 32 + .../cdh/v5_4_0/resources/solr-service.json | 188 +++++ .../v5_4_0/resources/solr-solr_server.json | 302 +++++++ .../cdh/v5_4_0/resources/spark-gateway.json | 62 ++ .../cdh/v5_4_0/resources/spark-service.json | 80 ++ .../spark-spark_yarn_history_server.json | 146 ++++ .../cdh/v5_4_0/resources/sqoop-service.json | 62 ++ .../v5_4_0/resources/sqoop-sqoop_server.json | 272 +++++++ .../cdh/v5_4_0/resources/yarn-gateway.json | 344 ++++++++ .../cdh/v5_4_0/resources/yarn-jobhistory.json | 320 ++++++++ .../v5_4_0/resources/yarn-nodemanager.json | 476 +++++++++++ .../resources/yarn-resourcemanager.json | 506 ++++++++++++ .../cdh/v5_4_0/resources/yarn-service.json | 290 +++++++ .../v5_4_0/resources/zookeeper-server.json | 410 ++++++++++ .../v5_4_0/resources/zookeeper-service.json | 158 ++++ sahara/plugins/cdh/v5_4_0/validation.py | 268 +++++++ sahara/plugins/cdh/v5_4_0/versionhandler.py | 120 +++ .../cdh/version_5_4_0/cluster.json | 24 + .../cdh/version_5_4_0/datanode.json | 11 + .../cdh/version_5_4_0/namenode.json | 14 + .../cdh/version_5_4_0/secondary-namenode.json | 12 + sahara/tests/integration/configs/config.py | 2 +- .../plugins/cdh/test_cloudera_utils_540.py | 66 ++ .../plugins/cdh/test_config_helper_540.py | 37 + .../tests/unit/plugins/cdh/test_utils_540.py | 65 ++ .../tests/unit/plugins/cdh/v5_4_0/__init__.py | 0 .../plugins/cdh/v5_4_0/test_edp_engine.py | 94 +++ 80 files changed, 16791 insertions(+), 1 deletion(-) create mode 100644 etc/scenario/sahara-ci/cdh-5.4.0.yaml create mode 100644 sahara/plugins/cdh/v5_4_0/__init__.py create mode 100644 sahara/plugins/cdh/v5_4_0/cloudera_utils.py create mode 100644 sahara/plugins/cdh/v5_4_0/config_helper.py create mode 100644 sahara/plugins/cdh/v5_4_0/db_helper.py create mode 100644 sahara/plugins/cdh/v5_4_0/deploy.py create mode 100644 sahara/plugins/cdh/v5_4_0/edp_engine.py create mode 100644 sahara/plugins/cdh/v5_4_0/plugin_utils.py create mode 100644 sahara/plugins/cdh/v5_4_0/resources/cdh_config.py create mode 100755 sahara/plugins/cdh/v5_4_0/resources/cdh_config.sh create mode 100644 sahara/plugins/cdh/v5_4_0/resources/create_hive_db.sql create mode 100644 sahara/plugins/cdh/v5_4_0/resources/create_sentry_db.sql create mode 100644 sahara/plugins/cdh/v5_4_0/resources/flume-agent.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/flume-service.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hbase-gateway.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hbase-hbaserestserver.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hbase-hbasethriftserver.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hbase-master.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hbase-regionserver.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hbase-service.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hdfs-balancer.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hdfs-datanode.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hdfs-failovercontroller.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hdfs-gateway.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hdfs-httpfs.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hdfs-journalnode.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hdfs-namenode.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hdfs-nfsgateway.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hdfs-secondarynamenode.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hdfs-service.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hive-gateway.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hive-hivemetastore.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hive-hiveserver2.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hive-service.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hive-site.xml create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hive-webhcat.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hue-hue_server.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hue-kt_renewer.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/hue-service.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/impala-catalogserver.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/impala-impalad.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/impala-llama.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/impala-service.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/impala-statestore.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/ks_indexer-hbase_indexer.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/ks_indexer-service.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/mapred-site.xml create mode 100644 sahara/plugins/cdh/v5_4_0/resources/oozie-oozie_server.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/oozie-service.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/priority-one-confs.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/sentry-sentry_server.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/sentry-service.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/solr-gateway.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/solr-service.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/solr-solr_server.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/spark-gateway.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/spark-service.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/spark-spark_yarn_history_server.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/sqoop-service.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/sqoop-sqoop_server.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/yarn-gateway.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/yarn-jobhistory.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/yarn-nodemanager.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/yarn-resourcemanager.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/yarn-service.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/zookeeper-server.json create mode 100644 sahara/plugins/cdh/v5_4_0/resources/zookeeper-service.json create mode 100644 sahara/plugins/cdh/v5_4_0/validation.py create mode 100644 sahara/plugins/cdh/v5_4_0/versionhandler.py create mode 100644 sahara/plugins/default_templates/cdh/version_5_4_0/cluster.json create mode 100644 sahara/plugins/default_templates/cdh/version_5_4_0/datanode.json create mode 100644 sahara/plugins/default_templates/cdh/version_5_4_0/namenode.json create mode 100644 sahara/plugins/default_templates/cdh/version_5_4_0/secondary-namenode.json create mode 100644 sahara/tests/unit/plugins/cdh/test_cloudera_utils_540.py create mode 100644 sahara/tests/unit/plugins/cdh/test_config_helper_540.py create mode 100644 sahara/tests/unit/plugins/cdh/test_utils_540.py create mode 100644 sahara/tests/unit/plugins/cdh/v5_4_0/__init__.py create mode 100644 sahara/tests/unit/plugins/cdh/v5_4_0/test_edp_engine.py diff --git a/MANIFEST.in b/MANIFEST.in index 7c632492..bbcbf3b9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -23,6 +23,10 @@ include sahara/plugins/cdh/v5_3_0/resources/cdh_config.py include sahara/plugins/cdh/v5_3_0/resources/*.sh include sahara/plugins/cdh/v5_3_0/resources/*.json include sahara/plugins/cdh/v5_3_0/resources/*.sql +include sahara/plugins/cdh/v5_4_0/resources/cdh_config.py +include sahara/plugins/cdh/v5_4_0/resources/*.sh +include sahara/plugins/cdh/v5_4_0/resources/*.json +include sahara/plugins/cdh/v5_4_0/resources/*.sql include sahara/plugins/vanilla/hadoop2/resources/*.sh include sahara/plugins/vanilla/hadoop2/resources/*.sql include sahara/plugins/vanilla/hadoop2/resources/*.template diff --git a/etc/scenario/sahara-ci/cdh-5.4.0.yaml b/etc/scenario/sahara-ci/cdh-5.4.0.yaml new file mode 100644 index 00000000..75229fca --- /dev/null +++ b/etc/scenario/sahara-ci/cdh-5.4.0.yaml @@ -0,0 +1,68 @@ +clusters: + - plugin_name: cdh + plugin_version: 5.4.0 + image: %cdh_5_4_0_image% + node_group_templates: + - name: worker-dn + flavor_id: %ci_flavor_id% + node_processes: + - HDFS_DATANODE + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + node_configs: + &ng_configs + DATANODE: + dfs_datanode_du_reserved: 0 + - name: worker-nm + flavor_id: %ci_flavor_id% + node_processes: + - YARN_NODEMANAGER + auto_security_group: true + - name: worker-nm-dn + flavor_id: %ci_flavor_id% + node_processes: + - YARN_NODEMANAGER + - HDFS_DATANODE + volumes_per_node: 2 + volumes_size: 2 + auto_security_group: true + node_configs: + *ng_configs + - name: manager + flavor_id: %large_flavor_id% + node_processes: + - CLOUDERA_MANAGER + auto_security_group: true + - name: master-core + flavor_id: %medium_flavor_id% + node_processes: + - HDFS_NAMENODE + - YARN_RESOURCEMANAGER + auto_security_group: true + - name: master-additional + flavor_id: %medium_flavor_id% + node_processes: + - OOZIE_SERVER + - YARN_JOBHISTORY + - HDFS_SECONDARYNAMENODE + - HIVE_METASTORE + - HIVE_SERVER2 + auto_security_group: true + cluster_template: + name: cdh540 + node_group_templates: + manager: 1 + master-core: 1 + master-additional: 1 + worker-nm-dn: 1 + worker-nm: 1 + worker-dn: 1 + cluster_configs: + HDFS: + dfs_replication: 1 + cluster: + name: %cluster_name% + scenario: + - run_jobs + edp_jobs_flow: hadoop_2 diff --git a/sahara/plugins/cdh/v5_4_0/__init__.py b/sahara/plugins/cdh/v5_4_0/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sahara/plugins/cdh/v5_4_0/cloudera_utils.py b/sahara/plugins/cdh/v5_4_0/cloudera_utils.py new file mode 100644 index 00000000..2043acaf --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/cloudera_utils.py @@ -0,0 +1,405 @@ +# Copyright (c) 2015 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six + +from sahara.i18n import _ +from sahara.plugins.cdh import cloudera_utils as cu +from sahara.plugins.cdh.v5_4_0 import config_helper as c_helper +from sahara.plugins.cdh.v5_4_0 import plugin_utils as pu +from sahara.plugins.cdh.v5_4_0 import validation as v +from sahara.swift import swift_helper +from sahara.utils import cluster_progress_ops as cpo +from sahara.utils import xmlutils + + +HDFS_SERVICE_TYPE = 'HDFS' +YARN_SERVICE_TYPE = 'YARN' +OOZIE_SERVICE_TYPE = 'OOZIE' +HIVE_SERVICE_TYPE = 'HIVE' +HUE_SERVICE_TYPE = 'HUE' +SPARK_SERVICE_TYPE = 'SPARK_ON_YARN' +ZOOKEEPER_SERVICE_TYPE = 'ZOOKEEPER' +HBASE_SERVICE_TYPE = 'HBASE' +FLUME_SERVICE_TYPE = 'FLUME' +SENTRY_SERVICE_TYPE = 'SENTRY' +SOLR_SERVICE_TYPE = 'SOLR' +SQOOP_SERVICE_TYPE = 'SQOOP' +KS_INDEXER_SERVICE_TYPE = 'KS_INDEXER' +IMPALA_SERVICE_TYPE = 'IMPALA' + + +def _merge_dicts(a, b): + res = {} + + def update(cfg): + for service, configs in six.iteritems(cfg): + if not res.get(service): + res[service] = {} + + res[service].update(configs) + + update(a) + update(b) + return res + + +class ClouderaUtilsV540(cu.ClouderaUtils): + FLUME_SERVICE_NAME = 'flume01' + SOLR_SERVICE_NAME = 'solr01' + SQOOP_SERVICE_NAME = 'sqoop01' + KS_INDEXER_SERVICE_NAME = 'ks_indexer01' + IMPALA_SERVICE_NAME = 'impala01' + SENTRY_SERVICE_NAME = 'sentry01' + CM_API_VERSION = 8 + + def __init__(self): + cu.ClouderaUtils.__init__(self) + self.pu = pu.PluginUtilsV540() + + def get_service_by_role(self, process, cluster=None, instance=None): + cm_cluster = None + if cluster: + cm_cluster = self.get_cloudera_cluster(cluster) + elif instance: + cm_cluster = self.get_cloudera_cluster(instance.cluster) + else: + raise ValueError(_("'cluster' or 'instance' argument missed")) + + if process in ['AGENT']: + return cm_cluster.get_service(self.FLUME_SERVICE_NAME) + elif process in ['SENTRY_SERVER']: + return cm_cluster.get_service(self.SENTRY_SERVICE_NAME) + elif process in ['SQOOP_SERVER']: + return cm_cluster.get_service(self.SQOOP_SERVICE_NAME) + elif process in ['SOLR_SERVER']: + return cm_cluster.get_service(self.SOLR_SERVICE_NAME) + elif process in ['HBASE_INDEXER']: + return cm_cluster.get_service(self.KS_INDEXER_SERVICE_NAME) + elif process in ['CATALOGSERVER', 'STATESTORE', 'IMPALAD', 'LLAMA']: + return cm_cluster.get_service(self.IMPALA_SERVICE_NAME) + else: + return super(ClouderaUtilsV540, self).get_service_by_role( + process, cluster, instance) + + @cpo.event_wrapper( + True, step=_("First run cluster"), param=('cluster', 1)) + @cu.cloudera_cmd + def first_run(self, cluster): + cm_cluster = self.get_cloudera_cluster(cluster) + yield cm_cluster.first_run() + + @cpo.event_wrapper(True, step=_("Create services"), param=('cluster', 1)) + def create_services(self, cluster): + api = self.get_api_client(cluster) + + fullversion = ('5.0.0' if cluster.hadoop_version == '5' + else cluster.hadoop_version) + cm_cluster = api.create_cluster(cluster.name, + fullVersion=fullversion) + + if len(self.pu.get_zookeepers(cluster)) > 0: + cm_cluster.create_service(self.ZOOKEEPER_SERVICE_NAME, + ZOOKEEPER_SERVICE_TYPE) + cm_cluster.create_service(self.HDFS_SERVICE_NAME, HDFS_SERVICE_TYPE) + cm_cluster.create_service(self.YARN_SERVICE_NAME, YARN_SERVICE_TYPE) + cm_cluster.create_service(self.OOZIE_SERVICE_NAME, OOZIE_SERVICE_TYPE) + if self.pu.get_hive_metastore(cluster): + cm_cluster.create_service(self.HIVE_SERVICE_NAME, + HIVE_SERVICE_TYPE) + if self.pu.get_hue(cluster): + cm_cluster.create_service(self.HUE_SERVICE_NAME, HUE_SERVICE_TYPE) + if self.pu.get_spark_historyserver(cluster): + cm_cluster.create_service(self.SPARK_SERVICE_NAME, + SPARK_SERVICE_TYPE) + if self.pu.get_hbase_master(cluster): + cm_cluster.create_service(self.HBASE_SERVICE_NAME, + HBASE_SERVICE_TYPE) + if len(self.pu.get_flumes(cluster)) > 0: + cm_cluster.create_service(self.FLUME_SERVICE_NAME, + FLUME_SERVICE_TYPE) + if self.pu.get_sentry(cluster): + cm_cluster.create_service(self.SENTRY_SERVICE_NAME, + SENTRY_SERVICE_TYPE) + if len(self.pu.get_solrs(cluster)) > 0: + cm_cluster.create_service(self.SOLR_SERVICE_NAME, + SOLR_SERVICE_TYPE) + if self.pu.get_sqoop(cluster): + cm_cluster.create_service(self.SQOOP_SERVICE_NAME, + SQOOP_SERVICE_TYPE) + if len(self.pu.get_hbase_indexers(cluster)) > 0: + cm_cluster.create_service(self.KS_INDEXER_SERVICE_NAME, + KS_INDEXER_SERVICE_TYPE) + if self.pu.get_catalogserver(cluster): + cm_cluster.create_service(self.IMPALA_SERVICE_NAME, + IMPALA_SERVICE_TYPE) + + def await_agents(self, cluster, instances): + self._await_agents(cluster, instances, c_helper.AWAIT_AGENTS_TIMEOUT) + + @cpo.event_wrapper( + True, step=_("Configure services"), param=('cluster', 1)) + def configure_services(self, cluster): + cm_cluster = self.get_cloudera_cluster(cluster) + + if len(self.pu.get_zookeepers(cluster)) > 0: + zookeeper = cm_cluster.get_service(self.ZOOKEEPER_SERVICE_NAME) + zookeeper.update_config(self._get_configs(ZOOKEEPER_SERVICE_TYPE, + cluster=cluster)) + + hdfs = cm_cluster.get_service(self.HDFS_SERVICE_NAME) + hdfs.update_config(self._get_configs(HDFS_SERVICE_TYPE, + cluster=cluster)) + + yarn = cm_cluster.get_service(self.YARN_SERVICE_NAME) + yarn.update_config(self._get_configs(YARN_SERVICE_TYPE, + cluster=cluster)) + + oozie = cm_cluster.get_service(self.OOZIE_SERVICE_NAME) + oozie.update_config(self._get_configs(OOZIE_SERVICE_TYPE, + cluster=cluster)) + + if self.pu.get_hive_metastore(cluster): + hive = cm_cluster.get_service(self.HIVE_SERVICE_NAME) + hive.update_config(self._get_configs(HIVE_SERVICE_TYPE, + cluster=cluster)) + + if self.pu.get_hue(cluster): + hue = cm_cluster.get_service(self.HUE_SERVICE_NAME) + hue.update_config(self._get_configs(HUE_SERVICE_TYPE, + cluster=cluster)) + + if self.pu.get_spark_historyserver(cluster): + spark = cm_cluster.get_service(self.SPARK_SERVICE_NAME) + spark.update_config(self._get_configs(SPARK_SERVICE_TYPE, + cluster=cluster)) + + if self.pu.get_hbase_master(cluster): + hbase = cm_cluster.get_service(self.HBASE_SERVICE_NAME) + hbase.update_config(self._get_configs(HBASE_SERVICE_TYPE, + cluster=cluster)) + + if len(self.pu.get_flumes(cluster)) > 0: + flume = cm_cluster.get_service(self.FLUME_SERVICE_NAME) + flume.update_config(self._get_configs(FLUME_SERVICE_TYPE, + cluster=cluster)) + + if self.pu.get_sentry(cluster): + sentry = cm_cluster.get_service(self.SENTRY_SERVICE_NAME) + sentry.update_config(self._get_configs(SENTRY_SERVICE_TYPE, + cluster=cluster)) + + if len(self.pu.get_solrs(cluster)) > 0: + solr = cm_cluster.get_service(self.SOLR_SERVICE_NAME) + solr.update_config(self._get_configs(SOLR_SERVICE_TYPE, + cluster=cluster)) + + if self.pu.get_sqoop(cluster): + sqoop = cm_cluster.get_service(self.SQOOP_SERVICE_NAME) + sqoop.update_config(self._get_configs(SQOOP_SERVICE_TYPE, + cluster=cluster)) + + if len(self.pu.get_hbase_indexers(cluster)) > 0: + ks_indexer = cm_cluster.get_service(self.KS_INDEXER_SERVICE_NAME) + ks_indexer.update_config( + self._get_configs(KS_INDEXER_SERVICE_TYPE, cluster=cluster)) + + if self.pu.get_catalogserver(cluster): + impala = cm_cluster.get_service(self.IMPALA_SERVICE_NAME) + impala.update_config(self._get_configs(IMPALA_SERVICE_TYPE, + cluster=cluster)) + + def _get_configs(self, service, cluster=None, node_group=None): + def get_hadoop_dirs(mount_points, suffix): + return ','.join([x + suffix for x in mount_points]) + + all_confs = {} + if cluster: + zk_count = v._get_inst_count(cluster, 'ZOOKEEPER_SERVER') + hbm_count = v._get_inst_count(cluster, 'HBASE_MASTER') + snt_count = v._get_inst_count(cluster, 'SENTRY_SERVER') + ks_count = v._get_inst_count(cluster, 'KEY_VALUE_STORE_INDEXER') + imp_count = v._get_inst_count(cluster, 'IMPALA_CATALOGSERVER') + hive_count = v._get_inst_count(cluster, 'HIVE_METASTORE') + slr_count = v._get_inst_count(cluster, 'SOLR_SERVER') + sqp_count = v._get_inst_count(cluster, 'SQOOP_SERVER') + core_site_safety_valve = '' + if self.pu.c_helper.is_swift_enabled(cluster): + configs = swift_helper.get_swift_configs() + confs = {c['name']: c['value'] for c in configs} + core_site_safety_valve = xmlutils.create_elements_xml(confs) + all_confs = { + 'HDFS': { + 'zookeeper_service': + self.ZOOKEEPER_SERVICE_NAME if zk_count else '', + 'dfs_block_local_path_access_user': + 'impala' if imp_count else '', + 'core_site_safety_valve': core_site_safety_valve + }, + 'HIVE': { + 'mapreduce_yarn_service': self.YARN_SERVICE_NAME, + 'sentry_service': + self.SENTRY_SERVICE_NAME if snt_count else '', + 'zookeeper_service': + self.ZOOKEEPER_SERVICE_NAME if zk_count else '' + }, + 'OOZIE': { + 'mapreduce_yarn_service': self.YARN_SERVICE_NAME, + 'hive_service': + self.HIVE_SERVICE_NAME if hive_count else '', + 'zookeeper_service': + self.ZOOKEEPER_SERVICE_NAME if zk_count else '' + }, + 'YARN': { + 'hdfs_service': self.HDFS_SERVICE_NAME, + 'zookeeper_service': + self.ZOOKEEPER_SERVICE_NAME if zk_count else '' + }, + 'HUE': { + 'hive_service': self.HIVE_SERVICE_NAME, + 'oozie_service': self.OOZIE_SERVICE_NAME, + 'sentry_service': + self.SENTRY_SERVICE_NAME if snt_count else '', + 'solr_service': + self.SOLR_SERVICE_NAME if slr_count else '', + 'zookeeper_service': + self.ZOOKEEPER_SERVICE_NAME if zk_count else '', + 'hbase_service': + self.HBASE_SERVICE_NAME if hbm_count else '', + 'impala_service': + self.IMPALA_SERVICE_NAME if imp_count else '', + 'sqoop_service': + self.SQOOP_SERVICE_NAME if sqp_count else '' + }, + 'SPARK_ON_YARN': { + 'yarn_service': self.YARN_SERVICE_NAME + }, + 'HBASE': { + 'hdfs_service': self.HDFS_SERVICE_NAME, + 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME, + 'hbase_enable_indexing': 'true' if ks_count else 'false', + 'hbase_enable_replication': + 'true' if ks_count else 'false' + }, + 'FLUME': { + 'hdfs_service': self.HDFS_SERVICE_NAME, + 'solr_service': + self.SOLR_SERVICE_NAME if slr_count else '', + 'hbase_service': + self.HBASE_SERVICE_NAME if hbm_count else '' + }, + 'SENTRY': { + 'hdfs_service': self.HDFS_SERVICE_NAME, + 'sentry_server_config_safety_valve': ( + c_helper.SENTRY_IMPALA_CLIENT_SAFETY_VALVE + if imp_count else '') + }, + 'SOLR': { + 'hdfs_service': self.HDFS_SERVICE_NAME, + 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME + }, + 'SQOOP': { + 'mapreduce_yarn_service': self.YARN_SERVICE_NAME + }, + 'KS_INDEXER': { + 'hbase_service': self.HBASE_SERVICE_NAME, + 'solr_service': self.SOLR_SERVICE_NAME + }, + 'IMPALA': { + 'hdfs_service': self.HDFS_SERVICE_NAME, + 'hbase_service': + self.HBASE_SERVICE_NAME if hbm_count else '', + 'hive_service': self.HIVE_SERVICE_NAME, + 'sentry_service': + self.SENTRY_SERVICE_NAME if snt_count else '', + 'zookeeper_service': + self.ZOOKEEPER_SERVICE_NAME if zk_count else '' + } + } + hive_confs = { + 'HIVE': { + 'hive_metastore_database_type': 'postgresql', + 'hive_metastore_database_host': + self.pu.get_manager(cluster).internal_ip, + 'hive_metastore_database_port': '7432', + 'hive_metastore_database_password': + self.pu.db_helper.get_hive_db_password(cluster) + } + } + hue_confs = { + 'HUE': { + 'hue_webhdfs': self.pu.get_role_name( + self.pu.get_namenode(cluster), 'NAMENODE') + } + } + sentry_confs = { + 'SENTRY': { + 'sentry_server_database_type': 'postgresql', + 'sentry_server_database_host': + self.pu.get_manager(cluster).internal_ip, + 'sentry_server_database_port': '7432', + 'sentry_server_database_password': + self.pu.db_helper.get_sentry_db_password(cluster) + } + } + + all_confs = _merge_dicts(all_confs, hue_confs) + all_confs = _merge_dicts(all_confs, hive_confs) + all_confs = _merge_dicts(all_confs, sentry_confs) + all_confs = _merge_dicts(all_confs, cluster.cluster_configs) + + if node_group: + paths = node_group.storage_paths() + + ng_default_confs = { + 'NAMENODE': { + 'dfs_name_dir_list': get_hadoop_dirs(paths, '/fs/nn') + }, + 'SECONDARYNAMENODE': { + 'fs_checkpoint_dir_list': + get_hadoop_dirs(paths, '/fs/snn') + }, + 'DATANODE': { + 'dfs_data_dir_list': get_hadoop_dirs(paths, '/fs/dn'), + 'dfs_datanode_data_dir_perm': 755, + 'dfs_datanode_handler_count': 30 + }, + 'NODEMANAGER': { + 'yarn_nodemanager_local_dirs': + get_hadoop_dirs(paths, '/yarn/local') + }, + 'SERVER': { + 'maxSessionTimeout': 60000 + }, + 'HIVESERVER2': { + 'hiveserver2_enable_impersonation': + 'false' if snt_count else 'true', + 'hive_hs2_config_safety_valve': ( + c_helper.HIVE_SERVER2_SENTRY_SAFETY_VALVE + if snt_count else '') + }, + 'HIVEMETASTORE': { + 'hive_metastore_config_safety_valve': ( + c_helper.HIVE_METASTORE_SENTRY_SAFETY_VALVE + if snt_count else '') + } + } + + ng_user_confs = self.pu.convert_process_configs( + node_group.node_configs) + all_confs = _merge_dicts(all_confs, ng_user_confs) + all_confs = _merge_dicts(all_confs, ng_default_confs) + + return all_confs.get(service, {}) diff --git a/sahara/plugins/cdh/v5_4_0/config_helper.py b/sahara/plugins/cdh/v5_4_0/config_helper.py new file mode 100644 index 00000000..07846d41 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/config_helper.py @@ -0,0 +1,334 @@ +# Copyright (c) 2015 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_serialization import jsonutils as json + +from sahara.plugins import provisioning as p +from sahara.utils import files as f + +CDH5_UBUNTU_REPO = ('deb [arch=amd64] http://archive.cloudera.com/cdh5' + '/ubuntu/precise/amd64/cdh precise-cdh5.4.0 contrib' + '\ndeb-src http://archive.cloudera.com/cdh5/ubuntu' + '/precise/amd64/cdh precise-cdh5.4.0 contrib') + +DEFAULT_CDH5_UBUNTU_REPO_KEY_URL = ('http://archive.cloudera.com/cdh5/ubuntu' + '/precise/amd64/cdh/archive.key') + +CM5_UBUNTU_REPO = ('deb [arch=amd64] http://archive.cloudera.com/cm5' + '/ubuntu/precise/amd64/cm precise-cm5.4.0 contrib' + '\ndeb-src http://archive.cloudera.com/cm5/ubuntu' + '/precise/amd64/cm precise-cm5.4.0 contrib') + +DEFAULT_CM5_UBUNTU_REPO_KEY_URL = ('http://archive.cloudera.com/cm5/ubuntu' + '/precise/amd64/cm/archive.key') + +CDH5_CENTOS_REPO = ('[cloudera-cdh5]' + '\nname=Cloudera\'s Distribution for Hadoop, Version 5' + '\nbaseurl=http://archive.cloudera.com/cdh5/redhat/6' + '/x86_64/cdh/5.4.0/' + '\ngpgkey = http://archive.cloudera.com/cdh5/redhat/6' + '/x86_64/cdh/RPM-GPG-KEY-cloudera' + '\ngpgcheck = 1') + +CM5_CENTOS_REPO = ('[cloudera-manager]' + '\nname=Cloudera Manager' + '\nbaseurl=http://archive.cloudera.com/cm5/redhat/6' + '/x86_64/cm/5.4.0/' + '\ngpgkey = http://archive.cloudera.com/cm5/redhat/6' + '/x86_64/cm/RPM-GPG-KEY-cloudera' + '\ngpgcheck = 1') + +DEFAULT_SWIFT_LIB_URL = ('https://repository.cloudera.com/artifactory/repo/org' + '/apache/hadoop/hadoop-openstack/2.6.0-cdh5.4.0' + '/hadoop-openstack-2.6.0-cdh5.4.0.jar') +DEFAULT_EXTJS_LIB_URL = 'http://dev.sencha.com/deploy/ext-2.2.zip' + +HIVE_SERVER2_SENTRY_SAFETY_VALVE = ( + '' + '\n hive.security.authorization.task.factory' + '\n org.apache.sentry.binding.hive.SentryHiveAuthorizationTask' + 'FactoryImpl' + '\n' + '\n' + '\n hive.server2.session.hook' + '\n org.apache.sentry.binding.hive.HiveAuthzBindingSessionHook' + '' + '\n' + '\n' + '\n hive.sentry.conf.url' + '\n file:///{{CMF_CONF_DIR}}/sentry-site.xml' + '\n') + +HIVE_METASTORE_SENTRY_SAFETY_VALVE = ( + '' + '\n hive.metastore.client.impl' + '\n org.apache.sentry.binding.metastore.SentryHiveMetaStore' + 'Client' + '\n Sets custom Hive metastore client which Sentry uses' + ' to filter out metadata.' + '\n' + '\n' + '\n hive.metastore.pre.event.listeners' + '\n org.apache.sentry.binding.metastore.MetastoreAuthzBinding' + '' + '\n list of comma separated listeners for metastore' + ' events.' + '\n' + '\n' + '\n hive.metastore.event.listeners' + '\n org.apache.sentry.binding.metastore.SentryMetastorePost' + 'EventListener' + '\n list of comma separated listeners for metastore,' + ' post events.' + '\n') + +SENTRY_IMPALA_CLIENT_SAFETY_VALVE = ( + '' + '\n sentry.service.client.server.rpc-port' + '\n 3893' + '\n' + '\n' + '\n sentry.service.client.server.rpc-address' + '\n hostname' + '\n' + '\n' + '\n sentry.service.client.server.rpc-connection-timeout' + '\n 200000' + '\n' + '\n' + '\n sentry.service.security.mode' + '\n none' + '\n') + +CDH5_REPO_URL = p.Config( + 'CDH5 repo list URL', 'general', 'cluster', priority=1, + default_value="") + +CDH5_REPO_KEY_URL = p.Config( + 'CDH5 repo key URL (for debian-based only)', 'general', 'cluster', + priority=1, default_value="") + +CM5_REPO_URL = p.Config( + 'CM5 repo list URL', 'general', 'cluster', priority=1, + default_value="") + +CM5_REPO_KEY_URL = p.Config( + 'CM5 repo key URL (for debian-based only)', 'general', 'cluster', + priority=1, default_value="") + +ENABLE_SWIFT = p.Config('Enable Swift', 'general', 'cluster', + config_type='bool', priority=1, + default_value=True) + +ENABLE_HBASE_COMMON_LIB = p.Config('Enable HBase Common Lib', + 'general', 'cluster', config_type='bool', + priority=1, default_value=True) + +SWIFT_LIB_URL = p.Config( + 'Hadoop OpenStack library URL', 'general', 'cluster', priority=1, + default_value=DEFAULT_SWIFT_LIB_URL, + description=("Library that adds Swift support to CDH. The file will be " + "downloaded from VM.")) + + +EXTJS_LIB_URL = p.Config( + "ExtJS library URL", 'general', 'cluster', priority=1, + default_value=DEFAULT_EXTJS_LIB_URL, + description=("Ext 2.2 library is required for Oozie Web Console. " + "The file will be downloaded from VM with oozie.")) + +AWAIT_AGENTS_TIMEOUT = p.Config( + 'Await Cloudera agents timeout', 'general', 'cluster', config_type='int', + priority=1, default_value=300, is_optional=True, + description='Timeout for Cloudera agents connecting to Cloudera' + ' Manager, in seconds') + +AWAIT_MANAGER_STARTING_TIMEOUT = p.Config( + 'Timeout for Cloudera Manager starting', 'general', 'cluster', + config_type='int', priority=1, default_value=300, is_optional=True, + description='Timeout for Cloudera Manager starting, in seconds') + + +def _get_cluster_plugin_configs(): + return [CDH5_REPO_URL, CDH5_REPO_KEY_URL, CM5_REPO_URL, CM5_REPO_KEY_URL, + ENABLE_SWIFT, ENABLE_HBASE_COMMON_LIB, SWIFT_LIB_URL, + EXTJS_LIB_URL, AWAIT_AGENTS_TIMEOUT, + AWAIT_MANAGER_STARTING_TIMEOUT] + + +# ng wide configs + +def _load_json(path_to_file): + data = f.get_file_text(path_to_file) + return json.loads(data) + + +path_to_config = 'plugins/cdh/v5_4_0/resources/' +hdfs_confs = _load_json(path_to_config + 'hdfs-service.json') +namenode_confs = _load_json(path_to_config + 'hdfs-namenode.json') +datanode_confs = _load_json(path_to_config + 'hdfs-datanode.json') +secnamenode_confs = _load_json(path_to_config + 'hdfs-secondarynamenode.json') +yarn_confs = _load_json(path_to_config + 'yarn-service.json') +resourcemanager_confs = _load_json( + path_to_config + 'yarn-resourcemanager.json') +mapred_confs = _load_json(path_to_config + 'yarn-gateway.json') +nodemanager_confs = _load_json(path_to_config + 'yarn-nodemanager.json') +jobhistory_confs = _load_json(path_to_config + 'yarn-jobhistory.json') +oozie_service_confs = _load_json(path_to_config + 'oozie-service.json') +oozie_role_confs = _load_json(path_to_config + 'oozie-oozie_server.json') +hive_service_confs = _load_json(path_to_config + 'hive-service.json') +hive_metastore_confs = _load_json(path_to_config + 'hive-hivemetastore.json') +hive_hiveserver_confs = _load_json(path_to_config + 'hive-hiveserver2.json') +hive_webhcat_confs = _load_json(path_to_config + 'hive-webhcat.json') +hue_service_confs = _load_json(path_to_config + 'hue-service.json') +hue_role_confs = _load_json(path_to_config + 'hue-hue_server.json') +spark_service_confs = _load_json(path_to_config + 'spark-service.json') +spark_role_confs = _load_json( + path_to_config + 'spark-spark_yarn_history_server.json') +zookeeper_service_confs = _load_json(path_to_config + 'zookeeper-service.json') +zookeeper_server_confs = _load_json(path_to_config + 'zookeeper-server.json') +hbase_confs = _load_json(path_to_config + 'hbase-service.json') +master_confs = _load_json(path_to_config + 'hbase-master.json') +regionserver_confs = _load_json(path_to_config + 'hbase-regionserver.json') +flume_service_confs = _load_json(path_to_config + 'flume-service.json') +flume_agent_confs = _load_json(path_to_config + 'flume-agent.json') +sentry_service_confs = _load_json(path_to_config + 'sentry-service.json') +sentry_server_confs = _load_json(path_to_config + + 'sentry-sentry_server.json') +solr_service_confs = _load_json(path_to_config + 'solr-service.json') +solr_server_confs = _load_json(path_to_config + 'solr-solr_server.json') +sqoop_service_confs = _load_json(path_to_config + 'sqoop-service.json') +sqoop_server_confs = _load_json(path_to_config + + 'sqoop-sqoop_server.json') +ks_indexer_service_confs = _load_json(path_to_config + + 'ks_indexer-service.json') +ks_indexer_role_confs = _load_json(path_to_config + + 'ks_indexer-hbase_indexer.json') +impala_service_confs = _load_json(path_to_config + 'impala-service.json') +impala_catalogserver_confs = _load_json(path_to_config + + 'impala-catalogserver.json') +impala_impalad_confs = _load_json(path_to_config + + 'impala-impalad.json') +impala_llama_confs = _load_json(path_to_config + + 'impala-llama.json') +impala_statestore_confs = _load_json(path_to_config + + 'impala-statestore.json') + +priority_one_confs = _load_json(path_to_config + 'priority-one-confs.json') + + +def _prepare_value(value): + if not value: + return "" + + return value.replace('\n', ' ') + + +def _init_configs(confs, app_target, scope): + cfgs = [] + for cfg in confs: + priority = 1 if cfg['name'] in priority_one_confs else 2 + c = p.Config(cfg['name'], app_target, scope, priority=priority, + default_value=_prepare_value(cfg['value']), + description=cfg['desc'], is_optional=True) + cfgs.append(c) + + return cfgs + + +def _get_ng_plugin_configs(): + cfg = [] + cfg += _init_configs(hdfs_confs, 'HDFS', 'cluster') + cfg += _init_configs(namenode_confs, 'NAMENODE', 'node') + cfg += _init_configs(datanode_confs, 'DATANODE', 'node') + cfg += _init_configs(secnamenode_confs, 'SECONDARYNAMENODE', 'node') + cfg += _init_configs(yarn_confs, 'YARN', 'cluster') + cfg += _init_configs(resourcemanager_confs, 'RESOURCEMANAGER', 'node') + cfg += _init_configs(mapred_confs, 'MAPREDUCE', 'node') + cfg += _init_configs(nodemanager_confs, 'NODEMANAGER', 'node') + cfg += _init_configs(jobhistory_confs, 'JOBHISTORY', 'node') + cfg += _init_configs(oozie_service_confs, 'OOZIE', 'cluster') + cfg += _init_configs(oozie_role_confs, 'OOZIE', 'node') + cfg += _init_configs(hive_service_confs, 'HIVE', 'cluster') + cfg += _init_configs(hive_metastore_confs, 'HIVEMETASTORE', 'node') + cfg += _init_configs(hive_hiveserver_confs, 'HIVESERVER', 'node') + cfg += _init_configs(hive_webhcat_confs, 'WEBHCAT', 'node') + cfg += _init_configs(hue_service_confs, 'HUE', 'cluster') + cfg += _init_configs(hue_role_confs, 'HUE', 'node') + cfg += _init_configs(spark_service_confs, 'SPARK_ON_YARN', 'cluster') + cfg += _init_configs(spark_role_confs, 'SPARK_ON_YARN', 'node') + cfg += _init_configs(zookeeper_service_confs, 'ZOOKEEPER', 'cluster') + cfg += _init_configs(zookeeper_server_confs, 'ZOOKEEPER', 'node') + cfg += _init_configs(hbase_confs, 'HBASE', 'cluster') + cfg += _init_configs(master_confs, 'MASTER', 'node') + cfg += _init_configs(regionserver_confs, 'REGIONSERVER', 'node') + cfg += _init_configs(flume_service_confs, 'FLUME', 'cluster') + cfg += _init_configs(flume_agent_confs, 'FLUME', 'node') + cfg += _init_configs(sentry_service_confs, 'SENTRY', 'cluster') + cfg += _init_configs(sentry_server_confs, 'SENTRY', 'node') + cfg += _init_configs(solr_service_confs, 'SOLR', 'cluster') + cfg += _init_configs(solr_server_confs, 'SOLR', 'node') + cfg += _init_configs(sqoop_service_confs, 'SQOOP', 'cluster') + cfg += _init_configs(sqoop_server_confs, 'SQOOP', 'node') + cfg += _init_configs(ks_indexer_service_confs, 'KS_INDEXER', 'cluster') + cfg += _init_configs(ks_indexer_role_confs, 'KS_INDEXER', 'node') + cfg += _init_configs(impala_service_confs, 'IMPALA', 'cluster') + cfg += _init_configs(impala_catalogserver_confs, 'CATALOGSERVER', 'node') + cfg += _init_configs(impala_impalad_confs, 'IMPALAD', 'node') + cfg += _init_configs(impala_statestore_confs, 'STATESTORE', 'node') + return cfg + + +def get_plugin_configs(): + cluster_wide = _get_cluster_plugin_configs() + ng_wide = _get_ng_plugin_configs() + return cluster_wide + ng_wide + + +def _get_config_value(cluster, key): + return cluster.cluster_configs.get( + 'general', {}).get(key.name, key.default_value) + + +def get_cdh5_repo_url(cluster): + return _get_config_value(cluster, CDH5_REPO_URL) + + +def get_cdh5_key_url(cluster): + return _get_config_value(cluster, CDH5_REPO_KEY_URL) + + +def get_cm5_repo_url(cluster): + return _get_config_value(cluster, CM5_REPO_URL) + + +def get_cm5_key_url(cluster): + return _get_config_value(cluster, CM5_REPO_KEY_URL) + + +def is_swift_enabled(cluster): + return _get_config_value(cluster, ENABLE_SWIFT) + + +def is_hbase_common_lib_enabled(cluster): + return _get_config_value(cluster, ENABLE_HBASE_COMMON_LIB) + + +def get_swift_lib_url(cluster): + return _get_config_value(cluster, SWIFT_LIB_URL) + + +def get_extjs_lib_url(cluster): + return _get_config_value(cluster, EXTJS_LIB_URL) diff --git a/sahara/plugins/cdh/v5_4_0/db_helper.py b/sahara/plugins/cdh/v5_4_0/db_helper.py new file mode 100644 index 00000000..e4306096 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/db_helper.py @@ -0,0 +1,41 @@ +# Copyright (c) 2015 Intel Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara.plugins.cdh import db_helper as dh +from sahara.utils import files + + +def get_hive_db_password(cluster): + return dh.get_password_from_db(cluster, 'hive_db_password') + + +def get_sentry_db_password(cluster): + return dh.get_password_from_db(cluster, 'sentry_db_password') + + +def create_hive_database(cluster, remote): + db_password = get_hive_db_password(cluster) + create_db_script = files.get_file_text( + 'plugins/cdh/v5_4_0/resources/create_hive_db.sql') + create_db_script = create_db_script % db_password + dh.remote_execute_db_script(remote, create_db_script) + + +def create_sentry_database(cluster, remote): + db_password = get_sentry_db_password(cluster) + create_db_script = files.get_file_text( + 'plugins/cdh/v5_4_0/resources/create_sentry_db.sql') + create_db_script = create_db_script % db_password + dh.remote_execute_db_script(remote, create_db_script) diff --git a/sahara/plugins/cdh/v5_4_0/deploy.py b/sahara/plugins/cdh/v5_4_0/deploy.py new file mode 100644 index 00000000..d54f6f97 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/deploy.py @@ -0,0 +1,205 @@ +# Copyright (c) 2015 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara.i18n import _ +from sahara.plugins.cdh import commands as cmd +from sahara.plugins.cdh.v5_4_0 import cloudera_utils as cu +from sahara.plugins import utils as gu +from sahara.service.edp import hdfs_helper as h +from sahara.utils import cluster_progress_ops as cpo + +PACKAGES = [ + 'cloudera-manager-agent', + 'cloudera-manager-daemons', + 'cloudera-manager-server', + 'cloudera-manager-server-db-2', + 'flume-ng', + 'hadoop-hdfs-datanode', + 'hadoop-hdfs-namenode', + 'hadoop-hdfs-secondarynamenode', + 'hadoop-mapreduce', + 'hadoop-mapreduce-historyserver', + 'hadoop-yarn-nodemanager', + 'hadoop-yarn-resourcemanager', + 'hbase', + 'hbase-solr', + 'hive-hcatalog', + 'hive-metastore', + 'hive-server2', + 'hive-webhcat-server', + 'hue', + 'impala', + 'impala-server', + 'impala-state-store', + 'impala-catalog', + 'impala-shell', + 'ntp', + 'oozie', + 'oracle-j2sdk1.7', + 'sentry', + 'solr-server', + 'solr-doc', + 'search', + 'spark-history-server', + 'sqoop2', + 'unzip', + 'zookeeper' +] + +CU = cu.ClouderaUtilsV540() + + +def configure_cluster(cluster): + instances = gu.get_instances(cluster) + + if not cmd.is_pre_installed_cdh(CU.pu.get_manager(cluster).remote()): + CU.pu.configure_os(instances) + CU.pu.install_packages(instances, PACKAGES) + + CU.pu.start_cloudera_agents(instances) + CU.pu.start_cloudera_manager(cluster) + CU.update_cloudera_password(cluster) + CU.await_agents(cluster, instances) + CU.create_mgmt_service(cluster) + CU.create_services(cluster) + CU.configure_services(cluster) + CU.configure_instances(instances, cluster) + CU.deploy_configs(cluster) + + +@cpo.event_wrapper( + True, step=_("Start roles: NODEMANAGER, DATANODE"), param=('cluster', 0)) +def _start_roles(cluster, instances): + for instance in instances: + if 'HDFS_DATANODE' in instance.node_group.node_processes: + hdfs = CU.get_service_by_role('DATANODE', instance=instance) + CU.start_roles(hdfs, CU.pu.get_role_name(instance, 'DATANODE')) + + if 'YARN_NODEMANAGER' in instance.node_group.node_processes: + yarn = CU.get_service_by_role('NODEMANAGER', instance=instance) + CU.start_roles(yarn, CU.pu.get_role_name(instance, 'NODEMANAGER')) + + +def scale_cluster(cluster, instances): + if not instances: + return + + if not cmd.is_pre_installed_cdh(instances[0].remote()): + CU.pu.configure_os(instances) + CU.pu.install_packages(instances, PACKAGES) + + CU.pu.start_cloudera_agents(instances) + CU.await_agents(cluster, instances) + CU.configure_instances(instances, cluster) + CU.update_configs(instances) + CU.pu.configure_swift(cluster, instances) + CU.refresh_datanodes(cluster) + _start_roles(cluster, instances) + + +def decommission_cluster(cluster, instances): + dns = [] + nms = [] + for i in instances: + if 'HDFS_DATANODE' in i.node_group.node_processes: + dns.append(CU.pu.get_role_name(i, 'DATANODE')) + if 'YARN_NODEMANAGER' in i.node_group.node_processes: + nms.append(CU.pu.get_role_name(i, 'NODEMANAGER')) + + if dns: + CU.decommission_nodes(cluster, 'DATANODE', dns) + + if nms: + CU.decommission_nodes(cluster, 'NODEMANAGER', nms) + + CU.delete_instances(cluster, instances) + + CU.refresh_datanodes(cluster) + CU.refresh_yarn_nodes(cluster) + + +@cpo.event_wrapper(True, step=_("Prepare cluster"), param=('cluster', 0)) +def _prepare_cluster(cluster): + if CU.pu.get_oozie(cluster): + CU.pu.install_extjs(cluster) + + if CU.pu.get_hive_metastore(cluster): + CU.pu.configure_hive(cluster) + + if CU.pu.get_sentry(cluster): + CU.pu.configure_sentry(cluster) + + +@cpo.event_wrapper( + True, step=_("Finish cluster starting"), param=('cluster', 0)) +def _finish_cluster_starting(cluster): + if CU.pu.get_hive_metastore(cluster): + CU.pu.put_hive_hdfs_xml(cluster) + + server = CU.pu.get_hbase_master(cluster) + if CU.pu.c_helper.is_hbase_common_lib_enabled(cluster) and server: + with server.remote() as r: + h.create_hbase_common_lib(r) + + if CU.pu.get_flumes(cluster): + flume = CU.get_service_by_role('AGENT', cluster) + CU.start_service(flume) + + +def start_cluster(cluster): + _prepare_cluster(cluster) + + CU.first_run(cluster) + + CU.pu.configure_swift(cluster) + + _finish_cluster_starting(cluster) + + +def get_open_ports(node_group): + ports = [9000] # for CM agent + + ports_map = { + 'CLOUDERA_MANAGER': [7180, 7182, 7183, 7432, 7184, 8084, 8086, 10101, + 9997, 9996, 8087, 9998, 9999, 8085, 9995, 9994], + 'HDFS_NAMENODE': [8020, 8022, 50070, 50470], + 'HDFS_SECONDARYNAMENODE': [50090, 50495], + 'HDFS_DATANODE': [50010, 1004, 50075, 1006, 50020], + 'YARN_RESOURCEMANAGER': [8030, 8031, 8032, 8033, 8088], + 'YARN_NODEMANAGER': [8040, 8041, 8042], + 'YARN_JOBHISTORY': [10020, 19888], + 'HIVE_METASTORE': [9083], + 'HIVE_SERVER2': [10000], + 'HUE_SERVER': [8888], + 'OOZIE_SERVER': [11000, 11001], + 'SPARK_YARN_HISTORY_SERVER': [18088], + 'ZOOKEEPER_SERVER': [2181, 3181, 4181, 9010], + 'HBASE_MASTER': [60000], + 'HBASE_REGIONSERVER': [60020], + 'FLUME_AGENT': [41414], + 'SENTRY_SERVER': [8038], + 'SOLR_SERVER': [8983, 8984], + 'SQOOP_SERVER': [8005, 12000], + 'KEY_VALUE_STORE_INDEXER': [], + 'IMPALA_CATALOGSERVER': [25020, 26000], + 'IMPALA_STATESTORE': [25010, 24000], + 'IMPALAD': [21050, 21000, 23000, 25000, 28000, 22000] + } + + for process in node_group.node_processes: + if process in ports_map: + ports.extend(ports_map[process]) + + return ports diff --git a/sahara/plugins/cdh/v5_4_0/edp_engine.py b/sahara/plugins/cdh/v5_4_0/edp_engine.py new file mode 100644 index 00000000..20facb0d --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/edp_engine.py @@ -0,0 +1,71 @@ +# Copyright (c) 2015 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara.plugins.cdh import confighints_helper as ch_helper +from sahara.plugins.cdh.v5_4_0 import cloudera_utils as cu +from sahara.plugins import exceptions as ex +from sahara.plugins import utils as u +from sahara.service.edp import hdfs_helper +from sahara.service.edp.oozie import engine as edp_engine +from sahara.utils import edp + +CU = cu.ClouderaUtilsV540() + + +class EdpOozieEngine(edp_engine.OozieJobEngine): + + def get_hdfs_user(self): + return 'hdfs' + + def create_hdfs_dir(self, remote, dir_name): + hdfs_helper.create_dir_hadoop2(remote, dir_name, self.get_hdfs_user()) + + def get_oozie_server_uri(self, cluster): + oozie_ip = CU.pu.get_oozie(cluster).management_ip + return 'http://%s:11000/oozie' % oozie_ip + + def get_name_node_uri(self, cluster): + namenode_ip = CU.pu.get_namenode(cluster).fqdn() + return 'hdfs://%s:8020' % namenode_ip + + def get_resource_manager_uri(self, cluster): + resourcemanager_ip = CU.pu.get_resourcemanager(cluster).fqdn() + return '%s:8032' % resourcemanager_ip + + def get_oozie_server(self, cluster): + return CU.pu.get_oozie(cluster) + + def validate_job_execution(self, cluster, job, data): + oo_count = u.get_instances_count(cluster, 'OOZIE_SERVER') + if oo_count != 1: + raise ex.InvalidComponentCountException( + 'OOZIE_SERVER', '1', oo_count) + + super(EdpOozieEngine, self).validate_job_execution(cluster, job, data) + + @staticmethod + def get_possible_job_config(job_type): + if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE): + return {'job_config': ch_helper.get_possible_hive_config_from( + 'plugins/cdh/v5_4_0/resources/hive-site.xml')} + if edp.compare_job_type(job_type, + edp.JOB_TYPE_MAPREDUCE, + edp.JOB_TYPE_MAPREDUCE_STREAMING): + return {'job_config': ch_helper.get_possible_mapreduce_config_from( + 'plugins/cdh/v5_4_0/resources/mapred-site.xml')} + if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG): + return {'job_config': ch_helper.get_possible_pig_config_from( + 'plugins/cdh/v5_4_0/resources/mapred-site.xml')} + return edp_engine.OozieJobEngine.get_possible_job_config(job_type) diff --git a/sahara/plugins/cdh/v5_4_0/plugin_utils.py b/sahara/plugins/cdh/v5_4_0/plugin_utils.py new file mode 100644 index 00000000..3962b2fc --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/plugin_utils.py @@ -0,0 +1,133 @@ +# Copyright (c) 2015 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara.conductor import resource as res +from sahara.plugins.cdh import plugin_utils as pu +from sahara.plugins.cdh.v5_4_0 import config_helper as c_helper +from sahara.plugins.cdh.v5_4_0 import db_helper +from sahara.plugins import utils as u + + +class PluginUtilsV540(pu.AbstractPluginUtils): + def __init__(self): + self.c_helper = c_helper + self.db_helper = db_helper + + def get_role_name(self, instance, service): + # NOTE: role name must match regexp "[_A-Za-z][-_A-Za-z0-9]{0,63}" + shortcuts = { + 'AGENT': 'A', + 'ALERTPUBLISHER': 'AP', + 'CATALOGSERVER': 'ICS', + 'DATANODE': 'DN', + 'EVENTSERVER': 'ES', + 'HBASE_INDEXER': 'LHBI', + 'HIVEMETASTORE': 'HVM', + 'HIVESERVER2': 'HVS', + 'HOSTMONITOR': 'HM', + 'IMPALAD': 'ID', + 'JOBHISTORY': 'JS', + 'MASTER': 'M', + 'NAMENODE': 'NN', + 'NODEMANAGER': 'NM', + 'OOZIE_SERVER': 'OS', + 'REGIONSERVER': 'RS', + 'RESOURCEMANAGER': 'RM', + 'SECONDARYNAMENODE': 'SNN', + 'SENTRY_SERVER': 'SNT', + 'SERVER': 'S', + 'SERVICEMONITOR': 'SM', + 'SOLR_SERVER': 'SLR', + 'SPARK_YARN_HISTORY_SERVER': 'SHS', + 'SQOOP_SERVER': 'S2S', + 'STATESTORE': 'ISS', + 'WEBHCAT': 'WHC' + } + return '%s_%s' % (shortcuts.get(service, service), + instance.hostname().replace('-', '_')) + + def get_sentry(self, cluster): + return u.get_instance(cluster, 'SENTRY_SERVER') + + def get_flumes(self, cluster): + return u.get_instances(cluster, 'FLUME_AGENT') + + def get_solrs(self, cluster): + return u.get_instances(cluster, 'SOLR_SERVER') + + def get_sqoop(self, cluster): + return u.get_instance(cluster, 'SQOOP_SERVER') + + def get_hbase_indexers(self, cluster): + return u.get_instances(cluster, 'KEY_VALUE_STORE_INDEXER') + + def get_catalogserver(self, cluster): + return u.get_instance(cluster, 'IMPALA_CATALOGSERVER') + + def get_statestore(self, cluster): + return u.get_instance(cluster, 'IMPALA_STATESTORE') + + def get_impalads(self, cluster): + return u.get_instances(cluster, 'IMPALAD') + + def convert_process_configs(self, configs): + p_dict = { + "CLOUDERA": ['MANAGER'], + "NAMENODE": ['NAMENODE'], + "DATANODE": ['DATANODE'], + "SECONDARYNAMENODE": ['SECONDARYNAMENODE'], + "RESOURCEMANAGER": ['RESOURCEMANAGER'], + "NODEMANAGER": ['NODEMANAGER'], + "JOBHISTORY": ['JOBHISTORY'], + "OOZIE": ['OOZIE_SERVER'], + "HIVESERVER": ['HIVESERVER2'], + "HIVEMETASTORE": ['HIVEMETASTORE'], + "WEBHCAT": ['WEBHCAT'], + "HUE": ['HUE_SERVER'], + "SPARK_ON_YARN": ['SPARK_YARN_HISTORY_SERVER'], + "ZOOKEEPER": ['SERVER'], + "MASTER": ['MASTER'], + "REGIONSERVER": ['REGIONSERVER'], + "FLUME": ['AGENT'], + "CATALOGSERVER": ['CATALOGSERVER'], + "STATESTORE": ['STATESTORE'], + "IMPALAD": ['IMPALAD'], + "KS_INDEXER": ['HBASE_INDEXER'], + "SENTRY": ['SENTRY_SERVER'], + "SOLR": ['SOLR_SERVER'], + "SQOOP": ['SQOOP_SERVER'] + } + if isinstance(configs, res.Resource): + configs = configs.to_dict() + for k in configs.keys(): + if k in p_dict.keys(): + item = configs[k] + del configs[k] + newkey = p_dict[k][0] + configs[newkey] = item + return res.Resource(configs) + + def configure_sentry(self, cluster): + manager = self.get_manager(cluster) + with manager.remote() as r: + self.db_helper.create_sentry_database(cluster, r) + + def start_cloudera_manager(self, cluster): + self._start_cloudera_manager( + cluster, c_helper.AWAIT_MANAGER_STARTING_TIMEOUT) + + def get_config_value(self, service, name, cluster=None): + configs = c_helper.get_plugin_configs() + return self._get_config_value(service, name, configs, cluster) diff --git a/sahara/plugins/cdh/v5_4_0/resources/cdh_config.py b/sahara/plugins/cdh/v5_4_0/resources/cdh_config.py new file mode 100644 index 00000000..6affa23d --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/cdh_config.py @@ -0,0 +1,135 @@ +# Copyright (c) 2015 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_serialization import jsonutils as json +import six + +from sahara.plugins.cdh.client import api_client + +# -- cm config -- + +cm_address = 'localhost' +cm_port = 7180 +cm_username = 'admin' +cm_password = 'admin' + +hdfs_service_name = 'hdfs01' +yarn_service_name = 'yarn01' +oozie_service_name = 'oozie01' +hive_service_name = 'hive01' +hue_service_name = 'hue01' +spark_service_name = 'spark_on_yarn01' +zookeeper_service_name = 'zookeeper01' +hbase_service_name = 'hbase01' +flume_service_name = 'flume01' +sqoop_service_name = 'sqoop01' +solr_service_name = 'solr01' +ks_indexer_service_name = 'ks_indexer01' +impala_service_name = 'impala01' +sentry_service_name = 'sentry01' + + +def get_cm_api(): + return api_client.ApiResource(cm_address, server_port=cm_port, + username=cm_username, password=cm_password) + + +def get_cluster(api): + return api.get_all_clusters()[0] + + +def process_service(service, service_name): + for role_cfgs in service.get_all_role_config_groups(): + role_cm_cfg = role_cfgs.get_config(view='full') + role_cfg = parse_config(role_cm_cfg) + role_name = role_cfgs.roleType.lower() + write_cfg(role_cfg, '%s-%s.json' % (service_name, role_name)) + + service_cm_cfg = service.get_config(view='full')[0] + service_cfg = parse_config(service_cm_cfg) + write_cfg(service_cfg, '%s-service.json' % service_name) + + +def parse_config(config): + cfg = [] + for name, value in six.iteritems(config): + p = { + 'name': value.name, + 'value': value.default, + 'display_name': value.displayName, + 'desc': value.description + } + cfg.append(p) + + return cfg + + +def write_cfg(cfg, file_name): + to_write = json.dumps(cfg, sort_keys=True, indent=4, + separators=(',', ': ')) + + with open(file_name, 'w') as f: + f.write(to_write) + + +def main(): + client = get_cm_api() + cluster = get_cluster(client) + + hdfs = cluster.get_service(hdfs_service_name) + process_service(hdfs, 'hdfs') + + yarn = cluster.get_service(yarn_service_name) + process_service(yarn, 'yarn') + + oozie = cluster.get_service(oozie_service_name) + process_service(oozie, 'oozie') + + hive = cluster.get_service(hive_service_name) + process_service(hive, 'hive') + + hue = cluster.get_service(hue_service_name) + process_service(hue, 'hue') + + spark = cluster.get_service(spark_service_name) + process_service(spark, 'spark') + + zookeeper = cluster.get_service(zookeeper_service_name) + process_service(zookeeper, 'zookeeper') + + hbase = cluster.get_service(hbase_service_name) + process_service(hbase, 'hbase') + + flume = cluster.get_service(flume_service_name) + process_service(flume, 'flume') + + sqoop = cluster.get_service(sqoop_service_name) + process_service(sqoop, 'sqoop') + + solr = cluster.get_service(solr_service_name) + process_service(solr, 'solr') + + ks_indexer = cluster.get_service(ks_indexer_service_name) + process_service(ks_indexer, 'ks_indexer') + + impala = cluster.get_service(impala_service_name) + process_service(impala, 'impala') + + sentry = cluster.get_service(sentry_service_name) + process_service(sentry, 'sentry') + + +if __name__ == '__main__': + main() diff --git a/sahara/plugins/cdh/v5_4_0/resources/cdh_config.sh b/sahara/plugins/cdh/v5_4_0/resources/cdh_config.sh new file mode 100755 index 00000000..4cfb7b7a --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/cdh_config.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +tox -evenv -- python $(dirname $0)/cdh_config.py $* diff --git a/sahara/plugins/cdh/v5_4_0/resources/create_hive_db.sql b/sahara/plugins/cdh/v5_4_0/resources/create_hive_db.sql new file mode 100644 index 00000000..7545eeb7 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/create_hive_db.sql @@ -0,0 +1,4 @@ +CREATE ROLE hive LOGIN PASSWORD '%s'; +CREATE DATABASE metastore OWNER hive encoding 'UTF8'; +GRANT ALL PRIVILEGES ON DATABASE metastore TO hive; +COMMIT; diff --git a/sahara/plugins/cdh/v5_4_0/resources/create_sentry_db.sql b/sahara/plugins/cdh/v5_4_0/resources/create_sentry_db.sql new file mode 100644 index 00000000..30cb6ab3 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/create_sentry_db.sql @@ -0,0 +1,4 @@ +CREATE ROLE sentry LOGIN PASSWORD '%s'; +CREATE DATABASE sentry OWNER sentry encoding 'UTF8'; +GRANT ALL PRIVILEGES ON DATABASE sentry TO sentry; +COMMIT; diff --git a/sahara/plugins/cdh/v5_4_0/resources/flume-agent.json b/sahara/plugins/cdh/v5_4_0/resources/flume-agent.json new file mode 100644 index 00000000..2aa05e72 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/flume-agent.json @@ -0,0 +1,284 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Used to select an agent configuration to use from flume.conf. Multiple agents may share the same agent name, in which case they will be assigned the same agent configuration.", + "display_name": "Agent Name", + "name": "agent_name", + "value": "tier1" + }, + { + "desc": "

This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.

Each rule has some or all of the following fields:


Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}

This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.

", + "display_name": "Rules to Extract Events from Log Files", + "name": "log_event_whitelist", + "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Text that goes into morphlines.conf file used by the Flume-NG Solr sink. The text goes verbatim into the config file except that $ZK_HOST is replaced by the ZooKeeper quorum of the Solr service.", + "display_name": "Morphlines File", + "name": "agent_morphlines_conf_file", + "value": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Application configuration file in HOCON format (Human-Optimized Config Object Notation). \n# HOCON syntax is defined at http://github.com/typesafehub/config/blob/master/HOCON.md\n# and also used by Akka (http://www.akka.io) and Play (http://www.playframework.org/).\n# For more examples see http://doc.akka.io/docs/akka/2.1.2/general/configuration.html\n\n# morphline.conf example file\n# this is a comment\n\n# Specify server locations in a SOLR_LOCATOR variable; used later in variable substitutions:\nSOLR_LOCATOR : {\n # Name of solr collection\n collection : collection1\n \n # ZooKeeper ensemble\n zkHost : \"$ZK_HOST\"\n \n # Relative or absolute path to a directory containing conf/solrconfig.xml and conf/schema.xml\n # If this path is uncommented it takes precedence over the configuration stored in ZooKeeper. \n # solrHomeDir : \"example/solr/collection1\"\n \n # The maximum number of documents to send to Solr per network batch (throughput knob)\n # batchSize : 100\n}\n\n# Specify an array of one or more morphlines, each of which defines an ETL \n# transformation chain. A morphline consists of one or more (potentially \n# nested) commands. A morphline is a way to consume records (e.g. Flume events, \n# HDFS files or blocks), turn them into a stream of records, and pipe the stream \n# of records through a set of easily configurable transformations on it's way to \n# Solr (or a MapReduceIndexerTool RecordWriter that feeds via a Reducer into Solr).\nmorphlines : [\n {\n # Name used to identify a morphline. E.g. used if there are multiple morphlines in a \n # morphline config file\n id : morphline1 \n \n # Import all morphline commands in these java packages and their subpackages.\n # Other commands that may be present on the classpath are not visible to this morphline.\n importCommands : [\"org.kitesdk.**\", \"org.apache.solr.**\"]\n \n commands : [ \n { \n # Parse Avro container file and emit a record for each avro object\n readAvroContainer {\n # Optionally, require the input record to match one of these MIME types:\n # supportedMimeTypes : [avro/binary]\n \n # Optionally, use a custom Avro schema in JSON format inline:\n # schemaString : \"\"\"\"\"\"\n \n # Optionally, use a custom Avro schema file in JSON format:\n # schemaFile : /path/to/syslog.avsc\n }\n } \n \n { \n # Consume the output record of the previous command and pipe another record downstream.\n #\n # extractAvroPaths is a command that uses zero or more avro path expressions to extract \n # values from an Avro object. Each expression consists of a record output field name (on \n # the left side of the colon ':') as well as zero or more path steps (on the right hand \n # side), each path step separated by a '/' slash. Avro arrays are traversed with the '[]'\n # notation.\n #\n # The result of a path expression is a list of objects, each of which is added to the \n # given record output field.\n # \n # The path language supports all Avro concepts, including nested structures, records, \n # arrays, maps, unions, etc, as well as a flatten option that collects the primitives in \n # a subtree into a flat list.\n extractAvroPaths {\n flatten : false\n paths : { \n id : /id \n text : /text \n user_friends_count : /user_friends_count\n user_location : /user_location\n user_description : /user_description\n user_statuses_count : /user_statuses_count\n user_followers_count : /user_followers_count\n user_name : /user_name\n user_screen_name : /user_screen_name\n created_at : /created_at\n retweet_count : /retweet_count\n retweeted : /retweeted\n in_reply_to_user_id : /in_reply_to_user_id\n source : /source\n in_reply_to_status_id : /in_reply_to_status_id\n media_url_https : /media_url_https\n expanded_url : /expanded_url\n }\n }\n }\n \n # Consume the output record of the previous command and pipe another record downstream.\n #\n # convert timestamp field to native Solr timestamp format\n # e.g. 2012-09-06T07:14:34Z to 2012-09-06T07:14:34.000Z\n {\n convertTimestamp {\n field : created_at\n inputFormats : [\"yyyy-MM-dd'T'HH:mm:ss'Z'\", \"yyyy-MM-dd\"]\n inputTimezone : America/Los_Angeles\n# outputFormat : \"yyyy-MM-dd'T'HH:mm:ss.SSSZ\" \n outputTimezone : UTC\n }\n }\n \n # Consume the output record of the previous command and pipe another record downstream.\n #\n # Command that sanitizes record fields that are unknown to Solr schema.xml by either \n # deleting them (renameToPrefix is absent or a zero length string), or by moving them to a\n # field prefixed with the given renameToPrefix (e.g. renameToPrefix = \"ignored_\" to use \n # typical dynamic Solr fields).\n #\n # Recall that Solr throws an exception on any attempt to load a document that contains a \n # field that isn't specified in schema.xml.\n {\n sanitizeUnknownSolrFields {\n # Location from which to fetch Solr schema\n solrLocator : ${SOLR_LOCATOR}\n \n # renameToPrefix : \"ignored_\"\n }\n } \n \n # log the record at DEBUG level to SLF4J\n { logDebug { format : \"output record: {}\", args : [\"@{}\"] } } \n \n # load the record into a SolrServer or MapReduce SolrOutputFormat.\n { \n loadSolr {\n solrLocator : ${SOLR_LOCATOR}\n }\n }\n ]\n }\n]\n" + }, + { + "desc": "When computing the overall Agent health, consider the host's health.", + "display_name": "Agent Host Health Test", + "name": "flume_agent_host_health_enabled", + "value": "true" + }, + { + "desc": "The health test thresholds on the duration of the metrics request to the web server.", + "display_name": "Web Metric Collection Duration", + "name": "agent_web_metric_collection_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" + }, + { + "desc": "The maximum size, in megabytes, per log file for Agent logs. Typically used by log4j or logback.", + "display_name": "Agent Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "Text that goes verbatim into grok-dictionary.conf file used by the Flume-NG Solr sink.", + "display_name": "Grok Dictionary File", + "name": "agent_grok_dictionary_conf_file", + "value": "USERNAME [a-zA-Z0-9._-]+\nUSER %{USERNAME}\nINT (?:[+-]?(?:[0-9]+))\nBASE10NUM (?[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+)))\nNUMBER (?:%{BASE10NUM})\nBASE16NUM (?(?\"(?>\\\\.|[^\\\\\"]+)+\"|\"\"|(?>'(?>\\\\.|[^\\\\']+)+')|''|(?>`(?>\\\\.|[^\\\\`]+)+`)|``))\nUUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}\n\n# Networking\nMAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})\nCISCOMAC (?:(?:[A-Fa-f0-9]{4}\\.){2}[A-Fa-f0-9]{4})\nWINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})\nCOMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})\nIP (?/(?>[\\w_%!$@:.,-]+|\\\\.)*)+\n#UNIXPATH (?/dev/pts/%{NONNEGINT})\nBSDTTY (?>/dev/tty[pq][a-z0-9])\nTTY (?:%{BSDTTY}|%{LINUXTTY})\nWINPATH (?>[A-Za-z]+:|\\\\)(?:\\\\[^\\\\?*]*)+\nURIPROTO [A-Za-z]+(\\+[A-Za-z+]+)?\nURIHOST %{IPORHOST}(?::%{POSINT:port})?\n# uripath comes loosely from RFC1738, but mostly from what Firefox\n# doesn't turn into %XX\nURIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=#%_\\-]*)+\n#URIPARAM \\?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?\nURIPARAM \\?[A-Za-z0-9$.+!*'|(){},~#%&/=:;_?\\-\\[\\]]*\nURIPATHPARAM %{URIPATH}(?:%{URIPARAM})?\nURI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?\n\n# Months: January, Feb, 3, 03, 12, December\nMONTH \\b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\\b\nMONTHNUM (?:0?[1-9]|1[0-2])\nMONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])\n\n# Days: Monday, Tue, Thu, etc...\nDAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)\n\n# Years?\nYEAR (?>\\d\\d){1,2}\n# Time: HH:MM:SS\n#TIME \\d{2}:\\d{2}(?::\\d{2}(?:\\.\\d+)?)?\n# I'm still on the fence about using grok to perform the time match,\n# since it's probably slower.\n# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\\.%{POSINT})?)?\nHOUR (?:2[0123]|[01]?[0-9])\nMINUTE (?:[0-5][0-9])\n# '60' is a leap second in most time standards and thus is valid.\nSECOND (?:(?:[0-5][0-9]|60)(?:[:.,][0-9]+)?)\nTIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])\n# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)\nDATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}\nDATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}\nISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))\nISO8601_SECOND (?:%{SECOND}|60)\nTIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?\nDATE %{DATE_US}|%{DATE_EU}\nDATESTAMP %{DATE}[- ]%{TIME}\nTZ (?:[PMCE][SD]T)\nDATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}\nDATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}\n\n# Syslog Dates: Month Day HH:MM:SS\nSYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}\nPROG (?:[\\w._/%-]+)\nSYSLOGPROG %{PROG:program}(?:\\[%{POSINT:pid}\\])?\nSYSLOGHOST %{IPORHOST}\nSYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>\nHTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}\n\n# Shortcuts\nQS %{QUOTEDSTRING}\n\n# Log formats\nSYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:\nCOMBINEDAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \\[%{HTTPDATE:timestamp}\\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent}\n\n# Log Levels\nLOGLEVEL ([T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", + "display_name": "Web Metric Collection", + "name": "agent_web_metric_collection_enabled", + "value": "true" + }, + { + "desc": "Verbatim contents of flume.conf. Multiple agents may be configured from the same configuration file; the Agent Name setting can be overridden to select which agent configuration to use for each agent.

To integrate with a secured cluster, you can use the substitution strings \"$KERBEROS_PRINCIPAL\" and \"$KERBEROS_KEYTAB\", which will be replaced by the principal name and the keytab path respectively.", + "display_name": "Configuration File", + "name": "agent_config_file", + "value": "# Please paste flume.conf here. Example:\n\n# Sources, channels, and sinks are defined per\n# agent name, in this case 'tier1'.\ntier1.sources = source1\ntier1.channels = channel1\ntier1.sinks = sink1\n\n# For each source, channel, and sink, set\n# standard properties.\ntier1.sources.source1.type = netcat\ntier1.sources.source1.bind = 127.0.0.1\ntier1.sources.source1.port = 9999\ntier1.sources.source1.channels = channel1\ntier1.channels.channel1.type = memory\ntier1.sinks.sink1.type = logger\ntier1.sinks.sink1.channel = channel1\n\n# Other properties are specific to each type of\n# source, channel, or sink. In this case, we\n# specify the capacity of the memory channel.\ntier1.channels.channel1.capacity = 100\n" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "true" + }, + { + "desc": "The maximum number of rolled log files to keep for Agent logs. Typically used by log4j or logback.", + "display_name": "Agent Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "Enables the health test that the Agent's process state is consistent with the role configuration", + "display_name": "Agent Process Health Test", + "name": "flume_agent_scm_health_enabled", + "value": "true" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "flume_agent_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "The port on which the Flume web server listens for requests.", + "display_name": "HTTP Port", + "name": "agent_http_port", + "value": "41414" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "Agent Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of Agent in Bytes", + "name": "agent_java_heapsize", + "value": "1073741824" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "Disables import of ZooKeeper configuration from the HBase classpath. This prevents zoo.cfg from overriding hbase-site.xml for Zookeeper quorum information. This option is only supported on CDH 4.4 or later deployments.", + "display_name": "HBase sink prefer hbase-site.xml over Zookeeper config", + "name": "agent_disable_zoo_cfg", + "value": "true" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "Agent Environment Advanced Configuration Snippet (Safety Valve)", + "name": "AGENT_role_env_safety_valve", + "value": null + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "Directory where Flume Agent will place its log files.", + "display_name": "Flume Agent Log Directory", + "name": "flume_agent_log_dir", + "value": "/var/log/flume-ng" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Text that goes verbatim into custom-mimetypes.xml file used by the Flume-NG Solr sink.", + "display_name": "Custom Mime-types File", + "name": "agent_custom_mimetypes_file", + "value": "\n\n\n\n\n \n \n \n\n \n \n \n \n \n \n\n \n \n \n \n \n \n \n \n \n \n \n \n" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here. Note that Flume agent only uses options that start with -D and -X (including -XX).", + "display_name": "Java Configuration Options for Flume Agent", + "name": "flume_agent_java_opts", + "value": "" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "List of Flume plugin directories. This overrides the default Flume plugin directory.", + "display_name": "Plugin directories", + "name": "agent_plugin_dirs", + "value": "/usr/lib/flume-ng/plugins.d:/var/lib/flume-ng/plugins.d" + }, + { + "desc": "Home directory for Flume user. The File Channel uses paths for checkpoint and data directories that are within the user home.", + "display_name": "Flume Home Directory", + "name": "agent_home_dir", + "value": "/var/lib/flume-ng" + }, + { + "desc": "The minimum log level for Agent logs", + "display_name": "Agent Logging Threshold", + "name": "log_threshold", + "value": "INFO" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/flume-service.json b/sahara/plugins/cdh/v5_4_0/resources/flume-service.json new file mode 100644 index 00000000..ce949b4a --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/flume-service.json @@ -0,0 +1,104 @@ +[ + { + "desc": "Name of the Solr service that this Flume service instance depends on", + "display_name": "Solr Service", + "name": "solr_service", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Service Level Health Alerts", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "Name of the Hbase service that this Flume service instance depends on", + "display_name": "Hbase Service", + "name": "hbase_service", + "value": null + }, + { + "desc": "The password for the Flume TLS/SSL Certificate Trust Store File. Note that this password is not required to access the trust store: this field can be left blank. This password provides optional integrity checking of the file. The contents of trust stores are certificates, and certificates are public information.", + "display_name": "Flume TLS/SSL Certificate Trust Store Password", + "name": "flume_truststore_password", + "value": null + }, + { + "desc": "The group that this service's processes should run as.", + "display_name": "System Group", + "name": "process_groupname", + "value": "flume" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Flume Service Environment Advanced Configuration Snippet (Safety Valve)", + "name": "flume_env_safety_valve", + "value": null + }, + { + "desc": "Sets the maximum number of Flume components that will be returned under Flume Metric Details. Increasing this value will negatively impact the interactive performance of the Flume Metrics Details page.", + "display_name": "Maximum displayed Flume metrics components", + "name": "flume_context_groups_request_limit", + "value": "1000" + }, + { + "desc": "The location on disk of the trust store, in .jks format, used to confirm the authenticity of TLS/SSL servers that Flume might connect to. This is used when Flume is the client in a TLS/SSL connection. This trust store must contain the certificate(s) used to sign the service(s) being connected to. If this parameter is not provided, the default list of well-known certificate authorities is used instead.", + "display_name": "Flume TLS/SSL Certificate Trust Store File", + "name": "flume_truststore_file", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "The frequency in which the log4j event publication appender will retry sending undelivered log events to the Event server, in seconds", + "display_name": "Log Event Retry Frequency", + "name": "log_event_retry_frequency", + "value": "30" + }, + { + "desc": "The user that this service's processes should run as.", + "display_name": "System User", + "name": "process_username", + "value": "flume" + }, + { + "desc": "Kerberos principal used by all roles of this service.", + "display_name": "Kerberos Principal", + "name": "kerberos_princ_name", + "value": "flume" + }, + { + "desc": "

The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n  \"streamThreshold\": 10, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Service Triggers", + "name": "service_triggers", + "value": "[]" + }, + { + "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", + "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", + "name": "smon_derived_configs_safety_valve", + "value": null + }, + { + "desc": "Name of the HDFS service that this Flume service instance depends on", + "display_name": "HDFS Service", + "name": "hdfs_service", + "value": null + }, + { + "desc": "The health test thresholds of the overall Agent health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Agents falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Agents falls below the critical threshold.", + "display_name": "Healthy Agent Monitoring Thresholds", + "name": "flume_agents_healthy_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"95.0\"}" + }, + { + "desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.", + "display_name": "Enable Log Event Capture", + "name": "catch_events", + "value": "true" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hbase-gateway.json b/sahara/plugins/cdh/v5_4_0/resources/hbase-gateway.json new file mode 100644 index 00000000..5dbc173e --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hbase-gateway.json @@ -0,0 +1,56 @@ +[ + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "Gateway Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "For advanced use only, a string to be inserted into the client configuration for hbase-site.xml.", + "display_name": "HBase Client Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", + "name": "hbase_client_config_safety_valve", + "value": null + }, + { + "desc": "The directory where the client configs will be deployed", + "display_name": "Deploy Directory", + "name": "client_config_root_dir", + "value": "/etc/hbase" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the client configuration for hbase-env.sh", + "display_name": "HBase Client Environment Advanced Configuration Snippet for hbase-env.sh (Safety Valve)", + "name": "hbase_client_env_safety_valve", + "value": null + }, + { + "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", + "display_name": "Alternatives Priority", + "name": "client_config_priority", + "value": "90" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "These are Java command line arguments. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Client Java Configuration Options", + "name": "hbase_client_java_opts", + "value": "-XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:+CMSIncrementalMode -Djava.net.preferIPv4Stack=true" + }, + { + "desc": "Maximum size in bytes for the Java process heap memory. Passed to Java -Xmx.", + "display_name": "Client Java Heap Size in Bytes", + "name": "hbase_client_java_heapsize", + "value": "268435456" + }, + { + "desc": "The minimum log level for Gateway logs", + "display_name": "Gateway Logging Threshold", + "name": "log_threshold", + "value": "INFO" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hbase-hbaserestserver.json b/sahara/plugins/cdh/v5_4_0/resources/hbase-hbaserestserver.json new file mode 100644 index 00000000..487e376f --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hbase-hbaserestserver.json @@ -0,0 +1,278 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "

This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.

Each rule has some or all of the following fields:


Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}

This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.

", + "display_name": "Rules to Extract Events from Log Files", + "name": "log_event_whitelist", + "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for HBase REST Server", + "name": "hbase_restserver_java_opts", + "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "Maximum size of the HBase REST Server thread pool. The server can process this number of concurrent requests. Setting this too high can lead to out of memory errors.", + "display_name": "HBase REST Server Maximum Threads", + "name": "hbase_restserver_threads_max", + "value": "100" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "hbaserestserver_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "HBase REST Server Environment Advanced Configuration Snippet (Safety Valve)", + "name": "HBASERESTSERVER_role_env_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", + "display_name": "HBase REST Server Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", + "name": "hbase_restserver_config_safety_valve", + "value": null + }, + { + "desc": "Directory where HBase REST Server will place its log files.", + "display_name": "HBase REST Server Log Directory", + "name": "hbase_restserver_log_dir", + "value": "/var/log/hbase" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "The maximum number of rolled log files to keep for HBase REST Server logs. Typically used by log4j or logback.", + "display_name": "HBase REST Server Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of HBase REST Server in Bytes", + "name": "hbase_restserver_java_heapsize", + "value": "1073741824" + }, + { + "desc": "Enables the health test that the HBase REST Server's process state is consistent with the role configuration", + "display_name": "HBase REST Server Process Health Test", + "name": "hbaserestserver_scm_health_enabled", + "value": "true" + }, + { + "desc": "The port that HBase REST Server binds to.", + "display_name": "HBase REST Server Port", + "name": "hbase_restserver_port", + "value": "20550" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "If true, HBase REST Server Web UI will bind to a wildcard address (0.0.0.0). Otherwise it will bind to a host name. Only available in CDH 4.3 and later.", + "display_name": "HBase REST Server Web UI Bind to Wildcard Address", + "name": "hbase_restserver_info_bind_to_wildcard", + "value": "true" + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "HBase REST Server Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "When false, all HTTP methods are permitted (GET/PUT/POST/DELETE). When true, only GET is permitted.", + "display_name": "Enable HBase REST Server Read Only Mode", + "name": "hbase_restserver_readonly", + "value": "false" + }, + { + "desc": "The port that HBase REST Server Web UI binds to.", + "display_name": "HBase REST Server Web UI Port", + "name": "hbase_restserver_info_port", + "value": "8085" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "When computing the overall HBase REST Server health, consider the host's health.", + "display_name": "HBase REST Server Host Health Test", + "name": "hbaserestserver_host_health_enabled", + "value": "true" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The maximum size, in megabytes, per log file for HBase REST Server logs. Typically used by log4j or logback.", + "display_name": "HBase REST Server Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The host name or IP address of the DNS name server which an HBase REST Server should use to determine the host name used for communication and display purposes.", + "display_name": "HBase REST Server DNS Name Server", + "name": "hbase_restserver_dns_nameserver", + "value": null + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "HBase REST Server will bind to this address.", + "display_name": "HBase REST Server Host Address", + "name": "hbase_restserver_host", + "value": "0.0.0.0" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "The name of the DNS network interface from which an HBase REST Server should report its IP address.", + "display_name": "HBase REST Server DNS Network Interface", + "name": "hbase_restserver_dns_interface", + "value": null + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "Minimum size of the HBase REST Server thread pool. The server will maintain at least this number of threads in the pool at all times. The thread pool can grow up to the maximum size set by hbase.rest.threads.max.", + "display_name": "HBase REST Server Minimum Threads", + "name": "hbase_restserver_threads_min", + "value": "2" + }, + { + "desc": "The minimum log level for HBase REST Server logs", + "display_name": "HBase REST Server Logging Threshold", + "name": "log_threshold", + "value": "INFO" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hbase-hbasethriftserver.json b/sahara/plugins/cdh/v5_4_0/resources/hbase-hbasethriftserver.json new file mode 100644 index 00000000..588ea74e --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hbase-hbasethriftserver.json @@ -0,0 +1,284 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "

This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.

Each rule has some or all of the following fields:


Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}

This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.

", + "display_name": "Rules to Extract Events from Log Files", + "name": "log_event_whitelist", + "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "The \"core size\" of the thread pool. New threads are created on every connection until this many threads are created.", + "display_name": "HBase Thrift Server Min Worker Threads", + "name": "hbase_thriftserver_min_worker_threads", + "value": "200" + }, + { + "desc": "The port that HBase Thrift Server binds to.", + "display_name": "HBase Thrift Server Port", + "name": "hbase_thriftserver_port", + "value": "9090" + }, + { + "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", + "display_name": "HBase Thrift Server Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", + "name": "hbase_thriftserver_config_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "The maximum number of rolled log files to keep for HBase Thrift Server logs. Typically used by log4j or logback.", + "display_name": "HBase Thrift Server Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of HBase Thrift Server in Bytes", + "name": "hbase_thriftserver_java_heapsize", + "value": "1073741824" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "If true, HBase Thrift Server Web UI will bind to a wildcard address (0.0.0.0). Otherwise it will bind to a host name. Only available in CDH 4.3 and later.", + "display_name": "HBase Thrift Server Web UI Bind to Wildcard Address", + "name": "hbase_thriftserver_info_bind_to_wildcard", + "value": "true" + }, + { + "desc": "Address to bind the HBase Thrift Server to. When using the THsHaServer or the TNonblockingServer, always binds to 0.0.0.0 irrespective of this configuration value.", + "display_name": "HBase Thrift Server Bind Address", + "name": "hbase_thriftserver_bindaddress", + "value": "0.0.0.0" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "Type of HBase Thrift Server.", + "display_name": "HBase Thrift Server Type", + "name": "hbase_thriftserver_type", + "value": "threadpool" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "HBase Thrift Server Environment Advanced Configuration Snippet (Safety Valve)", + "name": "HBASETHRIFTSERVER_role_env_safety_valve", + "value": null + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "HBase Thrift Server Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "When computing the overall HBase Thrift Server health, consider the host's health.", + "display_name": "HBase Thrift Server Host Health Test", + "name": "hbasethriftserver_host_health_enabled", + "value": "true" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Enables the health test that the HBase Thrift Server's process state is consistent with the role configuration", + "display_name": "HBase Thrift Server Process Health Test", + "name": "hbasethriftserver_scm_health_enabled", + "value": "true" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for HBase Thrift Server", + "name": "hbase_thriftserver_java_opts", + "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" + }, + { + "desc": "Use framed transport. When using the THsHaServer or TNonblockingServer, framed transport is always used irrespective of this configuration value.", + "display_name": "Enable HBase Thrift Server Framed Transport", + "name": "hbase_thriftserver_framed", + "value": "false" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The maximum size, in megabytes, per log file for HBase Thrift Server logs. Typically used by log4j or logback.", + "display_name": "HBase Thrift Server Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "The port that HBase Thrift Server Web UI binds to.", + "display_name": "HBase Thrift Server Web UI Port", + "name": "hbase_thriftserver_info_port", + "value": "9095" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "The host name or IP address of the DNS name server which an HBase Thrift Server should use to determine the host name used for communication and display purposes.", + "display_name": "HBase Thrift Server DNS Name Server", + "name": "hbase_thriftserver_dns_nameserver", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "The name of the DNS network interface from which an HBase Thrift Server should report its IP address.", + "display_name": "HBase Thrift Server DNS Network Interface", + "name": "hbase_thriftserver_dns_interface", + "value": null + }, + { + "desc": "Use the TCompactProtocol instead of the default TBinaryProtocol. TCompactProtocol is a binary protocol that is more compact than the default and typically more efficient.", + "display_name": "Enable HBase Thrift Server Compact Protocol", + "name": "hbase_thriftserver_compact", + "value": "false" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "Directory where HBase Thrift Server will place its log files.", + "display_name": "HBase Thrift Server Log Directory", + "name": "hbase_thriftserver_log_dir", + "value": "/var/log/hbase" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "hbasethriftserver_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "The minimum log level for HBase Thrift Server logs", + "display_name": "HBase Thrift Server Logging Threshold", + "name": "log_threshold", + "value": "INFO" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hbase-master.json b/sahara/plugins/cdh/v5_4_0/resources/hbase-master.json new file mode 100644 index 00000000..24212217 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hbase-master.json @@ -0,0 +1,350 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "

This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.

Each rule has some or all of the following fields:


Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}

This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.

", + "display_name": "Rules to Extract Events from Log Files", + "name": "log_event_whitelist", + "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"IPC Server handler.*ClosedChannelException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"IPC Server Responder, call.*output error\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Daughter regiondir does not exist: .*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"File.*might still be open.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"File.*might still be open.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Moving table .+ state to enabled but was already enabled\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Received OPENED for region.*but region was in the state.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" + }, + { + "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", + "display_name": "Health Check Startup Tolerance", + "name": "master_startup_tolerance", + "value": "5" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "Number of pooled threads to handle region closing in the master.", + "display_name": "Region Closing Threads", + "name": "hbase_master_executor_closeregion_threads", + "value": "5" + }, + { + "desc": "When computing the overall Master health, consider the host's health.", + "display_name": "Master Host Health Test", + "name": "master_host_health_enabled", + "value": "true" + }, + { + "desc": "Number of pooled threads to handle the recovery of the region servers in the master.", + "display_name": "RegionServer Recovery Threads", + "name": "hbase_master_executor_serverops_threads", + "value": "5" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Directory where Master will place its log files.", + "display_name": "Master Log Directory", + "name": "hbase_master_log_dir", + "value": "/var/log/hbase" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The health test thresholds on the duration of the metrics request to the web server.", + "display_name": "Web Metric Collection Duration", + "name": "master_web_metric_collection_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" + }, + { + "desc": "Time period in seconds to reset long-running metrics (e.g. compactions). This is an HBase specific configuration.", + "display_name": "Extended Period", + "name": "hbase_metrics_extended_period", + "value": "3600" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "Master Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "master_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "The port that the HBase Master binds to.", + "display_name": "HBase Master Port", + "name": "hbase_master_port", + "value": "60000" + }, + { + "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", + "display_name": "Web Metric Collection", + "name": "master_web_metric_collection_enabled", + "value": "true" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "List of org.apache.hadoop.hbase.coprocessor.MasterObserver coprocessors that are loaded by default on the active HMaster process. For any implemented coprocessor methods, the listed classes will be called in order. After implementing your own MasterObserver, just put it in HBase's classpath and add the fully qualified class name here.", + "display_name": "HBase Coprocessor Master Classes", + "name": "hbase_coprocessor_master_classes", + "value": "" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of HBase Master in Bytes", + "name": "hbase_master_java_heapsize", + "value": "1073741824" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", + "display_name": "Garbage Collection Duration Thresholds", + "name": "master_gc_duration_thresholds", + "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "When true, HBase Master will bind to 0.0.0.0. Only available with CDH 4.3 and later.", + "display_name": "HBase Master Bind to Wildcard Address", + "name": "hbase_master_bind_to_wildcard_address", + "value": "true" + }, + { + "desc": "Enables the health test that a client can connect to the HBase Master", + "display_name": "HBase Master Canary Health Test", + "name": "master_canary_health_enabled", + "value": "true" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", + "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", + "name": "hadoop_metrics2_safety_valve", + "value": null + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "The port for the HBase Master web UI. Set to -1 to disable the HBase Master web UI.", + "display_name": "HBase Master Web UI Port", + "name": "hbase_master_info_port", + "value": "60010" + }, + { + "desc": "The period to review when computing the moving average of garbage collection time.", + "display_name": "Garbage Collection Duration Monitoring Period", + "name": "master_gc_duration_window", + "value": "5" + }, + { + "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", + "display_name": "Master Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", + "name": "hbase_master_config_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for HBase Master", + "name": "hbase_master_java_opts", + "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The maximum size, in megabytes, per log file for Master logs. Typically used by log4j or logback.", + "display_name": "Master Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "Maximum time an HLog remains in the .oldlogdir directory until an HBase Master thread deletes it.", + "display_name": "Maximum Time to Keep HLogs", + "name": "hbase_master_logcleaner_ttl", + "value": "60000" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "Master Environment Advanced Configuration Snippet (Safety Valve)", + "name": "MASTER_role_env_safety_valve", + "value": null + }, + { + "desc": "Number of RPC Server instances spun up on HBase Master.", + "display_name": "HBase Master Handler Count", + "name": "hbase_master_handler_count", + "value": "25" + }, + { + "desc": "The address for the HBase Master web UI", + "display_name": "HBase Master Web UI Address", + "name": "hbase_master_info_bindAddress", + "value": null + }, + { + "desc": "A comma-separated list of LogCleanerDelegate(s) that are used in LogsCleaner. WAL/HLog cleaner(s) are called in order, so put the log cleaner that prunes the most log files in the front. To implement your own LogCleanerDelegate, add it to HBase's classpath and add the fully-qualified class name here. You should always add the above default log cleaners in the list, unless you have a special reason not to.", + "display_name": "HBase Master Log Cleaner Plugins", + "name": "hbase_master_logcleaner_plugins", + "value": null + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "The maximum number of rolled log files to keep for Master logs. Typically used by log4j or logback.", + "display_name": "Master Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "The host name or IP address of the DNS name server which an HBase Master should use to determine the host name used for communication and display purposes.", + "display_name": "HBase Master DNS Name Server", + "name": "hbase_master_dns_nameserver", + "value": null + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "Number of pooled threads to handle region opening in the master.", + "display_name": "Region Opening Threads", + "name": "hbase_master_executor_openregion_threads", + "value": "5" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "The name of the DNS network interface from which an HBase Master should report its IP address.", + "display_name": "HBase Master DNS Network Interface", + "name": "hbase_master_dns_interface", + "value": null + }, + { + "desc": "The minimum log level for Master logs", + "display_name": "Master Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "Enables the health test that the Master's process state is consistent with the role configuration", + "display_name": "Master Process Health Test", + "name": "master_scm_health_enabled", + "value": "true" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hbase-regionserver.json b/sahara/plugins/cdh/v5_4_0/resources/hbase-regionserver.json new file mode 100644 index 00000000..803a4b8a --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hbase-regionserver.json @@ -0,0 +1,584 @@ +[ + { + "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", + "display_name": "RegionServer Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", + "name": "hbase_regionserver_config_safety_valve", + "value": null + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "Maximum number of HStoreFiles to compact per minor compaction.", + "display_name": "Maximum Number of HStoreFiles Compaction", + "name": "hbase_hstore_compaction_max", + "value": null + }, + { + "desc": "The address for the HBase RegionServer web UI", + "display_name": "HBase RegionServer Web UI Address", + "name": "hbase_regionserver_info_bindAddress", + "value": null + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "Number of threads to use while loading and unloading regions to or from a RegionServer. Can be used to increase the speed of decommissioning or rolling restart operations.", + "display_name": "Region Mover Threads", + "name": "hbase_regionserver_regionmover_thread_count", + "value": "1" + }, + { + "desc": "The health test thresholds of the average size of the HBase RegionServer flush queue over a recent period. See HBase RegionServer Flush Queue Monitoring Period.", + "display_name": "HBase RegionServer Flush Queue Monitoring Thresholds", + "name": "regionserver_flush_queue_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10.0\"}" + }, + { + "desc": "The amount of time to wait for the HBase Region Server to fully start up and connect to the HBase Master before enforcing the connectivity check.", + "display_name": "HBase Region Server Connectivity Tolerance at Startup", + "name": "regionserver_connectivity_tolerance", + "value": "180" + }, + { + "desc": "The hashing algorithm for use in HashFunction. Two values are supported: 'murmur' (for MurmurHash) and 'jenkins' (for JenkinsHash).", + "display_name": "HBase Hash Type", + "name": "hbase_hash_type", + "value": "murmur" + }, + { + "desc": "The port for the HBase RegionServer web UI. Set to -1 to disable RegionServer web UI.", + "display_name": "HBase RegionServer Web UI port", + "name": "hbase_regionserver_info_port", + "value": "60030" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "The period over which to compute the moving average of the compaction queue size.", + "display_name": "HBase RegionServer Compaction Queue Monitoring Period", + "name": "regionserver_compaction_queue_window", + "value": "5" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for HBase RegionServer", + "name": "hbase_regionserver_java_opts", + "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "RegionServer Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds of the latency that the RegionServer sees for HDFS read operations", + "display_name": "HBase RegionServer HDFS Read Latency Thresholds", + "name": "regionserver_read_latency_thresholds", + "value": "{\"critical\":\"100.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds of the size used by the HStoreFile index. Specified as a percentage of the total heap size.", + "display_name": "Percentage of Heap Used by HStoreFile Index", + "name": "regionserver_store_file_idx_size_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10.0\"}" + }, + { + "desc": "Maximum number of Write-Ahead Log (WAL) files. This value multiplied by HDFS Block Size (dfs.blocksize) is the size of the WAL that will need to be replayed when a server crashes. This value is inversely proportional to the frequency of flushes to disk.", + "display_name": "Maximum number of Write-Ahead Log (WAL) files", + "name": "hbase_regionserver_maxlogs", + "value": "32" + }, + { + "desc": "List of coprocessors that are loaded by default on all tables. For any override coprocessor method, these classes will be called in order. After implementing your own coprocessor, just put it in HBase's classpath and add the fully qualified class name here. A coprocessor can also be loaded on demand by setting HTableDescriptor.", + "display_name": "HBase Coprocessor Region Classes", + "name": "hbase_coprocessor_region_classes", + "value": "" + }, + { + "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", + "display_name": "Garbage Collection Duration Thresholds", + "name": "regionserver_gc_duration_thresholds", + "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" + }, + { + "desc": "Blocks writes if the size of the memstore increases to the value of 'hbase.hregion.block.memstore' multiplied by the value of 'hbase.hregion.flush.size' bytes. This setting is useful for preventing runaway memstore during spikes in update traffic. Without an upper-bound, memstore fills such that when it flushes, the resultant process of flushing files take a long time to compact or split, or worse, an \"out of memory\" error occurs.", + "display_name": "HBase Memstore Block Multiplier", + "name": "hbase_hregion_memstore_block_multiplier", + "value": "2" + }, + { + "desc": "Sync the HLog to HDFS after this interval, in milliseconds, if it has not accumulated the number of HLog Entries specified to trigger a sync.", + "display_name": "Synch Interval of HLog Entries", + "name": "hbase_regionserver_optionallogflushinterval", + "value": "1000" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "

This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.

Each rule has some or all of the following fields:


Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}

This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.

", + "display_name": "Rules to Extract Events from Log Files", + "name": "log_event_whitelist", + "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \"IPC Server handler.*took.*appending an edit to hlog.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"ABORTING region server serverName.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"DFSOutputStream ResponseProcessor exception.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Error Recovery for block blk.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Failed init\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Problem renewing lease for DFSClient.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"remote error telling master we are up\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Session.*for server.*closing socket connection and attempting reconnect.\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Starting .*compaction on region (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_REGION_STARTED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^completed compaction on region (.+) after (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_REGION_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Starting compaction on (.+) in region (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_COLUMN_FAMILY_STARTED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"COLUMN_FAMILY\", \"group1\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^completed compaction: regionName=(.+), storeName=(.+), fileCount=(.+), fileSize=(.+), priority=(.+), time=(.+); duration=(.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_COLUMN_FAMILY_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"COLUMN_FAMILY\", \"group2\": \"FILE_COUNT\", \"group3\": \"FILE_SIZE\", \"group4\": \"PRIORITY\", \"group5\": \"REQUEST_TIME_NANOS\", \"group6\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Completed compaction: Request = regionName=(.+), storeName=(.+), fileCount=(.+), fileSize=(.+), priority=(.+), time=(.+); duration=(.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_COLUMN_FAMILY_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"COLUMN_FAMILY\", \"group2\": \"FILE_COUNT\", \"group3\": \"FILE_SIZE\", \"group4\": \"PRIORITY\", \"group5\": \"REQUEST_TIME_NANOS\", \"group6\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^aborted compaction: regionName=(.+), storeName=(.+), fileCount=(.+), fileSize=(.+), priority=(.+), time=(.+); duration=(.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_COLUMN_FAMILY_ABORTED\", \"attribute:SEVERITY\": \"IMPORTANT\", \"group0\": \"REGION\", \"group1\": \"COLUMN_FAMILY\", \"group2\": \"FILE_COUNT\", \"group3\": \"FILE_SIZE\", \"group4\": \"PRIORITY\", \"group5\": \"REQUEST_TIME_NANOS\", \"group6\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Finished memstore flush of .+ for region (.+) in (.+), sequenceid=(.+), compaction requested=(.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_FLUSH_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"DURATION\", \"group2\": \"SEQUENCE_ID\", \"group3\": \"COMPACTION_REQUESTED\" },\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Flush of region (.+) due to global heap pressure$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_FLUSH_DUE_TO_HEAP_PRESSURE\", \"attribute:SEVERITY\": \"IMPORTANT\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"WARN\", \"content\": \"^Region (.+) has too many store files; delaying flush up to 90000ms$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_FLUSH_DELAYED_TOO_MANY_STORE_FILES\", \"attribute:SEVERITY\": \"CRITICAL\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Starting split of region (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_SPLIT_STARTED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Running rollback/cleanup of failed split of (.+);.+$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_SPLIT_ABORTED\", \"attribute:SEVERITY\": \"IMPORTANT\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Region split, hbase:meta updated, and report to master. Parent=(.+), new regions: (.+,.*,.+), (.+,.*,.+). Split took (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_SPLIT_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"DAUGHTER_REGIONS\", \"group2\": \"DAUGHTER_REGIONS\", \"group3\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Region split, META updated, and report to master. Parent=(.+), new regions: (.+,.*,.+), (.+,.*,.+). Split took (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_SPLIT_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"DAUGHTER_REGIONS\", \"group2\": \"DAUGHTER_REGIONS\", \"group3\": \"DURATION\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" + }, + { + "desc": "The host name or IP address of the DNS name server which a RegionServer should use to determine the host name used by the HBase Master for communication and display purposes.", + "display_name": "RegionServer DNS Nameserver", + "name": "hbase_regionserver_dns_nameserver", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The health test thresholds of the total size of RegionServer's memstores. Specified as a percentage of the configured upper limit. See Maximum Size of All Memstores in RegionServer.", + "display_name": "HBase RegionServer Memstore Size Thresholds", + "name": "regionserver_memstore_size_thresholds", + "value": "{\"critical\":\"100.0\",\"warning\":\"95.0\"}" + }, + { + "desc": "Percentage of maximum heap (-Xmx setting) to allocate to block cache used by HFile/StoreFile. To disable, set this value to 0 .", + "display_name": "HFile Block Cache Size", + "name": "hfile_block_cache_size", + "value": "0.4" + }, + { + "desc": "The maximum size, in megabytes, per log file for RegionServer logs. Typically used by log4j or logback.", + "display_name": "RegionServer Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "If the memstores in a region are this size or larger when closing, run a pre-flush process to clear out memstores before putting up the region closed flag and taking the region offline. On close, a flush process is run under the close flag up to empty memory. During this time, the region is offline and no writes are taken. If the memstore content is large, the flush process could take a long time to complete. The pre-flush process cleans out the bulk of the memstore before putting up the close flag and taking the region offline, so that the flush process that runs under the close flag has little to do.", + "display_name": "HBase Memstore Pre-close Flush Size", + "name": "hbase_hregion_preclose_flush_size", + "value": "5242880" + }, + { + "desc": "If this number of HStoreFiles in any one HStore is exceeded, then a compaction is run to rewrite all HStoreFiles files as one HStoreFile. (One HStoreFile is written per flush of memstore.) You can delay compaction by specifying a larger number, but the compaction will take longer when it does run. During a compaction, updates cannot be flushed to disk. Long compactions require memory sufficient to carry the logging of all updates across the duration of the compaction. If too large, clients timeout during compaction.", + "display_name": "HStore Compaction Threshold", + "name": "hbase_hstore_compactionThreshold", + "value": "3" + }, + { + "desc": "The period over which to compute the moving average of the HDFS sync latency of the HBase RegionServer.", + "display_name": "HBase RegionServer HDFS Sync Latency Monitoring Period", + "name": "regionserver_sync_latency_window", + "value": "5" + }, + { + "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", + "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", + "name": "hadoop_metrics2_safety_valve", + "value": null + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "The maximum size byte array that should come from the MSLAB, in bytes.", + "display_name": "Maximum Byte Array from MSLAB Allocation Scheme", + "name": "hbase_hregion_memstore_mslab_max_allocation", + "value": "262144" + }, + { + "desc": "The health test thresholds for the latency of HDFS write operations that the RegionServer detects", + "display_name": "HBase RegionServer HDFS Sync Latency Thresholds", + "name": "regionserver_sync_latency_thresholds", + "value": "{\"critical\":\"5000.0\",\"warning\":\"500.0\"}" + }, + { + "desc": "Maximum size of all memstores in a RegionServer before new updates are blocked and flushes are forced.", + "display_name": "Maximum Size of All Memstores in RegionServer", + "name": "hbase_regionserver_global_memstore_upperLimit", + "value": "0.4" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "Directory where RegionServer will place its log files.", + "display_name": "RegionServer Log Directory", + "name": "hbase_regionserver_log_dir", + "value": "/var/log/hbase" + }, + { + "desc": "The period of time that an HRegion will block updates after reaching the HStoreFile limit that is specified by 'hbase.hstore.blockingStoreFiles'. After this time has elapsed, the HRegion will stop blocking updates even if a compaction has not been completed.", + "display_name": "HStore Blocking Wait Time", + "name": "hbase_hstore_blockingWaitTime", + "value": "90000" + }, + { + "desc": "Timeout for Canary to perform its checks.", + "display_name": "Canary Timeout", + "name": "hbase_regionserver_canary_timeout", + "value": "15" + }, + { + "desc": "Interval, in milliseconds, between messages from the RegionServer to the HBase Master. Use a high value such as 3000 for clusters that have more than 10 hosts.", + "display_name": "HBase RegionServer Message Interval", + "name": "hbase_regionserver_msginterval", + "value": "3000" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The lease period, in milliseconds, for the HBase RegionServer. Clients must report in within this period or else they are considered dead.", + "display_name": "HBase RegionServer Lease Period", + "name": "hbase_regionserver_lease_period", + "value": "60000" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "The minimum log level for RegionServer logs", + "display_name": "RegionServer Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", + "display_name": "Web Metric Collection", + "name": "regionserver_web_metric_collection_enabled", + "value": "true" + }, + { + "desc": "The name of the DNS Network Interface from which a RegionServer should report its IP address.", + "display_name": "RegionServer DNS Network Interface", + "name": "hbase_regionserver_dns_interface", + "value": null + }, + { + "desc": "The number of reservation blocks that are used to prevent unstable RegionServers caused by an OOME.", + "display_name": "RegionServer Reservation Blocks", + "name": "hbase_regionserver_nbreservationblocks", + "value": "4" + }, + { + "desc": "The implementation that should be used by the RegionServer for the write-ahead-log.", + "display_name": "WAL Provider", + "name": "hbase_regionserver_wal_provider", + "value": "multiwal" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "regionserver_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "The size of the chunks allocated by MSLAB, in bytes.", + "display_name": "Chunk Size Allocated by MSLAB Allocation Scheme", + "name": "hbase_hregion_memstore_mslab_chunksize", + "value": "2097152" + }, + { + "desc": "The HLog file reader implementation.", + "display_name": "HLog Reader Implementation", + "name": "hbase_regionserver_hlog_reader_impl", + "value": null + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "The period over which to compute the moving average of the HDFS read latency of the HBase RegionServer.", + "display_name": "HBase RegionServer HDFS Read Latency Monitoring Period", + "name": "regionserver_read_latency_window", + "value": "5" + }, + { + "desc": "The directory for HBase secure bulk Load.", + "display_name": "HBase Secure Bulk Load Directory", + "name": "hbase_bulkload_staging_dir", + "value": "/tmp/hbase-staging" + }, + { + "desc": "An interface that is assignable to HRegionInterface. Used in HBase Client for opening a proxy to a remote HBase RegionServer.", + "display_name": "HBase RegionServer Interface Class", + "name": "hbase_regionserver_class", + "value": null + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "When true, HBase RegionServer will bind to 0.0.0.0. Only available in CDH 4.3 and later.", + "display_name": "HBase RegionServer Bind to Wildcard Address", + "name": "hbase_regionserver_bind_to_wildcard_address", + "value": "true" + }, + { + "desc": "The time, in milliseconds, between 'major' compactions of all HStoreFiles in a region. To disable automated major compactions, set this value to 0.", + "display_name": "HBase HRegion Major Compaction", + "name": "hbase_hregion_majorcompaction", + "value": "604800000" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Enable MemStore-Local Allocation Buffer (MSLAB) Allocation Scheme. Note: This feature is experimental in CDH3.", + "display_name": "Enable MSLAB Allocation Scheme", + "name": "hbase_hregion_memstore_mslab_enabled", + "value": "true" + }, + { + "desc": "Enables the health test that the RegionServer is connected to the Master", + "display_name": "HBase RegionServer to Master Connectivity Test", + "name": "regionserver_master_connectivity_enabled", + "value": "true" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": "32768" + }, + { + "desc": "Limit for the number of regions after which no more region splitting should take place. This is not a hard limit for the number of regions but acts as a guideline for the RegionServer to stop splitting after a certain limit.", + "display_name": "HBase Region Split Limit", + "name": "hbase_regionserver_regionSplitLimit", + "value": "2147483647" + }, + { + "desc": "When using MultiWAL as the WALProvider, sets how many write-ahead-logs each region server should run. Will result in this number of HDFS pipelines. Writes for a given Region only go to a single pipeline, spreading total RegionServer load.", + "display_name": "Per-RegionServer Number of WAL Pipelines", + "name": "hbase_regionserver_wal_pipelines", + "value": "1" + }, + { + "desc": "Enables the health test that the RegionServer's process state is consistent with the role configuration", + "display_name": "RegionServer Process Health Test", + "name": "regionserver_scm_health_enabled", + "value": "true" + }, + { + "desc": "Memstore will be flushed to disk if size of the memstore exceeds this value in number of bytes. This value is checked by a thread that runs the frequency specified by hbase.server.thread.wakefrequency.", + "display_name": "HBase Memstore Flush Size", + "name": "hbase_hregion_memstore_flush_size", + "value": "134217728" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "When memstores are being forced to flush to make room in memory, keep flushing until this amount is reached. If this amount is equal to 'hbase.regionserver.global.memstore.upperLimit', then minimum possible flushing will occur when updates are blocked due to memstore limiting.", + "display_name": "Low Watermark for Memstore Flush", + "name": "hbase_regionserver_global_memstore_lowerLimit", + "value": "0.38" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "The period to review when computing the moving average of garbage collection time.", + "display_name": "Garbage Collection Duration Monitoring Period", + "name": "regionserver_gc_duration_window", + "value": "5" + }, + { + "desc": "Time period in seconds to reset long-running metrics (e.g. compactions). This is an HBase specific configuration.", + "display_name": "Extended Period", + "name": "hbase_metrics_extended_period", + "value": "3600" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "RegionServer Environment Advanced Configuration Snippet (Safety Valve)", + "name": "REGIONSERVER_role_env_safety_valve", + "value": null + }, + { + "desc": "Comma-separated list of codecs that the RegionServer requires to start. Use this setting to make sure that all RegionServers joining a cluster are installed with a particular set of codecs.", + "display_name": "RegionServer Codecs", + "name": "hbase_regionserver_codecs", + "value": "" + }, + { + "desc": "Number of handlers for processing priority requests in a RegionServer.", + "display_name": "HBase RegionServer Meta-Handler Count", + "name": "hbase_regionserver_metahandler_count", + "value": "10" + }, + { + "desc": "The maximum number of rolled log files to keep for RegionServer logs. Typically used by log4j or logback.", + "display_name": "RegionServer Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "Period, in milliseconds, at which to roll the commit log.", + "display_name": "HBase RegionServer Log Roll Period", + "name": "hbase_regionserver_logroll_period", + "value": "3600000" + }, + { + "desc": "The period over which to compute the moving average of the flush queue size.", + "display_name": "HBase RegionServer Flush Queue Monitoring Period", + "name": "regionserver_flush_queue_window", + "value": "5" + }, + { + "desc": "Maximum HStoreFile size. If any one of a column families' HStoreFiles has grown to exceed this value, the hosting HRegion is split in two.", + "display_name": "HBase Maximum File Size", + "name": "hbase_hregion_max_filesize", + "value": "10737418240" + }, + { + "desc": "The health test thresholds of the weighted average size of the HBase RegionServer compaction queue over a recent period. See HBase RegionServer Compaction Queue Monitoring Period.", + "display_name": "HBase RegionServer Compaction Queue Monitoring Thresholds", + "name": "regionserver_compaction_queue_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10.0\"}" + }, + { + "desc": "If there are more than this number of HStoreFiles in any one HStore, then updates are blocked for this HRegion until a compaction is completed, or until the value specified for 'hbase.hstore.blockingWaitTime' has been exceeded.", + "display_name": "HStore Blocking Store Files", + "name": "hbase_hstore_blockingStoreFiles", + "value": "10" + }, + { + "desc": "The health test thresholds on the duration of the metrics request to the web server.", + "display_name": "Web Metric Collection Duration", + "name": "regionserver_web_metric_collection_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "The HLog file writer implementation.", + "display_name": "HLog Writer Implementation", + "name": "hbase_regionserver_hlog_writer_impl", + "value": null + }, + { + "desc": "Duration between consecutive checks done by the Canary.", + "display_name": "Canary Interval", + "name": "hbase_regionserver_canary_interval", + "value": "6" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "The port that an HBase RegionServer binds to.", + "display_name": "HBase RegionServer Port", + "name": "hbase_regionserver_port", + "value": "60020" + }, + { + "desc": "Number of RPC Server instances spun up on RegionServers.", + "display_name": "HBase RegionServer Handler Count", + "name": "hbase_regionserver_handler_count", + "value": "30" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of HBase RegionServer in Bytes", + "name": "hbase_regionserver_java_heapsize", + "value": "4294967296" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "When computing the overall RegionServer health, consider the host's health.", + "display_name": "RegionServer Host Health Test", + "name": "regionserver_host_health_enabled", + "value": "true" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hbase-service.json b/sahara/plugins/cdh/v5_4_0/resources/hbase-service.json new file mode 100644 index 00000000..177ec1f4 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hbase-service.json @@ -0,0 +1,524 @@ +[ + { + "desc": "The user that this service's processes should run as.", + "display_name": "System User", + "name": "process_username", + "value": "hbase" + }, + { + "desc": "

Event filters are defined in a JSON object like the following:

\n\n
\n{\n  \"defaultAction\" : (\"accept\", \"discard\"),\n  \"rules\" : [\n    {\n      \"action\" : (\"accept\", \"discard\"),\n      \"fields\" : [\n        {\n          \"name\" : \"fieldName\",\n          \"match\" : \"regex\"\n        }\n      ]\n    }\n  ]\n}\n
\n\n

\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n

\n\n

\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n

\n\n

\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n

\n\n

\nThe following is the list of fields that can be filtered for HBase events:\n

\n\n", + "display_name": "Audit Event Filter", + "name": "navigator_audit_event_filter", + "value": "{\n \"comment\" : [\n \"Default filter for HBase services.\",\n \"Discards events that affect the internal -ROOT-, .META. and _acl_ tables.\"\n ],\n \"defaultAction\" : \"accept\",\n \"rules\" : [\n {\n \"action\" : \"discard\",\n \"fields\" : [\n { \"name\" : \"tableName\", \"match\" : \"(?:-ROOT-|.META.|_acl_|hbase:meta|hbase:acl)\" }\n ]\n }\n ]\n}\n" + }, + { + "desc": "Enables the canary that checks HBase region availability by scanning a row from every region.", + "display_name": "HBase Region Health Canary", + "name": "hbase_region_health_canary_enabled", + "value": "true" + }, + { + "desc": "An alert is published if the HBase Hbck tool runs slowly.", + "display_name": "HBase Hbck Slow Run Alert Enabled", + "name": "hbase_hbck_slow_run_alert_enabled", + "value": "true" + }, + { + "desc": "Name of the HDFS service that this HBase service instance depends on", + "display_name": "HDFS Service", + "name": "hdfs_service", + "value": null + }, + { + "desc": "Period of time, in milliseconds, to pause between connection retries to ZooKeeper. Used together with ${zookeeper.retries} in an exponential backoff fashion when making queries to ZooKeeper.", + "display_name": "ZooKeeper Connection Retry Pause Duration", + "name": "zookeeper_pause", + "value": null + }, + { + "desc": "When computing the overall HBase cluster health, consider the active HBase Master's health.", + "display_name": "Active Master Health Test", + "name": "hbase_master_health_enabled", + "value": "true" + }, + { + "desc": "Size of the threadpool used for hedged reads in hdfs clients. If a read from a block is slow, a parallel 'hedged' read will be started against a different block replica. The first one to return with a result is used while the other one is cancelled. This 'hedged' read feature helps rein in the outliers. A value of zero disables the feature.", + "display_name": "HDFS Hedged Read Threadpool Size", + "name": "hbase_server_dfs_client_hedged_read_threadpool_size", + "value": "0" + }, + { + "desc": "Configures whether the Hbck poller checks HDFS or not. Checking HBase tables and regions information on HDFS can take a while.", + "display_name": "HBase Hbck Check HDFS", + "name": "hbase_hbck_poller_check_hdfs", + "value": "true" + }, + { + "desc": "Path to the directory where audit logs will be written. The directory will be created if it doesn't exist.", + "display_name": "Audit Log Directory", + "name": "audit_event_log_dir", + "value": "/var/log/hbase/audit" + }, + { + "desc": "Use this to enable Http server usage on thrift, which is mainly needed for \"doAs\" functionality.", + "display_name": "Enable HBase Thrift Http Server", + "name": "hbase_thriftserver_http", + "value": "false" + }, + { + "desc": "An alert is published if the HBase Hbck tool finds any errors with matching codes. Possible error codes: UNKNOWN, NO_META_REGION, NULL_ROOT_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META, NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED, MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE, FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS, HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION, ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE", + "display_name": "HBase Hbck Alert Error Codes", + "name": "hbase_hbck_alert_error_codes", + "value": "NO_META_REGION,NULL_ROOT_REGION" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Service Level Health Alerts", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "Timeout for all HBase RPCs in milliseconds.", + "display_name": "RPC Timeout", + "name": "hbase_rpc_timeout", + "value": "60000" + }, + { + "desc": "For advanced use only, a string to be inserted into ssl-server.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "HBase Service Advanced Configuration Snippet (Safety Valve) for ssl-server.xml", + "name": "hbase_ssl_server_safety_valve", + "value": null + }, + { + "desc": "Action to take when the audit event queue is full. Drop the event or shutdown the affected process.", + "display_name": "Audit Queue Policy", + "name": "navigator_audit_queue_policy", + "value": "DROP" + }, + { + "desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.", + "display_name": "Enable Log Event Capture", + "name": "catch_events", + "value": "true" + }, + { + "desc": "The number of times to retry connections to ZooKeeper. Used for reading and writing root region location. Used together with ${zookeeper.pause} in an exponential backoff fashion when making queries to ZooKeeper.", + "display_name": "ZooKeeper Connection Retries", + "name": "zookeeper_retries", + "value": null + }, + { + "desc": "Path to the keystore file containing the server certificate and private key used for encrypted web UIs.", + "display_name": "SSL Server Keystore File Location", + "name": "ssl_server_keystore_location", + "value": null + }, + { + "desc": "Timeout for graceful shutdown of this HBase service. Once this timeout is reached, any remaining running roles are abruptly shutdown. A value of 0 means no timeout.", + "display_name": "Graceful Shutdown Timeout", + "name": "hbase_graceful_stop_timeout", + "value": "180" + }, + { + "desc": "The health test thresholds of the overall RegionServer health. The check returns \"Concerning\" health if the percentage of \"Healthy\" RegionServers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" RegionServers falls below the critical threshold.", + "display_name": "Healthy RegionServer Monitoring Thresholds", + "name": "hbase_regionservers_healthy_thresholds", + "value": "{\"critical\":\"90.0\",\"warning\":\"95.0\"}" + }, + { + "desc": "An alert is published if the HBase region health canary detects at least this many unhealthy regions. This setting takes precedence over the hbase_canary_alert_unhealthy_region_percent_threshold config.", + "display_name": "HBase Canary Unhealthy Region Count Alert Threshold", + "name": "hbase_canary_alert_unhealthy_region_count_threshold", + "value": null + }, + { + "desc": "Set to true to cause the hosting server (Master or RegionServer) to abort if a coprocessor throws a Throwable object that is not IOException or a subclass of IOException. Setting it to true might be useful in development environments where one wants to terminate the server as soon as possible to simplify coprocessor failure analysis.", + "display_name": "HBase Coprocessor Abort on Error", + "name": "hbase_coprocessor_abort_on_error", + "value": "false" + }, + { + "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", + "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", + "name": "smon_derived_configs_safety_valve", + "value": null + }, + { + "desc": "Write buffer size in bytes. A larger buffer requires more memory on both the client and the server because the server instantiates the passed write buffer to process it but reduces the number of remote procedure calls (RPC). To estimate the amount of server memory used, multiply the value of 'hbase.client.write.buffer' by the value of 'hbase.regionserver.handler.count'.", + "display_name": "HBase Client Write Buffer", + "name": "hbase_client_write_buffer", + "value": "2097152" + }, + { + "desc": "The user the management services impersonate when connecting to HBase. If no value is specified, the HBase superuser is used.", + "display_name": "HBase User to Impersonate", + "name": "hbase_user_to_impersonate", + "value": null + }, + { + "desc": "Timeout (in ms) for the distributed log splitting manager to receive response from a worker.", + "display_name": "SplitLog Manager Timeout", + "name": "hbase_service_splitlog_manager_timeout", + "value": "120000" + }, + { + "desc": "Allow indexing of tables in HBase by Lily HBase Indexer. Note: Replication must be enabled for indexing to work.", + "display_name": "Enable Indexing", + "name": "hbase_enable_indexing", + "value": "false" + }, + { + "desc": "An alert is published if the HBase Hbck tool detects at least this many errors across all tables in this service. Some errors are not associated with a region, e.g. 'RS_CONNECT_FAILURE'. If the value is not set, alerts will not be published based on the count of errors.", + "display_name": "HBase Hbck Error Count Alert Threshold", + "name": "hbase_hbck_alert_error_count_threshold", + "value": null + }, + { + "desc": "Maximum number of errors that the HBase Hbck poller will retain through a given run", + "display_name": "HBase Hbck Poller Maximum Error Count", + "name": "hbase_hbck_poller_max_error_count", + "value": "10000" + }, + { + "desc": "Enables the HBase Hbck Poller so that Hbck reports will be available. Enabling the Hbck poller will increase the amount of memory used by the Service Monitor. Consider increasing the Service Monitor Java heap size by an additional 3KB per region. For example, for a cluster with 10,000 regions, increase the JVM heap size by approximately 30MB.", + "display_name": "HBase Hbck Poller", + "name": "hbase_hbck_poller_enabled", + "value": "false" + }, + { + "desc": "Password for the server keystore file used for encrypted web UIs.", + "display_name": "SSL Server Keystore File Password", + "name": "ssl_server_keystore_password", + "value": null + }, + { + "desc": "Ratio of Lily HBase Indexers used by each HBase RegionServer while doing replication.", + "display_name": "Replication Source Ratio", + "name": "hbase_replication_source_ratio", + "value": "1.0" + }, + { + "desc": "Maximum size of audit log file in MB before it is rolled over.", + "display_name": "Maximum Audit Log File Size", + "name": "navigator_audit_log_max_file_size", + "value": "100" + }, + { + "desc": "For advanced use only, a string to be inserted into core-site.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "HBase Service Advanced Configuration Snippet (Safety Valve) for core-site.xml", + "name": "hbase_core_site_safety_valve", + "value": null + }, + { + "desc": "The timeout before injecting a snapshot timeout error when waiting for a snapshot completion.", + "display_name": "HBase Master Snapshot Waiting Timeout", + "name": "hbase_snapshot_master_timeoutMillis", + "value": "60000" + }, + { + "desc": "The maximum amount of time the Hbase RegionServer waits for a snapshot to complete.", + "display_name": "HBase RegionServer Snapshot Timeout", + "name": "hbase_snapshot_region_timeout", + "value": "60000" + }, + { + "desc": "If this is set to \"kerberos\", HBase REST Server will authenticate its clients. HBase Proxy User Hosts and Groups should be configured to allow specific users to access HBase through REST Server.", + "display_name": "HBase REST Authentication", + "name": "hbase_restserver_security_authentication", + "value": "simple" + }, + { + "desc": "Specifies the combined maximum allowed size of a KeyValue instance. This option configures an upper boundary for a single entry saved in a storage file. This option prevents a region from splitting if the data is too large. Set this option to a fraction of the maximum region size. To disable this check, use a value of zero or less.", + "display_name": "Maximum Size of HBase Client KeyValue", + "name": "hbase_client_keyvalue_maxsize", + "value": "10485760" + }, + { + "desc": "Enable snapshots. Disabling snapshots requires deletion of all snapshots before restarting the HBase master; the HBase master will not start if snapshots are disabled and snapshots exist.", + "display_name": "Enable Snapshots", + "name": "hbase_snapshot_enabled", + "value": "true" + }, + { + "desc": "An alert is published if the HBase Hbck tool detects at least this many regions with errors across all tables in this service. If the value is not set, alerts will not be published based on the count of regions with errors.", + "display_name": "HBase Hbck Region Error Count Alert Threshold", + "name": "hbase_hbck_alert_region_error_count_threshold", + "value": null + }, + { + "desc": "Maximum number of client retries. Used as a maximum for all operations such as fetching of the root region from the root RegionServer, getting a cell's value, and starting a row update.", + "display_name": "Maximum HBase Client Retries", + "name": "hbase_client_retries_number", + "value": "35" + }, + { + "desc": "Set to true to use HBase Secure RPC Engine for remote procedure calls (RPC). This is only effective in simple authentication mode. Does not provide authentication for RPC calls, but provides user information in the audit logs. Changing this setting requires a restart of this and all dependent services and redeployment of client configurations, along with a restart of the Service Monitor management role.", + "display_name": "HBase Secure RPC Engine", + "name": "hbase_secure_rpc_engine", + "value": "false" + }, + { + "desc": "For advanced use only, a list of configuration properties that will be used by the Service Monitor instead of the current client configuration for the service.", + "display_name": "Service Monitor Client Config Overrides", + "name": "smon_client_config_overrides", + "value": "zookeeper.recovery.retry0zookeeper.recovery.retry.intervalmill3000hbase.zookeeper.recoverable.waittime1000zookeeper.session.timeout30000hbase.rpc.timeout10000hbase.client.retries.number1hbase.client.rpc.maxattempts1hbase.client.operation.timeout10000" + }, + { + "desc": "AWS access key ID required to access S3 to store remote snapshots.", + "display_name": "AWS S3 Access Key ID for Remote Snapshots", + "name": "hbase_snapshot_s3_access_key_id", + "value": null + }, + { + "desc": "Allow HBase tables to be replicated.", + "display_name": "Enable Replication", + "name": "hbase_enable_replication", + "value": "false" + }, + { + "desc": "Enable HBase authorization.", + "display_name": "HBase Secure Authorization", + "name": "hbase_security_authorization", + "value": "false" + }, + { + "desc": "Period of time, in milliseconds, to pause between searches for work. Used as a sleep interval by service threads such as a META scanner and log roller.", + "display_name": "HBase Server Thread Wake Frequency", + "name": "hbase_server_thread_wakefrequency", + "value": "10000" + }, + { + "desc": "Enable collection of audit events from the service's roles.", + "display_name": "Enable Audit Collection", + "name": "navigator_audit_enabled", + "value": "true" + }, + { + "desc": "The root znode for HBase in ZooKeeper. All of HBase's ZooKeeper files that are configured with a relative path will go under this node. By default, all of HBase's ZooKeeper file paths are configured with a relative path, so they will all go under this directory unless changed.", + "display_name": "ZooKeeper Znode Parent", + "name": "zookeeper_znode_parent", + "value": "/hbase" + }, + { + "desc": "

The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n  \"streamThreshold\": 10, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Service Triggers", + "name": "service_triggers", + "value": "[]" + }, + { + "desc": "AWS secret access key required to access S3 to store remote snapshots.", + "display_name": "AWS S3 Secret Access Key for Remote Snapshots", + "name": "hbase_snapshot_s3_secret_access_key", + "value": null + }, + { + "desc": "Number of rows to fetch when calling next on a scanner if it is not served from memory. Higher caching values enable faster scanners but require more memory and some calls of next may take longer when the cache is empty.", + "display_name": "HBase Client Scanner Caching", + "name": "hbase_client_scanner_caching", + "value": "100" + }, + { + "desc": "Tables to exclude in the HBase Region Health Canary which will scan a row from every region.", + "display_name": "HBase Region Health Canary Exclude Tables", + "name": "hbase_region_health_canary_exclude_tables", + "value": "" + }, + { + "desc": "The maximum amount of time the HBase master waits for a snapshot to complete.", + "display_name": "HBase Master Snapshot Timeout", + "name": "hbase_snapshot_master_timeout_millis", + "value": "60000" + }, + { + "desc": "AWS S3 path where remote snapshots should be stored.", + "display_name": "AWS S3 Path for Remote Snapshots", + "name": "hbase_snapshot_s3_path", + "value": null + }, + { + "desc": "Start a process to periodically check that RegionServer is alive when RegionServer is started. Note: This canary is different from the Cloudera Service Monitoring canary and is provided by the HBase service itself.", + "display_name": "Enable HBase Canary", + "name": "hbase_regionserver_enable_canary", + "value": "false" + }, + { + "desc": "Enable HDFS short circuit read. This allows a client co-located with the DataNode to read HDFS file blocks directly. This gives a performance boost to distributed clients that are aware of locality.", + "display_name": "Enable HDFS Short Circuit Read", + "name": "dfs_client_read_shortcircuit", + "value": "true" + }, + { + "desc": "Path to ZooKeeper Node holding root region location. This is written by the HBase Master and read by clients and RegionServers. If a relative path is given, the parent folder will be ${zookeeper.znode.parent}. By default, the root location is stored at /hbase/root-region-server.", + "display_name": "ZooKeeper Znode Rootserver", + "name": "zookeeper_znode_rootserver", + "value": "root-region-server" + }, + { + "desc": "When computing the overall HBase cluster health, consider the health of the backup HBase Masters.", + "display_name": "Backup Masters Health Test", + "name": "hbase_backup_masters_health_enabled", + "value": "true" + }, + { + "desc": "Password that protects the private key contained in the server keystore used for encrypted web UIs.", + "display_name": "SSL Server Keystore Key Password", + "name": "ssl_server_keystore_keypassword", + "value": null + }, + { + "desc": "An alert is published if the HBase region health canary runs slowly.", + "display_name": "HBase Region Health Canary Slow Run Alert Enabled", + "name": "hbase_region_health_canary_slow_run_alert_enabled", + "value": "true" + }, + { + "desc": "Name of the scheduler pool to use for MR jobs created during export/import of remote snapshots in AWS S3.", + "display_name": "Scheduler Pool for Remote Snapshots in AWS S3", + "name": "hbase_snapshot_s3_scheduler_pool", + "value": null + }, + { + "desc": "An alert is published if the HBase region health canary detects at least this percentage of total regions are unhealthy. This threshold is used if the explicit count is not set via the hbase_canary_alert_unhealthy_region_count_threshold config.", + "display_name": "HBase Canary Unhealthy Region Percentage Alert Threshold", + "name": "hbase_canary_alert_unhealthy_region_percent_threshold", + "value": "0.1" + }, + { + "desc": "Duration to wait before starting up a 'hedged' read.", + "display_name": "HDFS Hedged Read Delay Threshold", + "name": "hbase_server_dfs_client_hedged_read_threshold_millis", + "value": "500" + }, + { + "desc": "Enable SSL encryption for HBase web UIs.", + "display_name": "Web UI SSL Encryption Enabled", + "name": "hbase_hadoop_ssl_enabled", + "value": "false" + }, + { + "desc": "List of users or groups, who are allowed full privileges, regardless of stored ACLs, across the cluster. Only used when HBase security is enabled.", + "display_name": "HBase Superusers", + "name": "hbase_superuser", + "value": "" + }, + { + "desc": "Use this to allow proxy users on thrift gateway, which is mainly needed for \"doAs\" functionality.", + "display_name": "Enable HBase Thrift Proxy Users", + "name": "hbase_thriftserver_support_proxyuser", + "value": "false" + }, + { + "desc": "Maximum number of hlog entries to replicate in one go. If this is large, and a consumer takes a while to process the events, the HBase RPC call will time out.", + "display_name": "Replication Batch Size", + "name": "hbase_replication_source_nb_capacity", + "value": "1000" + }, + { + "desc": "ZooKeeper session timeout in milliseconds. HBase passes this to the ZooKeeper quorum as the suggested maximum time for a session. See http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions The client sends a requested timeout, the server responds with the timeout that it can give the client.", + "display_name": "ZooKeeper Session Timeout", + "name": "zookeeper_session_timeout", + "value": "60000" + }, + { + "desc": "Comma-delimited list of hosts where you want to allow the HBase user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that does not correspond to a host name, such as '_no_host'. Note: This property is used only if HBase REST/Thrift Server Authentication is enabled.", + "display_name": "HBase Proxy User Hosts", + "name": "hbase_proxy_user_hosts_list", + "value": "*" + }, + { + "desc": "Comma-delimited list of groups that you want to allow the HBase user to impersonate. The default '*' allows all groups. To disable entirely, use a string that does not correspond to a group name, such as '_no_group_'. Note: This property is used only if HBase REST/Thrift Server Authentication is enabled.", + "display_name": "HBase Proxy User Groups", + "name": "hbase_proxy_user_groups_list", + "value": "*" + }, + { + "desc": "

\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n

\n\n

\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n

\n\n

Event trackers are defined in a JSON object like the following:

\n\n
\n{\n  \"timeToLive\" : [integer],\n  \"fields\" : [\n    {\n      \"type\" : [string],\n      \"name\" : [string]\n    }\n  ]\n}\n
\n\n

\nWhere:\n

\n\n\n\n

\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n

\n\n\n\n

\nThe following is the list of fields that can used to compare HBase events:\n

\n\n", + "display_name": "Audit Event Tracker", + "name": "navigator_event_tracker", + "value": "{\n \"comment\" : [\n \"Default event tracker for HBase services.\",\n \"Defines equality by comparing username, action, table name, family \",\n \"and qualifier of the events.\"\n ],\n \"timeToLive\" : 60000,\n \"fields\" : [\n { \"type\": \"value\", \"name\" : \"tableName\" },\n { \"type\": \"value\", \"name\" : \"family\" },\n { \"type\": \"value\", \"name\" : \"qualifier\" },\n { \"type\": \"value\", \"name\" : \"operation\" },\n { \"type\": \"username\", \"name\" : \"username\" }\n ]\n}\n" + }, + { + "desc": "The tolerance window that will be used in HBase service tests that depend on detection of the active HBase Master.", + "display_name": "HBase Active Master Detection Window", + "name": "hbase_active_master_detecton_window", + "value": "3" + }, + { + "desc": "For advanced use only, a string to be inserted into hbase-site.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "HBase Service Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", + "name": "hbase_service_config_safety_valve", + "value": null + }, + { + "desc": "Maximum number of rolled over audit logs to retain. The logs will not be deleted if they contain audit events that have not yet been propagated to Audit Server.", + "display_name": "Number of Audit Logs to Retain", + "name": "navigator_audit_log_max_backup_index", + "value": "10" + }, + { + "desc": "Choose the authentication mechanism used by HBase.", + "display_name": "HBase Secure Authentication", + "name": "hbase_security_authentication", + "value": "simple" + }, + { + "desc": "A general client pause time value. Used mostly as a time period to wait before retrying operations such as a failed get or region lookup.", + "display_name": "HBase Client Pause", + "name": "hbase_client_pause", + "value": "100" + }, + { + "desc": "Name of the ZooKeeper service that this HBase service instance depends on.", + "display_name": "ZooKeeper Service", + "name": "zookeeper_service", + "value": null + }, + { + "desc": "If this is set, HBase Thrift Server authenticates its clients. HBase Proxy User Hosts and Groups should be configured to allow specific users to access HBase through Thrift Server.", + "display_name": "HBase Thrift Authentication", + "name": "hbase_thriftserver_security_authentication", + "value": "none" + }, + { + "desc": "Kerberos principal used by all roles of this service.", + "display_name": "Kerberos Principal", + "name": "kerberos_princ_name", + "value": "hbase" + }, + { + "desc": "The group that this service's processes should run as.", + "display_name": "System Group", + "name": "process_groupname", + "value": "hbase" + }, + { + "desc": "The HDFS directory shared by HBase RegionServers.", + "display_name": "HDFS Root Directory", + "name": "hdfs_rootdir", + "value": "/hbase" + }, + { + "desc": "The frequency in which the log4j event publication appender will retry sending undelivered log events to the Event server, in seconds", + "display_name": "Log Event Retry Frequency", + "name": "log_event_retry_frequency", + "value": "30" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", + "display_name": "HBase Service Environment Advanced Configuration Snippet (Safety Valve)", + "name": "hbase_service_env_safety_valve", + "value": null + }, + { + "desc": "For advanced use only, a string to be inserted into the client configuration for navigator.client.properties.", + "display_name": "HBASE Client Advanced Configuration Snippet (Safety Valve) for navigator.client.properties", + "name": "navigator_client_config_safety_valve", + "value": null + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-balancer.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-balancer.json new file mode 100644 index 00000000..21501797 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hdfs-balancer.json @@ -0,0 +1,44 @@ +[ + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for Balancer", + "name": "balancer_java_opts", + "value": "" + }, + { + "desc": "

This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.

Each rule has some or all of the following fields:


Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}

This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.

", + "display_name": "Rules to Extract Events from Log Files", + "name": "log_event_whitelist", + "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" + }, + { + "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", + "display_name": "Balancer Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", + "name": "balancer_config_safety_valve", + "value": null + }, + { + "desc": "The policy that should be used to rebalance HDFS storage. The default DataNode policy balances the storage at the DataNode level. This is similar to the balancing policy from prior releases. The BlockPool policy balances the storage at the block pool level as well as at the DataNode level. The BlockPool policy is relevant only to a Federated HDFS service.", + "display_name": "Rebalancing Policy", + "name": "rebalancing_policy", + "value": "DataNode" + }, + { + "desc": "The percentage deviation from average utilization, after which a node will be rebalanced. (for example, '10.0' for 10%)", + "display_name": "Rebalancing Threshold", + "name": "rebalancer_threshold", + "value": "10.0" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of Balancer in Bytes", + "name": "balancer_java_heapsize", + "value": "1073741824" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-datanode.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-datanode.json new file mode 100644 index 00000000..7a8d8bc9 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hdfs-datanode.json @@ -0,0 +1,440 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "

This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.

Each rule has some or all of the following fields:


Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}

This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.

", + "display_name": "Rules to Extract Events from Log Files", + "name": "log_event_whitelist", + "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 5, \"content\":\"Datanode registration failed\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Got a command from standby NN - ignoring command:.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" + }, + { + "desc": "Specifies the maximum number of threads to use for transferring data in and out of the DataNode.", + "display_name": "Maximum Number of Transfer Threads", + "name": "dfs_datanode_max_xcievers", + "value": "4096" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "Port for DataNode's XCeiver Protocol. Combined with the DataNode's hostname to build its address.", + "display_name": "DataNode Transceiver Port", + "name": "dfs_datanode_port", + "value": "50010" + }, + { + "desc": "Comma-separated list of DataNode plug-ins to be activated. If one plug-in cannot be loaded, all the plug-ins are ignored.", + "display_name": "DateNode Plugins", + "name": "dfs_datanode_plugins_list", + "value": "" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "In some workloads, the data read from HDFS is known to be significantly large enough that it is unlikely to be useful to cache it in the operating system buffer cache. In this case, the DataNode may be configured to automatically purge all data from the buffer cache after it is delivered to the client. This may improve performance for some workloads by freeing buffer cache spare usage for more cacheable data. This behavior will always be disabled for workloads that read only short sections of a block (e.g HBase random-IO workloads). This property is supported in CDH3u3 or later deployments.", + "display_name": "Enable purging cache after reads", + "name": "dfs_datanode_drop_cache_behind_reads", + "value": "false" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Comma-delimited list of directories on the local file system where the DataNode stores HDFS block data. Typical values are /data/N/dfs/dn for N = 1, 2, 3... These directories should be mounted using the noatime option and the disks should be configured using JBOD. RAID is not recommended. Warning: be very careful when modifying this property. Removing or changing entries can result in data loss. If you want to hot swap drives in CDH 5.4+, override the value of this property for the specific DataNode role instance whose drive is to be hot-swapped; do not modify the property value in the role group. See Configuring Hot Swap for DataNodes for more important information and caveats.", + "display_name": "DataNode Data Directory", + "name": "dfs_data_dir_list", + "value": null + }, + { + "desc": "The health test thresholds for the weighted average extra time the pause monitor spent paused. Specified as a percentage of elapsed wall clock time.", + "display_name": "Pause Duration Thresholds", + "name": "datanode_pause_duration_thresholds", + "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" + }, + { + "desc": "The number of volumes that are allowed to fail before a DataNode stops offering service. By default, any volume failure will cause a DataNode to shutdown.", + "display_name": "DataNode Failed Volumes Tolerated", + "name": "dfs_datanode_failed_volumes_tolerated", + "value": "0" + }, + { + "desc": "In some workloads, the data written to HDFS is known to be significantly large enough that it is unlikely to be useful to cache it in the operating system buffer cache. In this case, the DataNode may be configured to automatically purge all data from the buffer cache after it is written to disk. This may improve performance for some workloads by freeing buffer cache spare usage for more cacheable data. This property is supported in CDH3u3 or later deployments.", + "display_name": "Enable purging cache after writes", + "name": "dfs_datanode_drop_cache_behind_writes", + "value": "false" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "If enabled, the DataNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.", + "display_name": "Bind DataNode to Wildcard Address", + "name": "dfs_datanode_bind_wildcard", + "value": "false" + }, + { + "desc": "The number of server threads for the DataNode.", + "display_name": "Handler Count", + "name": "dfs_datanode_handler_count", + "value": "3" + }, + { + "desc": "When computing the overall DataNode health, consider the host's health.", + "display_name": "DataNode Host Health Test", + "name": "datanode_host_health_enabled", + "value": "true" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "true" + }, + { + "desc": "The maximum number of rolled log files to keep for DataNode logs. Typically used by log4j or logback.", + "display_name": "DataNode Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "DataNode Environment Advanced Configuration Snippet (Safety Valve)", + "name": "DATANODE_role_env_safety_valve", + "value": null + }, + { + "desc": "Timeout in seconds for the Hue Thrift server running on each DataNode", + "display_name": "Hue Thrift Server Timeout", + "name": "dfs_thrift_timeout", + "value": "60" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "Whether DataNodes should use DataNode hostnames when connecting to DataNodes for data transfer. This property is supported in CDH3u4 or later deployments.", + "display_name": "Use DataNode Hostname", + "name": "dfs_datanode_use_datanode_hostname", + "value": "false" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for DataNode", + "name": "datanode_java_opts", + "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "datanode_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The health test thresholds on the duration of the metrics request to the web server.", + "display_name": "Web Metric Collection Duration", + "name": "datanode_web_metric_collection_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" + }, + { + "desc": "Enables the health test that the DataNode's process state is consistent with the role configuration", + "display_name": "DataNode Process Health Test", + "name": "datanode_scm_health_enabled", + "value": "true" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", + "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", + "name": "hadoop_metrics2_safety_valve", + "value": null + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "DataNode Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "DataNode Policy for picking which volume should get a new block. The Available Space policy is only available starting with CDH 4.3.", + "display_name": "DataNode Volume Choosing Policy", + "name": "dfs_datanode_volume_choosing_policy", + "value": "org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Port for the DataNode HTTP web UI. Combined with the DataNode's hostname to build its HTTP address.", + "display_name": "DataNode HTTP Web UI Port", + "name": "dfs_datanode_http_port", + "value": "50075" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "Directory where DataNode will place its log files.", + "display_name": "DataNode Log Directory", + "name": "datanode_log_dir", + "value": "/var/log/hadoop-hdfs" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "false" + }, + { + "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", + "display_name": "DataNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", + "name": "datanode_config_safety_valve", + "value": null + }, + { + "desc": "While reading block files, the DataNode can use the posix_fadvise system call to explicitly page data into the operating system buffer cache ahead of the current reader's position. This can improve performance especially when disks are highly contended. This configuration specifies the number of bytes ahead of the current read position which the DataNode will attempt to read ahead. A value of 0 disables this feature. This property is supported in CDH3u3 or later deployments.", + "display_name": "Number of read ahead bytes", + "name": "dfs_datanode_readahead_bytes", + "value": "4194304" + }, + { + "desc": "The maximum size, in megabytes, per log file for DataNode logs. Typically used by log4j or logback.", + "display_name": "DataNode Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "The maximum amount of memory a DataNode may use to cache data blocks in memory. Setting it to zero will disable caching.", + "display_name": "Maximum Memory Used for Caching", + "name": "dfs_datanode_max_locked_memory", + "value": "4294967296" + }, + { + "desc": "Only used when the DataNode Volume Choosing Policy is set to Available Space. Controls how much DataNode volumes are allowed to differ in terms of bytes of free disk space before they are considered imbalanced. If the free space of all the volumes are within this range of each other, the volumes will be considered balanced and block assignments will be done on a pure round robin basis.", + "display_name": "Available Space Policy Balanced Threshold", + "name": "dfs_datanode_available_space_balanced_threshold", + "value": "10737418240" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds of free space in a DataNode. Specified as a percentage of the capacity on the DataNode.", + "display_name": "DataNode Free Space Monitoring Thresholds", + "name": "datanode_free_space_thresholds", + "value": "{\"critical\":\"10.0\",\"warning\":\"20.0\"}" + }, + { + "desc": "Port for the various DataNode Protocols. Combined with the DataNode's hostname to build its IPC port address.", + "display_name": "DataNode Protocol Port", + "name": "dfs_datanode_ipc_port", + "value": "50020" + }, + { + "desc": "The period to review when computing the moving average of extra time the pause monitor spent paused.", + "display_name": "Pause Duration Monitoring Period", + "name": "datanode_pause_duration_window", + "value": "5" + }, + { + "desc": "Minimum number of running threads for the Hue Thrift server running on each DataNode", + "display_name": "Hue Thrift Server Min Threadcount", + "name": "dfs_thrift_threads_min", + "value": "10" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "If this configuration is enabled, the DataNode will instruct the operating system to enqueue all written data to the disk immediately after it is written. This differs from the usual OS policy which may wait for up to 30 seconds before triggering writeback. This may improve performance for some workloads by smoothing the IO profile for data written to disk. This property is supported in CDH3u3 or later deployments.", + "display_name": "Enable immediate enqueuing of data to disk after writes", + "name": "dfs_datanode_sync_behind_writes", + "value": "false" + }, + { + "desc": "The health test thresholds of the number of blocks on a DataNode", + "display_name": "DataNode Block Count Thresholds", + "name": "datanode_block_count_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"500000.0\"}" + }, + { + "desc": "Permissions for the directories on the local file system where the DataNode stores its blocks. The permissions must be octal. 755 and 700 are typical values.", + "display_name": "DataNode Data Directory Permissions", + "name": "dfs_datanode_data_dir_perm", + "value": "700" + }, + { + "desc": "Reserved space in bytes per volume for non Distributed File System (DFS) use.", + "display_name": "Reserved Space for Non DFS Use", + "name": "dfs_datanode_du_reserved", + "value": "10737418240" + }, + { + "desc": "The amount of time to wait for the DataNode to fully start up and connect to the NameNode before enforcing the connectivity check.", + "display_name": "DataNode Connectivity Tolerance at Startup", + "name": "datanode_connectivity_tolerance", + "value": "180" + }, + { + "desc": "The base port where the secure DataNode web UI listens. Combined with the DataNode's hostname to build its secure web UI address.", + "display_name": "Secure DataNode Web UI Port (SSL)", + "name": "dfs_datanode_https_port", + "value": "50475" + }, + { + "desc": "Maximum amount of bandwidth that each DataNode can use for balancing. Specified in bytes per second.", + "display_name": "DataNode Balancing Bandwidth", + "name": "dfs_balance_bandwidthPerSec", + "value": "10485760" + }, + { + "desc": "Maximum number of running threads for the Hue Thrift server running on each DataNode", + "display_name": "Hue Thrift Server Max Threadcount", + "name": "dfs_thrift_threads_max", + "value": "20" + }, + { + "desc": "The health test thresholds of transceivers usage in a DataNode. Specified as a percentage of the total configured number of transceivers.", + "display_name": "DataNode Transceivers Usage Thresholds", + "name": "datanode_transceivers_usage_thresholds", + "value": "{\"critical\":\"95.0\",\"warning\":\"75.0\"}" + }, + { + "desc": "Only used when the DataNode Volume Choosing Policy is set to Available Space. Controls what percentage of new block allocations will be sent to volumes with more available disk space than others. This setting should be in the range 0.0 - 1.0, though in practice 0.5 - 1.0, since there should be no reason to prefer that volumes with less available disk space receive more block allocations.", + "display_name": "Available Space Policy Balanced Preference", + "name": "dfs_datanode_available_space_balanced_preference", + "value": "0.75" + }, + { + "desc": "The health test thresholds of failed volumes in a DataNode.", + "display_name": "DataNode Volume Failures Thresholds", + "name": "datanode_volume_failures_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "The minimum log level for DataNode logs", + "display_name": "DataNode Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", + "display_name": "Web Metric Collection", + "name": "datanode_web_metric_collection_enabled", + "value": "true" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of DataNode in Bytes", + "name": "datanode_java_heapsize", + "value": "1073741824" + }, + { + "desc": "Enables the health test that verifies the DataNode is connected to the NameNode", + "display_name": "DataNode Connectivity Health Test", + "name": "datanode_connectivity_health_enabled", + "value": "true" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-failovercontroller.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-failovercontroller.json new file mode 100644 index 00000000..42671459 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hdfs-failovercontroller.json @@ -0,0 +1,230 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Directory where Failover Controller will place its log files.", + "display_name": "Failover Controller Log Directory", + "name": "failover_controller_log_dir", + "value": "/var/log/hadoop-hdfs" + }, + { + "desc": "

This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.

Each rule has some or all of the following fields:


Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}

This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.

", + "display_name": "Rules to Extract Events from Log Files", + "name": "log_event_whitelist", + "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" + }, + { + "desc": "The RPC timeout for the HA health monitor.", + "display_name": "HA Health Monitor RPC Timeout", + "name": "ha_health_monitor_rpc_timeout_ms", + "value": "45000" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", + "display_name": "Failover Controller Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", + "name": "fc_config_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "The maximum number of rolled log files to keep for Failover Controller logs. Typically used by log4j or logback.", + "display_name": "Failover Controller Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of Failover Controller in Bytes", + "name": "failover_controller_java_heapsize", + "value": "268435456" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "Failover Controller Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "Failover Controller Environment Advanced Configuration Snippet (Safety Valve)", + "name": "FAILOVERCONTROLLER_role_env_safety_valve", + "value": null + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "When computing the overall Failover Controller health, consider the host's health.", + "display_name": "Failover Controller Host Health Test", + "name": "failovercontroller_host_health_enabled", + "value": "true" + }, + { + "desc": "The maximum size, in megabytes, per log file for Failover Controller logs. Typically used by log4j or logback.", + "display_name": "Failover Controller Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "failovercontroller_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "Enables the health test that the Failover Controller's process state is consistent with the role configuration", + "display_name": "Failover Controller Process Health Test", + "name": "failovercontroller_scm_health_enabled", + "value": "true" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for Failover Controller", + "name": "failover_controller_java_opts", + "value": "" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "The minimum log level for Failover Controller logs", + "display_name": "Failover Controller Logging Threshold", + "name": "log_threshold", + "value": "INFO" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-gateway.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-gateway.json new file mode 100644 index 00000000..367af20f --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hdfs-gateway.json @@ -0,0 +1,68 @@ +[ + { + "desc": "For advanced use only, a string to be inserted into the client configuration for hdfs-site.xml.", + "display_name": "HDFS Client Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", + "name": "hdfs_client_config_safety_valve", + "value": null + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "Gateway Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "The directory where the client configs will be deployed", + "display_name": "Deploy Directory", + "name": "client_config_root_dir", + "value": "/etc/hadoop" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the client configuration for hadoop-env.sh", + "display_name": "HDFS Client Environment Advanced Configuration Snippet for hadoop-env.sh (Safety Valve)", + "name": "hdfs_client_env_safety_valve", + "value": null + }, + { + "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", + "display_name": "Alternatives Priority", + "name": "client_config_priority", + "value": "90" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Maximum size in bytes for the Java process heap memory. Passed to Java -Xmx.", + "display_name": "Client Java Heap Size in Bytes", + "name": "hdfs_client_java_heapsize", + "value": "268435456" + }, + { + "desc": "Enable HDFS short circuit read. This allows a client co-located with the DataNode to read HDFS file blocks directly. This gives a performance boost to distributed clients that are aware of locality.", + "display_name": "Enable HDFS Short Circuit Read", + "name": "dfs_client_read_shortcircuit", + "value": "false" + }, + { + "desc": "These are Java command line arguments. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Client Java Configuration Options", + "name": "hbase_client_java_opts", + "value": "-Djava.net.preferIPv4Stack=true" + }, + { + "desc": "Move deleted files to the trash so that they can be recovered if necessary. This client side configuration takes effect only if the HDFS service-wide trash is disabled (NameNode Filesystem Trash Interval set to 0) and is ignored otherwise. The trash is not automatically emptied when enabled with this configuration.", + "display_name": "Use Trash", + "name": "dfs_client_use_trash", + "value": "false" + }, + { + "desc": "The minimum log level for Gateway logs", + "display_name": "Gateway Logging Threshold", + "name": "log_threshold", + "value": "INFO" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-httpfs.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-httpfs.json new file mode 100644 index 00000000..6a523844 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hdfs-httpfs.json @@ -0,0 +1,290 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Kerberos principal used by the HttpFS roles.", + "display_name": "Role-Specific Kerberos Principal", + "name": "kerberos_role_princ_name", + "value": "httpfs" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "httpfs_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "The user that the HttpFS server process should run as.", + "display_name": "System User", + "name": "httpfs_process_username", + "value": "httpfs" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The group that the HttpFS server process should run as.", + "display_name": "System Group", + "name": "httpfs_process_groupname", + "value": "httpfs" + }, + { + "desc": "When computing the overall HttpFS health, consider the host's health.", + "display_name": "HttpFS Host Health Test", + "name": "httpfs_host_health_enabled", + "value": "true" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "The maximum number of rolled log files to keep for HttpFS logs. Typically used by log4j or logback.", + "display_name": "HttpFS Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Enables the health test that the HttpFS's process state is consistent with the role configuration", + "display_name": "HttpFS Process Health Test", + "name": "httpfs_scm_health_enabled", + "value": "true" + }, + { + "desc": "Password of the keystore used by HttpFS role for SSL.", + "display_name": "HttpFS Keystore Password", + "name": "httpfs_https_keystore_password", + "value": null + }, + { + "desc": "The password for the HttpFS TLS/SSL Certificate Trust Store File. Note that this password is not required to access the trust store: this field can be left blank. This password provides optional integrity checking of the file. The contents of trust stores are certificates, and certificates are public information.", + "display_name": "HttpFS TLS/SSL Certificate Trust Store Password", + "name": "httpfs_https_truststore_password", + "value": null + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "Directory where HttpFS will place its log files.", + "display_name": "HttpFS Log Directory", + "name": "httpfs_log_dir", + "value": "/var/log/hadoop-httpfs" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "HttpFS Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "For advanced use only, a string to be inserted into httpfs-site.xml for this role only.", + "display_name": "HttpFS Advanced Configuration Snippet (Safety Valve) for httpfs-site.xml", + "name": "httpfs_config_safety_valve", + "value": null + }, + { + "desc": "The maximum size, in megabytes, per log file for HttpFS logs. Typically used by log4j or logback.", + "display_name": "HttpFS Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for HttpFS", + "name": "httpfs_java_opts", + "value": "" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "HttpFS Environment Advanced Configuration Snippet (Safety Valve)", + "name": "HTTPFS_role_env_safety_valve", + "value": null + }, + { + "desc": "The secret to use for signing client authentication tokens.", + "display_name": "Signature Secret", + "name": "hdfs_httpfs_signature_secret", + "value": "hadoop httpfs secret" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Location of the keystore file used by HttpFS role for SSL.", + "display_name": "HttpFS Keystore File", + "name": "httpfs_https_keystore_file", + "value": "/var/run/hadoop-httpfs/.keystore" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of HttpFS in Bytes", + "name": "httpfs_java_heapsize", + "value": "268435456" + }, + { + "desc": "The port for the administration interface.", + "display_name": "Administration Port", + "name": "hdfs_httpfs_admin_port", + "value": "14001" + }, + { + "desc": "The location on disk of the trust store, in .jks format, used to confirm the authenticity of TLS/SSL servers that HttpFS might connect to. This is used when HttpFS is the client in a TLS/SSL connection. This trust store must contain the certificate(s) used to sign the service(s) being connected to. If this parameter is not provided, the default list of well-known certificate authorities is used instead.", + "display_name": "HttpFS TLS/SSL Certificate Trust Store File", + "name": "httpfs_https_truststore_file", + "value": null + }, + { + "desc": "Address of the load balancer used for HttpFS roles. Should be specified in host:port format. Note: Changing this property will regenerate Kerberos keytabs for all HttpFS roles.", + "display_name": "HttpFS Load Balancer", + "name": "httpfs_load_balancer", + "value": null + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "The port where the REST interface to HDFS is available. The REST interface is served over HTTPS if SSL is enabled for HttpFS, or over HTTP otherwise.", + "display_name": "REST Port", + "name": "hdfs_httpfs_http_port", + "value": "14000" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "The minimum log level for HttpFS logs", + "display_name": "HttpFS Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "Use SSL for HttpFS.", + "display_name": "Use SSL", + "name": "httpfs_use_ssl", + "value": "false" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-journalnode.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-journalnode.json new file mode 100644 index 00000000..5782c8d6 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hdfs-journalnode.json @@ -0,0 +1,308 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "

This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.

Each rule has some or all of the following fields:


Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}

This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.

", + "display_name": "Rules to Extract Events from Log Files", + "name": "log_event_whitelist", + "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of JournalNode in Bytes", + "name": "journalNode_java_heapsize", + "value": "268435456" + }, + { + "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", + "display_name": "Web Metric Collection", + "name": "journalnode_web_metric_collection_enabled", + "value": "true" + }, + { + "desc": "The health test thresholds on the duration of the metrics request to the web server.", + "display_name": "Web Metric Collection Duration", + "name": "journalnode_web_metric_collection_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "Enables the health test that the JournalNode's process state is consistent with the role configuration", + "display_name": "JournalNode Process Health Test", + "name": "journalnode_scm_health_enabled", + "value": "true" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "true" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "Directory on the local file system where NameNode edits are written.", + "display_name": "JournalNode Edits Directory", + "name": "dfs_journalnode_edits_dir", + "value": null + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Directory on the local file system where NameNode edits are written..", + "display_name": "JournalNode Edits Directory Free Space Monitoring Absolute Thresholds", + "name": "journalnode_edits_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Enables the health check that verifies the active NameNode's sync status to the JournalNode", + "display_name": "Active NameNode Sync Status Health Check", + "name": "journalnode_sync_status_enabled", + "value": "true" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "When computing the overall JournalNode health, consider the host's health.", + "display_name": "JournalNode Host Health Test", + "name": "journalnode_host_health_enabled", + "value": "true" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "The amount of time at JournalNode startup allowed for the active NameNode to get in sync with the JournalNode.", + "display_name": "Active NameNode Sync Status Startup Tolerance", + "name": "journalnode_sync_status_startup_tolerance", + "value": "180" + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "The base port where the secure JournalNode web UI listens. Combined with the JournalNode's hostname to build its secure web UI address.", + "display_name": "Secure JournalNode Web UI Port (SSL)", + "name": "dfs_journalnode_https_port", + "value": "8481" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "journalnode_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "JournalNode Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The health test thresholds for JournalNode fsync latency.", + "display_name": "JournalNode Fsync Latency Thresholds", + "name": "journalnode_fsync_latency_thresholds", + "value": "{\"critical\":\"3000.0\",\"warning\":\"1000.0\"}" + }, + { + "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", + "display_name": "JournalNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", + "name": "jn_config_safety_valve", + "value": null + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", + "display_name": "Garbage Collection Duration Thresholds", + "name": "journalnode_gc_duration_thresholds", + "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" + }, + { + "desc": "Port for the JournalNode's RPC. Combined with the JournalNode's hostname to build its RPC address.", + "display_name": "JournalNode RPC Port", + "name": "dfs_journalnode_rpc_port", + "value": "8485" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The maximum size, in megabytes, per log file for JournalNode logs. Typically used by log4j or logback.", + "display_name": "JournalNode Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "If enabled, the JournalNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.", + "display_name": "Bind JournalNode to Wildcard Address", + "name": "journalnode_bind_wildcard", + "value": "false" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "The period to review when computing the moving average of garbage collection time.", + "display_name": "Garbage Collection Duration Monitoring Period", + "name": "journalnode_gc_duration_window", + "value": "5" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Directory on the local file system where NameNode edits are written.. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Directory on the local file system where NameNode edits are written. Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "JournalNode Edits Directory Free Space Monitoring Percentage Thresholds", + "name": "journalnode_edits_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Directory where JournalNode will place its log files.", + "display_name": "JournalNode Log Directory", + "name": "journalnode_log_dir", + "value": "/var/log/hadoop-hdfs" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "JournalNode Environment Advanced Configuration Snippet (Safety Valve)", + "name": "JOURNALNODE_role_env_safety_valve", + "value": null + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for JournalNode", + "name": "journalNode_java_opts", + "value": "" + }, + { + "desc": "The maximum number of rolled log files to keep for JournalNode logs. Typically used by log4j or logback.", + "display_name": "JournalNode Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Port for the JournalNode HTTP web UI. Combined with the JournalNode hostname to build its HTTP address.", + "display_name": "JournalNode HTTP Port", + "name": "dfs_journalnode_http_port", + "value": "8480" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "The minimum log level for JournalNode logs", + "display_name": "JournalNode Logging Threshold", + "name": "log_threshold", + "value": "INFO" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-namenode.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-namenode.json new file mode 100644 index 00000000..6d02ba2b --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hdfs-namenode.json @@ -0,0 +1,548 @@ +[ + { + "desc": "Name of the journal located on each JournalNode filesystem.", + "display_name": "Quorum-based Storage Journal name", + "name": "dfs_namenode_quorum_journal_name", + "value": null + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "Indicate whether or not to avoid reading from stale DataNodes for which heartbeat messages have not been received by the NameNode for more than Stale DataNode Time Interval. Stale DataNodes are moved to the end of the node list returned for reading. See dfs.namenode.avoid.write.stale.datanode for a similar setting for writes.", + "display_name": "Avoid Reading Stale DataNode", + "name": "dfs_namenode_avoid_read_stale_datanode", + "value": "false" + }, + { + "desc": "The base port where the DFS NameNode web UI listens. If the port number is 0, then the server starts on a free port. Combined with the NameNode's hostname to build its HTTP address.", + "display_name": "NameNode Web UI Port", + "name": "dfs_http_port", + "value": "50070" + }, + { + "desc": "Enables the health test that the NameNode is not in safemode", + "display_name": "NameNode Safemode Health Test", + "name": "namenode_safe_mode_enabled", + "value": "true" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "The health test thresholds on the duration of the metrics request to the web server.", + "display_name": "Web Metric Collection Duration", + "name": "namenode_web_metric_collection_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" + }, + { + "desc": "The health check thresholds of the NameNode's RPC latency.", + "display_name": "NameNode RPC Latency Thresholds", + "name": "namenode_rpc_latency_thresholds", + "value": "{\"critical\":\"5000.0\",\"warning\":\"1000.0\"}" + }, + { + "desc": "The period to review when computing the moving average of extra time the pause monitor spent paused.", + "display_name": "Pause Duration Monitoring Period", + "name": "namenode_pause_duration_window", + "value": "5" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "The maximum number of outgoing replication threads a node can have at one time. This limit is waived for the highest priority replications. Configure dfs.namenode.replication.max-streams-hard-limit to set the absolute limit, including the highest-priority replications.", + "display_name": "Maximum Number of Replication Threads on a DataNode", + "name": "dfs_namenode_replication_max_streams", + "value": "20" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "NameNode Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "When the ratio of number stale DataNodes to total DataNodes marked is greater than this ratio, permit writing to stale nodes to prevent causing hotspots.", + "display_name": "Write Stale DataNode Ratio", + "name": "dfs_namenode_write_stale_datanode_ratio", + "value": "0.5" + }, + { + "desc": "The health test thresholds for the weighted average extra time the pause monitor spent paused. Specified as a percentage of elapsed wall clock time.", + "display_name": "Pause Duration Thresholds", + "name": "namenode_pause_duration_thresholds", + "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "Enables the health test that the NameNode's process state is consistent with the role configuration", + "display_name": "NameNode Process Health Test", + "name": "namenode_scm_health_enabled", + "value": "true" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "Enables the health test of the rolling metadata upgrade status of the NameNode. This covers rolling metadata upgrades. Nonrolling metadata upgrades are covered in a separate health test.", + "display_name": "HDFS Rolling Metadata Upgrade Status Health Test", + "name": "namenode_rolling_upgrade_status_enabled", + "value": "true" + }, + { + "desc": "Mountpoints that are mapped to this NameNode's Nameservice.", + "display_name": "Mountpoints", + "name": "nameservice_mountpoints", + "value": "/" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "namenode_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "

This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.

Each rule has some or all of the following fields:


Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}

This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.

", + "display_name": "Rules to Extract Events from Log Files", + "name": "log_event_whitelist", + "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"},\n {\"alert\": false, \"rate\": 1, \"threshold\":\"INFO\", \"content\":\"Triggering checkpoint.*\"}\n ]\n}\n" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of Namenode in Bytes", + "name": "namenode_java_heapsize", + "value": "1073741824" + }, + { + "desc": "The maximum size, in megabytes, per log file for NameNode logs. Typically used by log4j or logback.", + "display_name": "NameNode Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "Determines where on the local file system the NameNode should store the name table (fsimage). For redundancy, enter a comma-delimited list of directories to replicate the name table in all of the directories. Typical values are /data/N/dfs/nn where N=1..3.", + "display_name": "NameNode Data Directories", + "name": "dfs_name_dir_list", + "value": null + }, + { + "desc": "Default time interval for marking a DataNode as \"stale\". If the NameNode has not received heartbeat messages from a DataNode for more than this time interval, the DataNode is marked and treated as \"stale\" by default.", + "display_name": "Stale DataNode Time Interval", + "name": "dfs_namenode_stale_datanode_interval", + "value": "30000" + }, + { + "desc": "The port where the NameNode runs the HDFS protocol. Combined with the NameNode's hostname to build its address.", + "display_name": "NameNode Port", + "name": "namenode_port", + "value": "8020" + }, + { + "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", + "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", + "name": "hadoop_metrics2_safety_valve", + "value": null + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "The period to review when computing the moving average of the NameNode's RPC latency.", + "display_name": "NameNode RPC Latency Monitoring Window", + "name": "namenode_rpc_latency_window", + "value": "5" + }, + { + "desc": "Optional port for the service-rpc address which can be used by HDFS daemons instead of sharing the RPC address used by the clients.", + "display_name": "NameNode Service RPC Port", + "name": "dfs_namenode_servicerpc_address", + "value": null + }, + { + "desc": "The health check thresholds for the number of out-of-sync JournalNodes for this NameNode.", + "display_name": "NameNode Out-Of-Sync JournalNodes Thresholds", + "name": "namenode_out_of_sync_journal_nodes_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Determines where on the local file system the NameNode should store the name table (fsimage). For redundancy, enter a comma-delimited list of directories to replicate the name table in all of the directories. Typical values are /data/N/dfs/nn where N=1..3..", + "display_name": "NameNode Data Directories Free Space Monitoring Absolute Thresholds", + "name": "namenode_data_directories_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The absolute maximum number of outgoing replication threads a given node can have at one time. The regular limit (dfs.namenode.replication.max-streams) is waived for highest-priority block replications. Highest replication priority is for blocks that are at a very high risk of loss if the disk or server on which they remain fails. These are usually blocks with only one copy, or blocks with zero live copies but a copy in a node being decommissioned. dfs.namenode.replication.max-streams-hard-limit provides a limit on the total number of outgoing replication threads, including threads of all priorities.", + "display_name": "Hard Limit on the Number of Replication Threads on a Datanode", + "name": "dfs_namenode_replication_max_streams_hard_limit", + "value": "40" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "NameNode Environment Advanced Configuration Snippet (Safety Valve)", + "name": "NAMENODE_role_env_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Directory where NameNode will place its log files.", + "display_name": "NameNode Log Directory", + "name": "namenode_log_dir", + "value": "/var/log/hadoop-hdfs" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Enables the health test of the metadata upgrade status of the NameNode. This covers nonrolling metadata upgrades. Rolling metadata upgrades are covered in a separate health test.", + "display_name": "HDFS Metadata Upgrade Status Health Test", + "name": "namenode_upgrade_status_enabled", + "value": "true" + }, + { + "desc": "Nameservice of this NameNode. The Nameservice represents the interface to this NameNode and its High Availability partner. The Nameservice also represents the namespace associated with a federated NameNode.", + "display_name": "NameNode Nameservice", + "name": "dfs_federation_namenode_nameservice", + "value": null + }, + { + "desc": "The minimum log level for NameNode logs", + "display_name": "NameNode Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "Specifies the percentage of blocks that should satisfy the minimal replication requirement defined by dfs.replication.min. Enter a value less than or equal to 0 to wait for any particular percentage of blocks before exiting safemode. Values greater than 1 will make safemode permanent.", + "display_name": "Safemode Threshold Percentage", + "name": "dfs_safemode_threshold_pct", + "value": "0.999" + }, + { + "desc": "Directories on the local file system to store the NameNode edits. If not set, the edits are stored in the NameNode's Data Directories. The value of this configuration is automatically generated to be the Quorum-based Storage URI if there are JournalNodes and this NameNode is not Highly Available.", + "display_name": "NameNode Edits Directories", + "name": "dfs_namenode_edits_dir", + "value": null + }, + { + "desc": "If set to false and if one of the replicas of the NameNode storage fails, such as temporarily failure of NFS, this directory is not used until the NameNode restarts. If enabled, failed storage is re-checked on every checkpoint and, if it becomes valid, the NameNode will try to restore the edits and fsimage.", + "display_name": "Restore NameNode Directories at Checkpoint Time", + "name": "dfs_name_dir_restore", + "value": "false" + }, + { + "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", + "display_name": "Health Check Startup Tolerance", + "name": "namenode_startup_tolerance", + "value": "5" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for NameNode", + "name": "namenode_java_opts", + "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" + }, + { + "desc": "Enable Automatic Failover to maintain High Availability. Requires a ZooKeeper service and a High Availability NameNode partner.", + "display_name": "Enable Automatic Failover", + "name": "autofailover_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Determines where on the local file system the NameNode should store the name table (fsimage). For redundancy, enter a comma-delimited list of directories to replicate the name table in all of the directories. Typical values are /data/N/dfs/nn where N=1..3.. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Determines where on the local file system the NameNode should store the name table (fsimage). For redundancy, enter a comma-delimited list of directories to replicate the name table in all of the directories. Typical values are /data/N/dfs/nn where N=1..3. Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "NameNode Data Directories Free Space Monitoring Percentage Thresholds", + "name": "namenode_data_directories_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "For advanced use only, a string to be inserted into dfs_hosts_exclude.txt for this role only.", + "display_name": "NameNode Advanced Configuration Snippet (Safety Valve) for dfs_hosts_exclude.txt", + "name": "namenode_hosts_exclude_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "This determines the percentage amount of block invalidations (deletes) to do over a single DataNode heartbeat deletion command. The final deletion count is determined by applying this percentage to the number of live nodes in the system. The resultant number is the number of blocks from the deletion list chosen for proper invalidation over a single heartbeat of a single DataNode.", + "display_name": "Invalidate Work Percentage Per Iteration", + "name": "dfs_namenode_invalidate_work_pct_per_iteration", + "value": "0.32" + }, + { + "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", + "display_name": "Web Metric Collection", + "name": "namenode_web_metric_collection_enabled", + "value": "true" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The time between two periodic file system checkpoints.", + "display_name": "Filesystem Checkpoint Period", + "name": "fs_checkpoint_period", + "value": "3600" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "When computing the overall NameNode health, consider the host's health.", + "display_name": "NameNode Host Health Test", + "name": "namenode_host_health_enabled", + "value": "true" + }, + { + "desc": "The access time for HDFS file is precise upto this value. Setting the value of 0 disables access times for HDFS. When using the NFS Gateway role, make sure this property is enabled.", + "display_name": "Access Time Precision", + "name": "dfs_access_time_precision", + "value": "3600000" + }, + { + "desc": "Minimum number of running threads for the Hue Thrift server running on the NameNode", + "display_name": "Hue Thrift Server Min Threadcount", + "name": "dfs_thrift_threads_min", + "value": "10" + }, + { + "desc": "The number of transactions after which the NameNode or SecondaryNameNode will create a checkpoint of the namespace, regardless of whether the checkpoint period has expired.", + "display_name": "Filesystem Checkpoint Transaction Threshold", + "name": "fs_checkpoint_txns", + "value": "1000000" + }, + { + "desc": "Maximum number of running threads for the Hue Thrift server running on the NameNode", + "display_name": "Hue Thrift Server Max Threadcount", + "name": "dfs_thrift_threads_max", + "value": "20" + }, + { + "desc": "Full path to a custom topology script on the host file system. The topology script is used to determine the rack location of nodes. If left blank, a topology script will be provided that uses your hosts' rack information, visible in the \"Hosts\" page.", + "display_name": "Topology Script File Name", + "name": "topology_script_file_name", + "value": null + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "Indicate whether or not to avoid writing to stale DataNodes for which heartbeat messages have not been received by the NameNode for more than Stale DataNode Time Interval. Writes avoid using stale DataNodes unless more than a configured ratio (dfs.namenode.write.stale.datanode.ratio) of DataNodes are marked as stale. See dfs.namenode.avoid.read.stale.datanode for a similar setting for reads.", + "display_name": "Avoid Writing Stale DataNode", + "name": "dfs_namenode_avoid_write_stale_datanode", + "value": "false" + }, + { + "desc": "The health test thresholds of failed status directories in a NameNode.", + "display_name": "NameNode Directory Failures Thresholds", + "name": "namenode_directory_failures_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "Number of minutes between trash checkpoints. Also controls the number of minutes after which a trash checkpoint directory is deleted. To disable the trash feature, enter 0.", + "display_name": "Filesystem Trash Interval", + "name": "fs_trash_interval", + "value": "1440" + }, + { + "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", + "display_name": "NameNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", + "name": "namenode_config_safety_valve", + "value": null + }, + { + "desc": "This determines the total amount of block transfers to begin in parallel at a DataNode for replication, when such a command list is being sent over a DataNode heartbeat by the NameNode. The actual number is obtained by multiplying this value by the total number of live nodes in the cluster. The result number is the number of blocks to transfer immediately, per DataNode heartbeat.", + "display_name": "Replication Work Multiplier Per Iteration", + "name": "dfs_namenode_replication_work_multiplier_per_iteration", + "value": "10" + }, + { + "desc": "The maximum number of rolled log files to keep for NameNode logs. Typically used by log4j or logback.", + "display_name": "NameNode Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "Determines extension of safemode in milliseconds after the threshold level is reached.", + "display_name": "Safemode Extension", + "name": "dfs_safemode_extension", + "value": "30000" + }, + { + "desc": "Timeout in seconds for the Hue Thrift server running on the NameNode", + "display_name": "Hue Thrift Server Timeout", + "name": "dfs_thrift_timeout", + "value": "60" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "The number of server threads for the NameNode used for service calls. Only used when NameNode Service RPC Port is configured.", + "display_name": "NameNode Service Handler Count", + "name": "dfs_namenode_service_handler_count", + "value": "30" + }, + { + "desc": "The health test thresholds of the number of transactions since the last HDFS namespace checkpoint. Specified as a percentage of the configured checkpointing transaction limit.", + "display_name": "Filesystem Checkpoint Transactions Monitoring Thresholds", + "name": "namenode_checkpoint_transactions_thresholds", + "value": "{\"critical\":\"400.0\",\"warning\":\"200.0\"}" + }, + { + "desc": "The health test thresholds of the age of the HDFS namespace checkpoint. Specified as a percentage of the configured checkpoint interval.", + "display_name": "Filesystem Checkpoint Age Monitoring Thresholds", + "name": "namenode_checkpoint_age_thresholds", + "value": "{\"critical\":\"400.0\",\"warning\":\"200.0\"}" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "The base port where the secure NameNode web UI listens.", + "display_name": "Secure NameNode Web UI Port (SSL)", + "name": "dfs_https_port", + "value": "50470" + }, + { + "desc": "For advanced use only, a string to be inserted into dfs_hosts_allow.txt for this role only.", + "display_name": "NameNode Advanced Configuration Snippet (Safety Valve) for dfs_hosts_allow.txt", + "name": "namenode_hosts_allow_safety_valve", + "value": null + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "If enabled, the NameNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.", + "display_name": "Bind NameNode to Wildcard Address", + "name": "namenode_bind_wildcard", + "value": "false" + }, + { + "desc": "Comma-separated list of NameNode plug-ins to be activated. If one plug-in cannot be loaded, all the plug-ins are ignored.", + "display_name": "NameNode Plugins", + "name": "dfs_namenode_plugins_list", + "value": "" + }, + { + "desc": "Directory on a shared storage device, such as a Quorum-based Storage URI or a local directory that is an NFS mount from a NAS, to store the NameNode edits. The value of this configuration is automatically generated to be the Quourm Journal URI if there are JournalNodes and this NameNode is Highly Available.", + "display_name": "Shared Edits Directory", + "name": "dfs_namenode_shared_edits_dir", + "value": null + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "The number of server threads for the NameNode.", + "display_name": "NameNode Handler Count", + "name": "dfs_namenode_handler_count", + "value": "30" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "Specifies the number of DataNodes that must be live before the name node exits safemode. Enter a value less than or equal to 0 to take the number of live DataNodes into account when deciding whether to remain in safemode during startup. Values greater than the number of DataNodes in the cluster will make safemode permanent.", + "display_name": "Safemode Minimum DataNodes", + "name": "dfs_safemode_min_datanodes", + "value": "0" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-nfsgateway.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-nfsgateway.json new file mode 100644 index 00000000..f92bb37e --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hdfs-nfsgateway.json @@ -0,0 +1,254 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "

This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.

Each rule has some or all of the following fields:


Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}

This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.

", + "display_name": "Rules to Extract Events from Log Files", + "name": "log_event_whitelist", + "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "nfsgateway_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "NFS clients often reorder writes. As a result, sequential writes can arrive at the NFS Gateway in random order. This directory is used to temporarily save out-of-order writes before writing to HDFS. For each file, the out-of-order writes are dumped after they are accumulated to exceed certain threshold (e.g., 1MB) in memory. Please make sure this directory has enough space. For example, if the application uploads 10 files with each having 100MB, it is recommended that this directory have roughly 1GB of space in case write reorder happens (in the worst case) to every file.", + "display_name": "Temporary Dump Directory", + "name": "dfs_nfs3_dump_dir", + "value": "/tmp/.hdfs-nfs" + }, + { + "desc": "Enables the health test that the NFS Gateway's process state is consistent with the role configuration", + "display_name": "NFS Gateway Process Health Test", + "name": "nfsgateway_scm_health_enabled", + "value": "true" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of NFS Gateway in Bytes", + "name": "nfsgateway_java_heapsize", + "value": "268435456" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", + "display_name": "NFS Gateway Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", + "name": "nfsgateway_config_safety_valve", + "value": null + }, + { + "desc": "The port number of the system portmap or rpcbind service. This configuration is used by Cloudera Manager to verify if the system portmap or rpcbind service is running before starting NFS Gateway role. Cloudera Manager does not manage the system portmap or rpcbind service.", + "display_name": "Portmap (or Rpcbind) Port", + "name": "nfs3_portmap_port", + "value": "111" + }, + { + "desc": "The NFS Gateway server port.", + "display_name": "NFS Gateway Server Port", + "name": "nfs3_server_port", + "value": "2049" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for NFS Gateway", + "name": "nfsgateway_java_opts", + "value": "" + }, + { + "desc": "The maximum number of rolled log files to keep for NFS Gateway logs. Typically used by log4j or logback.", + "display_name": "NFS Gateway Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "When computing the overall NFS Gateway health, consider the host's health.", + "display_name": "NFS Gateway Host Health Test", + "name": "nfsgateway_host_health_enabled", + "value": "true" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "By default, NFS Gateway exported directories can be mounted by any client. For better access control, update this property with a list of host names and access privileges separated by whitespace characters. Host name format can be a single host, a Java regular expression, or an IPv4 address. The access privilege uses rw to specify readwrite and ro to specify readonly access. If the access privilege is not provided, the default is read-only. Examples of host name format and access privilege: \"192.168.0.0/22 rw\", \"host.*.example.com\", \"host1.test.org ro\".", + "display_name": "Allowed Hosts and Privileges", + "name": "dfs_nfs_exports_allowed_hosts", + "value": "* rw" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "The port number of the mount daemon implemented inside the NFS Gateway server role.", + "display_name": "NFS Gateway MountD Port", + "name": "nfs3_mountd_port", + "value": "4242" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "NFS Gateway Environment Advanced Configuration Snippet (Safety Valve)", + "name": "NFSGATEWAY_role_env_safety_valve", + "value": null + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "NFS Gateway Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "The maximum size, in megabytes, per log file for NFS Gateway logs. Typically used by log4j or logback.", + "display_name": "NFS Gateway Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "The minimum log level for NFS Gateway logs", + "display_name": "NFS Gateway Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "Directory where NFS Gateway will place its log files.", + "display_name": "NFS Gateway Log Directory", + "name": "nfsgateway_log_dir", + "value": "/var/log/hadoop-hdfs" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-secondarynamenode.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-secondarynamenode.json new file mode 100644 index 00000000..f4ecd63b --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hdfs-secondarynamenode.json @@ -0,0 +1,308 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "

This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.

Each rule has some or all of the following fields:


Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}

This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.

", + "display_name": "Rules to Extract Events from Log Files", + "name": "log_event_whitelist", + "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"},\n {\"alert\": false, \"rate\": 1, \"threshold\":\"INFO\", \"content\":\"Triggering checkpoint.*\"}\n ]\n}\n" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for Secondary NameNode", + "name": "secondarynamenode_java_opts", + "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Determines where on the local file system the DFS SecondaryNameNode should store the temporary images to merge. For redundancy, enter a comma-delimited list of directories to replicate the image in all of the directories. Typical values are /data/N/dfs/snn for N = 1, 2, 3....", + "display_name": "HDFS Checkpoint Directories Free Space Monitoring Absolute Thresholds", + "name": "secondarynamenode_checkpoint_directories_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of Secondary namenode in Bytes", + "name": "secondary_namenode_java_heapsize", + "value": "1073741824" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "The number of transactions after which the NameNode or SecondaryNameNode will create a checkpoint of the namespace, regardless of whether the checkpoint period has expired.", + "display_name": "Filesystem Checkpoint Transaction Threshold", + "name": "fs_checkpoint_txns", + "value": "1000000" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", + "display_name": "Web Metric Collection", + "name": "secondarynamenode_web_metric_collection_enabled", + "value": "true" + }, + { + "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", + "display_name": "SecondaryNameNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", + "name": "secondarynamenode_config_safety_valve", + "value": null + }, + { + "desc": "Enables the health test that the SecondaryNameNode's process state is consistent with the role configuration", + "display_name": "SecondaryNameNode Process Health Test", + "name": "secondarynamenode_scm_health_enabled", + "value": "true" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", + "display_name": "Garbage Collection Duration Thresholds", + "name": "secondarynamenode_gc_duration_thresholds", + "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" + }, + { + "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", + "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", + "name": "hadoop_metrics2_safety_valve", + "value": null + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "SecondaryNameNode Environment Advanced Configuration Snippet (Safety Valve)", + "name": "SECONDARYNAMENODE_role_env_safety_valve", + "value": null + }, + { + "desc": "If enabled, the SecondaryNameNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.", + "display_name": "Bind SecondaryNameNode to Wildcard Address", + "name": "secondary_namenode_bind_wildcard", + "value": "false" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "SecondaryNameNode Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "The time between two periodic file system checkpoints.", + "display_name": "Filesystem Checkpoint Period", + "name": "fs_checkpoint_period", + "value": "3600" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "The health test thresholds on the duration of the metrics request to the web server.", + "display_name": "Web Metric Collection Duration", + "name": "secondarynamenode_web_metric_collection_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Determines where on the local file system the DFS SecondaryNameNode should store the temporary images to merge. For redundancy, enter a comma-delimited list of directories to replicate the image in all of the directories. Typical values are /data/N/dfs/snn for N = 1, 2, 3.... Specified as a percentage of the capacity on that filesystem. This setting is not used if a Determines where on the local file system the DFS SecondaryNameNode should store the temporary images to merge. For redundancy, enter a comma-delimited list of directories to replicate the image in all of the directories. Typical values are /data/N/dfs/snn for N = 1, 2, 3... Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "HDFS Checkpoint Directories Free Space Monitoring Percentage Thresholds", + "name": "secondarynamenode_checkpoint_directories_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "When computing the overall SecondaryNameNode health, consider the host's health.", + "display_name": "SecondaryNameNode Host Health Test", + "name": "secondarynamenode_host_health_enabled", + "value": "true" + }, + { + "desc": "The maximum size, in megabytes, per log file for SecondaryNameNode logs. Typically used by log4j or logback.", + "display_name": "SecondaryNameNode Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "secondarynamenode_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "The period to review when computing the moving average of garbage collection time.", + "display_name": "Garbage Collection Duration Monitoring Period", + "name": "secondarynamenode_gc_duration_window", + "value": "5" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Determines where on the local file system the DFS SecondaryNameNode should store the temporary images to merge. For redundancy, enter a comma-delimited list of directories to replicate the image in all of the directories. Typical values are /data/N/dfs/snn for N = 1, 2, 3...", + "display_name": "HDFS Checkpoint Directories", + "name": "fs_checkpoint_dir_list", + "value": null + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "The maximum number of rolled log files to keep for SecondaryNameNode logs. Typically used by log4j or logback.", + "display_name": "SecondaryNameNode Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "Nameservice of this SecondaryNameNode", + "display_name": "SecondaryNameNode Nameservice", + "name": "dfs_secondarynamenode_nameservice", + "value": null + }, + { + "desc": "The minimum log level for SecondaryNameNode logs", + "display_name": "SecondaryNameNode Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "The base port where the secure SecondaryNameNode web UI listens.", + "display_name": "Secure SecondaryNameNode Web UI Port (SSL)", + "name": "dfs_secondary_https_port", + "value": "50495" + }, + { + "desc": "Directory where SecondaryNameNode will place its log files.", + "display_name": "SecondaryNameNode Log Directory", + "name": "secondarynamenode_log_dir", + "value": "/var/log/hadoop-hdfs" + }, + { + "desc": "The SecondaryNameNode HTTP port. If the port is 0, then the server starts on a free port. Combined with the SecondaryNameNode's hostname to build its HTTP address.", + "display_name": "SecondaryNameNode Web UI Port", + "name": "dfs_secondary_http_port", + "value": "50090" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-service.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-service.json new file mode 100644 index 00000000..6d34b4ca --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hdfs-service.json @@ -0,0 +1,746 @@ +[ + { + "desc": "Timeout in milliseconds for the parallel RPCs made in DistributedFileSystem#getFileBlockStorageLocations(). This value is only emitted for Impala.", + "display_name": "HDFS File Block Storage Location Timeout", + "name": "dfs_client_file_block_storage_locations_timeout", + "value": "10000" + }, + { + "desc": "The domain to use for the HTTP cookie that stores the authentication token. In order for authentiation to work correctly across all Hadoop nodes' web-consoles the domain must be correctly set. Important: when using IP addresses, browsers ignore cookies with domain settings. For this setting to work properly all nodes in the cluster must be configured to generate URLs with hostname.domain names on it.", + "display_name": "Hadoop HTTP Authentication Cookie Domain", + "name": "hadoop_http_auth_cookie_domain", + "value": "" + }, + { + "desc": "The user that this service's processes should run as (except the HttpFS server, which has its own user)", + "display_name": "System User", + "name": "process_username", + "value": "hdfs" + }, + { + "desc": "

Event filters are defined in a JSON object like the following:

\n\n
\n{\n  \"defaultAction\" : (\"accept\", \"discard\"),\n  \"rules\" : [\n    {\n      \"action\" : (\"accept\", \"discard\"),\n      \"fields\" : [\n        {\n          \"name\" : \"fieldName\",\n          \"match\" : \"regex\"\n        }\n      ]\n    }\n  ]\n}\n
\n\n

\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n

\n\n

\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n

\n\n

\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n

\n\n

\nThe following is the list of fields that can be filtered for HDFS events:\n

\n\n\n", + "display_name": "Audit Event Filter", + "name": "navigator_audit_event_filter", + "value": "{\n \"comment\" : [\n \"Default filter for HDFS services.\",\n \"Discards events generated by the internal Cloudera and/or HDFS users\",\n \"(cloudera-scm, hbase, mapred, hive, dr.who, solr, impala, and spark),\",\n \"'ls' actions performed by the hdfs user,\",\n \"and events that affect files in the /tmp directory.\"\n ],\n \"defaultAction\" : \"accept\",\n \"rules\" : [\n {\n \"action\" : \"discard\",\n \"fields\" : [\n {\n \"name\" : \"username\",\n \"match\" : \"(?:cloudera-scm|hbase|mapred|hive|dr.who|solr|impala|spark)(?:/.+)?\"\n }\n ]\n },\n {\n \"action\" : \"discard\",\n \"fields\" : [\n {\n \"name\" : \"username\",\n \"match\" : \"(?:hdfs)(?:/.+)?\"\n },\n {\n \"name\" : \"operation\",\n \"match\" : \"(?:listStatus|listCachePools|listCacheDirectives|getfileinfo)\"\n }\n ]\n },\n {\n \"action\" : \"discard\",\n \"fields\" : [\n { \"name\" : \"src\", \"match\" : \"/tmp(?:/.*)?\" }\n ]\n }\n ]\n}\n" + }, + { + "desc": "The password for the SSL keystore.", + "display_name": "Hadoop User Group Mapping LDAP SSL Keystore Password", + "name": "hadoop_group_mapping_ldap_keystore_passwd", + "value": "" + }, + { + "desc": "Comma-delimited list of hosts where you want to allow the Hue user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", + "display_name": "Hue Proxy User Hosts", + "name": "hue_proxy_user_hosts_list", + "value": "*" + }, + { + "desc": "The service monitor will use this directory to create files to test if the hdfs service is healthy. The directory and files are created with permissions specified by 'HDFS Health Canary Directory Permissions'", + "display_name": "HDFS Health Canary Directory", + "name": "firehose_hdfs_canary_directory", + "value": "/tmp/.cloudera_health_monitoring_canary_files" + }, + { + "desc": "Path to the directory where audit logs will be written. The directory will be created if it doesn't exist.", + "display_name": "Audit Log Directory", + "name": "audit_event_log_dir", + "value": "/var/log/hadoop-hdfs/audit" + }, + { + "desc": "Class for user to group mapping (get groups for a given user).", + "display_name": "Hadoop User Group Mapping Implementation", + "name": "hadoop_security_group_mapping", + "value": "org.apache.hadoop.security.ShellBasedUnixGroupsMapping" + }, + { + "desc": "Comma-delimited list of groups that you want to allow the YARN user to impersonate. The default '*' allows all groups. To disable entirely, use a string that does not correspond to a group name, such as '_no_group_'.", + "display_name": "YARN Proxy User Groups", + "name": "yarn_proxy_user_groups_list", + "value": "*" + }, + { + "desc": "Allows the oozie superuser to impersonate any members of a comma-delimited list of groups. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", + "display_name": "Oozie Proxy User Groups", + "name": "oozie_proxy_user_groups_list", + "value": "*" + }, + { + "desc": "Comma-separated list of compression codecs that can be used in job or map compression.", + "display_name": "Compression Codecs", + "name": "io_compression_codecs", + "value": "org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec" + }, + { + "desc": "Comma-separated list of users authorized to used Hadoop. This is emitted only if authorization is enabled.", + "display_name": "Authorized Users", + "name": "hadoop_authorized_users", + "value": "*" + }, + { + "desc": "Enable HDFS short circuit read. This allows a client co-located with the DataNode to read HDFS file blocks directly. This gives a performance boost to distributed clients that are aware of locality.", + "display_name": "Enable HDFS Short Circuit Read", + "name": "dfs_datanode_read_shortcircuit", + "value": "true" + }, + { + "desc": "The distinguished name of the user to bind as when connecting to the LDAP server. This may be left blank if the LDAP server supports anonymous binds.", + "display_name": "Hadoop User Group Mapping LDAP Bind User", + "name": "hadoop_group_mapping_ldap_bind_user", + "value": "" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Service Level Health Alerts", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "Path to the keystore file containing the server certificate and private key used for encrypted shuffle and encrypted web UIs. Applies to configurations of all daemon roles of this service.", + "display_name": "Hadoop SSL Server Keystore File Location", + "name": "ssl_server_keystore_location", + "value": null + }, + { + "desc": "Password for the SSL client truststore. Defines a cluster-wide default that can be overridden by individual services.", + "display_name": "Cluster-Wide Default SSL Client Truststore Password", + "name": "ssl_client_truststore_password", + "value": null + }, + { + "desc": "The password of the bind user.", + "display_name": "Hadoop User Group Mapping LDAP Bind User Password", + "name": "hadoop_group_mapping_ldap_bind_passwd", + "value": "" + }, + { + "desc": "Action to take when the audit event queue is full. Drop the event or shutdown the affected process.", + "display_name": "Audit Queue Policy", + "name": "navigator_audit_queue_policy", + "value": "DROP" + }, + { + "desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.", + "display_name": "Enable Log Event Capture", + "name": "catch_events", + "value": "true" + }, + { + "desc": "Enable/Disable the Log and Query Redaction Policy for this cluster.", + "display_name": "Enable Log and Query Redaction", + "name": "redaction_policy_enabled", + "value": "false" + }, + { + "desc": "For advanced use only, a string to be inserted into core-site.xml. Applies to all roles and client configurations in this HDFS service as well as all its dependent services. Any configs added here will be overridden by their default values in HDFS (which can be found in hdfs-default.xml).", + "display_name": "Cluster-wide Advanced Configuration Snippet (Safety Valve) for core-site.xml", + "name": "core_site_safety_valve", + "value": null + }, + { + "desc": "Enable WebHDFS interface", + "display_name": "Enable WebHDFS", + "name": "dfs_webhdfs_enabled", + "value": "true" + }, + { + "desc": "The name of the group of superusers.", + "display_name": "Superuser Group", + "name": "dfs_permissions_supergroup", + "value": "supergroup" + }, + { + "desc": "Path to the SSL client truststore file. Defines a cluster-wide default that can be overridden by individual services. This truststore must be in JKS format. The truststore contains certificates of trusted servers, or of Certificate Authorities trusted to identify servers. The contents of the truststore can be modified without restarting any roles. By default, changes to its contents are picked up within ten seconds. If not set, the default Java truststore is used to verify certificates.", + "display_name": "Cluster-Wide Default SSL Client Truststore Location", + "name": "ssl_client_truststore_location", + "value": null + }, + { + "desc": "Typically, HDFS clients and servers communicate by opening sockets via an IP address. In certain networking configurations, it is preferable to open sockets after doing a DNS lookup on the hostname. Enable this property to open sockets after doing a DNS lookup on the hostname. This property is supported in CDH3u4 or later deployments.", + "display_name": "Use DataNode Hostname", + "name": "dfs_client_use_datanode_hostname", + "value": "false" + }, + { + "desc": "Enter a FailoverProxyProvider implementation to configure two URIs to connect to during fail-over. The first configured address is tried first, and on a fail-over event the other address is tried.", + "display_name": "FailoverProxyProvider Class", + "name": "dfs_ha_proxy_provider", + "value": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" + }, + { + "desc": "The search base for the LDAP connection. This is a distinguished name, and will typically be the root of the LDAP directory.", + "display_name": "Hadoop User Group Mapping Search Base", + "name": "hadoop_group_mapping_ldap_base", + "value": "" + }, + { + "desc": "If false, permission checking is turned off for files in HDFS.", + "display_name": "Check HDFS Permissions", + "name": "dfs_permissions", + "value": "true" + }, + { + "desc": "Comma-delimited list of groups that you want to allow the Hue user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", + "display_name": "Hue Proxy User Groups", + "name": "hue_proxy_user_groups_list", + "value": "*" + }, + { + "desc": "Comma-separated list of groups authorized to used Hadoop. This is emitted only if authorization is enabled.", + "display_name": "Authorized Groups", + "name": "hadoop_authorized_groups", + "value": "" + }, + { + "desc": "Password that protects the private key contained in the server keystore used for encrypted shuffle and encrypted web UIs. Applies to all configurations of daemon roles of this service.", + "display_name": "Hadoop SSL Server Keystore Key Password", + "name": "ssl_server_keystore_keypassword", + "value": null + }, + { + "desc": "Comma-delimited list of hosts where you want to allow the oozie user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", + "display_name": "Oozie Proxy User Hosts", + "name": "oozie_proxy_user_hosts_list", + "value": "*" + }, + { + "desc": "Comma-delimited list of hosts where you want to allow the Cloudera Service Monitor user to impersonate other users. The default '*' allows all hosts. This property is used only if Service Monitor is using a different Kerberos principal than the Hue service. To disable entirely, use a string that does not correspond to a host name, such as '_no_host'.", + "display_name": "Service Monitor Proxy User Hosts", + "name": "smon_proxy_user_hosts_list", + "value": "*" + }, + { + "desc": "Comma-delimited list of groups that you want to allow the mapred user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", + "display_name": "Mapred Proxy User Groups", + "name": "mapred_proxy_user_groups_list", + "value": "*" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", + "display_name": "HDFS Service Environment Advanced Configuration Snippet (Safety Valve)", + "name": "hdfs_service_env_safety_valve", + "value": null + }, + { + "desc": "Allows the Cloudera Service Monitor user to impersonate any members of a comma-delimited list of groups. The default '*' allows all groups. This property is used only if Service Monitor is using a different Kerberos principal than the Hue service. To disable entirely, use a string that does not correspond to a group name, such as '_no_group_'.", + "display_name": "Service Monitor Proxy User Groups", + "name": "smon_proxy_user_groups_list", + "value": "*" + }, + { + "desc": "Additional mapping rules that will be inserted before rules generated from the list of trusted realms and before the default rule. After changing this value and restarting the service, any services depending on this one must be restarted as well. The hadoop.security.auth_to_local property is configured using this information.", + "display_name": "Additional Rules to Map Kerberos Principals to Short Names", + "name": "extra_auth_to_local_rules", + "value": null + }, + { + "desc": "Password for the server keystore file used for encrypted shuffle and encrypted web UIs. Applies to configurations of all daemon roles of this service.", + "display_name": "Hadoop SSL Server Keystore File Password", + "name": "ssl_server_keystore_password", + "value": null + }, + { + "desc": "Maximum size of audit log file in MB before it is rolled over.", + "display_name": "Maximum Audit Log File Size", + "name": "navigator_audit_log_max_file_size", + "value": "100" + }, + { + "desc": "Enables Kerberos authentication for Hadoop HTTP web consoles for all roles of this service using the SPNEGO protocol. Note: This is effective only if Kerberos is enabled for the HDFS service.", + "display_name": "Enable Kerberos Authentication for HTTP Web-Consoles", + "name": "hadoop_secure_web_ui", + "value": "false" + }, + { + "desc": "Comma-delimited list of groups to allow the HDFS user to impersonate. The default '*' allows all groups. To disable entirely, use a string that does not correspond to a group name, such as '_no_group_'.", + "display_name": "HDFS Proxy User Groups", + "name": "hdfs_proxy_user_groups_list", + "value": "*" + }, + { + "desc": "Quality of protection for secured RPC connections between NameNode and HDFS clients. For effective RPC protection, enable Kerberos authentication.", + "display_name": "Hadoop RPC Protection", + "name": "hadoop_rpc_protection", + "value": "authentication" + }, + { + "desc": "Comma-delimited list of groups that you want to allow the Hive user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", + "display_name": "Hive Proxy User Groups", + "name": "hive_proxy_user_groups_list", + "value": "*" + }, + { + "desc": "Comma-separated list of users authorized to perform admin operations on Hadoop. This is emitted only if authorization is enabled.", + "display_name": "Authorized Admin Users", + "name": "hadoop_authorized_admin_users", + "value": "*" + }, + { + "desc": "The health check thresholds of the number of missing blocks. Specified as a percentage of the total number of blocks.", + "display_name": "Missing Block Monitoring Thresholds", + "name": "hdfs_missing_blocks_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "The amount of time after NameNode(s) start that the lack of an active NameNode will be tolerated. This is intended to allow either the auto-failover daemon to make a NameNode active, or a specifically issued failover command to take effect. This is an advanced option that does not often need to be changed.", + "display_name": "NameNode Activation Startup Tolerance", + "name": "hdfs_namenode_activation_startup_tolerance", + "value": "180" + }, + { + "desc": "

Note: Do not edit this property in the classic layout. Switch to the new layout to be able to use preconfigured redaction rules and test your rules inline.

Use this property to define a list of rules to be followed for redacting sensitive information from log files and query strings. Click + to add a new redaction rule. You can choose between one of the preconfigured rules or add a custom rule. When specifying a custom rule, the Search field should contain a regular expression that will be matched against the data. If a match is found, it will be replaced by the contents of the Replace field.

Trigger is an optional field. It can be used to specify a simple string which will be searched in the data. If the string is found, only then will the redactor attempt to find a match for the Search regex. If no Trigger is specified, redaction will always be carried out by matching the Search regular expression. The Trigger field exists solely for performance reasons: simple string matching is faster than regular expression matching.

Test your rules by entering sample text into the Test Redaction Rules text box and click Test Redaction. If no rules match, the text you entered will be returned unchanged.

", + "display_name": "Log and Query Redaction Policy", + "name": "redaction_policy", + "value": null + }, + { + "desc": "Comma-delimited list of groups to allow the HttpFS user to impersonate. The default '*' allows all groups. To disable entirely, use a string that does not correspond to a group name, such as '_no_group_'.", + "display_name": "HttpFS Proxy User Groups", + "name": "httpfs_proxy_user_groups_list", + "value": "*" + }, + { + "desc": "Allows the flume user to impersonate any members of a comma-delimited list of groups. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", + "display_name": "Flume Proxy User Groups", + "name": "flume_proxy_user_groups_list", + "value": "*" + }, + { + "desc": "Comma-delimited list of hosts where you want to allow the mapred user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", + "display_name": "Mapred Proxy User Hosts", + "name": "mapred_proxy_user_hosts_list", + "value": "*" + }, + { + "desc": "For advanced use only, a list of configuration properties that will be used by the Service Monitor instead of the current client configuration for the service.", + "display_name": "Service Monitor Client Config Overrides", + "name": "smon_client_config_overrides", + "value": "dfs.socket.timeout3000dfs.datanode.socket.write.timeout3000ipc.client.connect.max.retries1fs.permissions.umask-mode000" + }, + { + "desc": "

The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n  \"streamThreshold\": 10, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Service Triggers", + "name": "service_triggers", + "value": "[]" + }, + { + "desc": "Enable SSL encryption for HDFS, MapReduce, and YARN web UIs, as well as encrypted shuffle for MapReduce and Yarn.", + "display_name": "Hadoop SSL Enabled", + "name": "hdfs_hadoop_ssl_enabled", + "value": "false" + }, + { + "desc": "List of Kerberos realms that Hadoop services should trust. If empty, defaults to the default_realm property configured in the krb5.conf file. After changing this value and restarting the service, all services depending on this service must also be restarted. Adds mapping rules for each domain to the hadoop.security.auth_to_local property in core-site.xml.", + "display_name": "Trusted Kerberos Realms", + "name": "trusted_realms", + "value": "" + }, + { + "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", + "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", + "name": "smon_derived_configs_safety_valve", + "value": null + }, + { + "desc": "Enables the health check that verifies that the failover controllers associated with this service are healthy and running.", + "display_name": "Failover Controllers Healthy", + "name": "failover_controllers_healthy_enabled", + "value": "true" + }, + { + "desc": "The attribute of the group object that identifies the users that are members of the group. The default will usually be appropriate for any LDAP installation.", + "display_name": "Hadoop User Group Mapping LDAP Group Membership Attribute", + "name": "hadoop_group_mapping_ldap_member_attr", + "value": "member" + }, + { + "desc": "Comma separated list of users allowed to do short circuit read. A short circuit read allows a client co-located with the data to read HDFS file blocks directly from HDFS. If empty, will default to the DataNode process' user.", + "display_name": "DataNode Local Path Access Users", + "name": "dfs_block_local_path_access_user", + "value": null + }, + { + "desc": "The timeout, in milliseconds, to use with the Cloudera Manager agent-based fencer.", + "display_name": "Timeout for Cloudera Manager Fencing Strategy", + "name": "dfs_ha_fencing_cloudera_manager_timeout_millis", + "value": "10000" + }, + { + "desc": "Enable collection of audit events from the service's roles.", + "display_name": "Enable Audit Collection", + "name": "navigator_audit_enabled", + "value": "true" + }, + { + "desc": "Maximum bandwidth used for image transfer in bytes per second. This can help keep normal NameNode operations responsive during checkpointing. A default value of 0 indicates that throttling is disabled.", + "display_name": "FsImage Transfer Bandwidth", + "name": "dfs_image_transfer_bandwidthPerSec", + "value": "0" + }, + { + "desc": "The user the management services impersonates when connecting to HDFS. If no value is specified, the HDFS superuser is used.", + "display_name": "HDFS User to Impersonate", + "name": "hdfs_user_to_impersonate", + "value": null + }, + { + "desc": "The short name of the Hue Kerberos principal. Normally, you do not need to specify this configuration. Cloudera Manager auto-configures this property so that Hue and Cloudera Manamgent Service work properly.", + "display_name": "Hue's Kerberos Principal Short Name", + "name": "hue_kerberos_principal_shortname", + "value": null + }, + { + "desc": "File path to the SSL keystore containing the SSL certificate required by the LDAP server.", + "display_name": "Hadoop User Group Mapping LDAP SSL Keystore", + "name": "hadoop_group_mapping_ldap_keystore", + "value": "" + }, + { + "desc": "Comma-delimited list of hosts where you want to allow the HDFS user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", + "display_name": "HDFS Proxy User Hosts", + "name": "hdfs_proxy_user_hosts_list", + "value": "*" + }, + { + "desc": "The minimal block replication.", + "display_name": "Minimal Block Replication", + "name": "dfs_replication_min", + "value": "1" + }, + { + "desc": "The maximal block replication.", + "display_name": "Maximal Block Replication", + "name": "dfs_replication_max", + "value": "512" + }, + { + "desc": "The service monitor will use these permissions to create the directory and files to test if the hdfs service is healthy. Permissions are specified using the 10-character unix-symbolic format e.g. '-rwxr-xr-x'.", + "display_name": "HDFS Health Canary Directory Permissions", + "name": "firehose_hdfs_canary_directory_permissions", + "value": "-rwxrwxrwx" + }, + { + "desc": "Enable authorization", + "display_name": "Hadoop Secure Authorization", + "name": "hadoop_security_authorization", + "value": "false" + }, + { + "desc": "The Key Management Server used by HDFS. This must be set to use encryption for data at rest.", + "display_name": "KMS Service", + "name": "kms_service", + "value": null + }, + { + "desc": "The attribute of the group object that identifies the group name. The default will usually be appropriate for all LDAP systems.", + "display_name": "Hadoop User Group Mapping LDAP Group Name Attribute", + "name": "hadoop_group_mapping_ldap_group_name_attr", + "value": "cn" + }, + { + "desc": "Enables DataNode support for the experimental DistributedFileSystem.getFileVBlockStorageLocations API. Applicable to CDH 4.1 and onwards.", + "display_name": "Enable HDFS Block Metadata API", + "name": "dfs_datanode_hdfs_blocks_metadata_enabled", + "value": "true" + }, + { + "desc": "The tolerance window that will be used in HDFS service tests that depend on detection of the active NameNode.", + "display_name": "Active NameNode Detection Window", + "name": "hdfs_active_namenode_detecton_window", + "value": "3" + }, + { + "desc": "Default block replication. The number of replications to make when the file is created. The default value is used if a replication number is not specified.", + "display_name": "Replication Factor", + "name": "dfs_replication", + "value": "3" + }, + { + "desc": "Comma-delimited list of groups that you want to allow the HTTP user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'. This is used by WebHCat.", + "display_name": "HTTP Proxy User Groups", + "name": "HTTP_proxy_user_groups_list", + "value": "*" + }, + { + "desc": "The name of the system group shared by all the core Hadoop services.", + "display_name": "Shared Hadoop Group Name", + "name": "hdfs_hadoop_group_name", + "value": "hadoop" + }, + { + "desc": "For paths where authorization is enforced by Sentry Synchronization, file permissions will use this parameter as the group. This group should normally include the hive and impala users.", + "display_name": "Sentry Authorization Provider Group", + "name": "sentry_authorization_provider_hdfs_group", + "value": "hive" + }, + { + "desc": "ACLs (Access Control Lists) enhance the existing HDFS permission model to support controlling file access for arbitrary combinations of users and groups instead of a single owner, single group, and all other users. When ACLs are disabled, the NameNode rejects all attempts to set an ACL.", + "display_name": "Enable Access Control Lists", + "name": "dfs_namenode_acls_enabled", + "value": "false" + }, + { + "desc": "An additional filter to use when searching for LDAP users. The default will usually be appropriate for Active Directory installations. If connecting to a generic LDAP server, ''sAMAccountName'' will likely be replaced with ''uid''. {0} is a special string used to denote where the username fits into the filter.", + "display_name": "Hadoop User Group Mapping LDAP User Search Filter", + "name": "hadoop_group_mapping_ldap_user_filter", + "value": "(&(objectClass=user)(sAMAccountName={0}))" + }, + { + "desc": "List of fencing methods to use for service fencing. shell(./cloudera_manager_agent_fencer.py) is a fencing mechanism designed to use the Cloudera Manager agent. The sshfence method uses SSH. If using custom fencers (that may communicate with shared store, power units, or network switches), use the shell to invoke them.", + "display_name": "HDFS High Availability Fencing Methods", + "name": "dfs_ha_fencing_methods", + "value": "shell(./cloudera_manager_agent_fencer.py)" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the environment of HDFS replication jobs.", + "display_name": "HDFS Replication Advanced Configuration Snippet (Safety Valve)", + "name": "hdfs_replication_env_safety_valve", + "value": null + }, + { + "desc": "The default block size in bytes for new HDFS files. Note that this value is also used as the HBase Region Server HLog block size.", + "display_name": "HDFS Block Size", + "name": "dfs_block_size", + "value": "134217728" + }, + { + "desc": "Path on the DataNode's local file system to a UNIX domain socket used for communication between the DataNode and local HDFS clients. This socket is used for Short Circuit Reads. Only the HDFS System User and \"root\" should have write access to the parent directory and all of its ancestors. This property is supported in CDH 4.2 or later deployments.", + "display_name": "UNIX Domain Socket path", + "name": "dfs_domain_socket_path", + "value": "/var/run/hdfs-sockets/dn" + }, + { + "desc": "Algorithm to encrypt data transfer between DataNodes and clients, and among DataNodes. 3des is more cryptographically secure, but rc4 is substantially faster.", + "display_name": "Data Transfer Encryption Algorithm", + "name": "dfs_encrypt_data_transfer_algorithm", + "value": "rc4" + }, + { + "desc": "The health check thresholds of the number of under-replicated blocks. Specified as a percentage of the total number of blocks.", + "display_name": "Under-replicated Block Monitoring Thresholds", + "name": "hdfs_under_replicated_blocks_thresholds", + "value": "{\"critical\":\"40.0\",\"warning\":\"10.0\"}" + }, + { + "desc": "For advanced use only, a string to be inserted into hdfs-site.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "HDFS Service Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", + "name": "hdfs_service_config_safety_valve", + "value": null + }, + { + "desc": "Comma-delimited list of hosts where you want to allow the HTTP user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'. This is used by WebHCat.", + "display_name": "HTTP Proxy User Hosts", + "name": "HTTP_proxy_user_hosts_list", + "value": "*" + }, + { + "desc": "

\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n

\n\n

\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n

\n\n

Event trackers are defined in a JSON object like the following:

\n\n
\n{\n  \"timeToLive\" : [integer],\n  \"fields\" : [\n    {\n      \"type\" : [string],\n      \"name\" : [string]\n    }\n  ]\n}\n
\n\n

\nWhere:\n

\n\n\n\n

\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n

\n\n\n\n

\nThe following is the list of fields that can be used to compare HDFS events:\n

\n\n\n", + "display_name": "Audit Event Tracker", + "name": "navigator_event_tracker", + "value": "{\n \"comment\" : [\n \"Default event tracker for HDFS services.\",\n \"Defines equality by comparing username, operation and source path \",\n \"of the events.\"\n ],\n \"timeToLive\" : 60000,\n \"fields\" : [\n { \"type\": \"value\", \"name\" : \"src\" },\n { \"type\": \"value\", \"name\" : \"operation\" },\n { \"type\": \"username\", \"name\" : \"username\" }\n ]\n}\n" + }, + { + "desc": "Choose the authentication mechanism used by Hadoop", + "display_name": "Hadoop Secure Authentication", + "name": "hadoop_security_authentication", + "value": "simple" + }, + { + "desc": "For advanced use only, a string to be inserted into hadoop-policy.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "HDFS Service Advanced Configuration Snippet (Safety Valve) for hadoop-policy.xml", + "name": "hadoop_policy_config_safety_valve", + "value": null + }, + { + "desc": "Enable encryption of data transfer between DataNodes and clients, and among DataNodes. For effective data transfer protection, enable Kerberos authentication and choose Privacy Quality of RPC Protection.", + "display_name": "Enable Data Transfer Encryption", + "name": "dfs_encrypt_data_transfer", + "value": "false" + }, + { + "desc": "When computing the overall HDFS cluster health, consider the active NameNode's health", + "display_name": "Active NameNode Role Health Check", + "name": "hdfs_namenode_health_enabled", + "value": "true" + }, + { + "desc": "For advanced use only, a string to be inserted into ssl-server.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "HDFS Service Advanced Configuration Snippet (Safety Valve) for ssl-server.xml", + "name": "hdfs_ssl_server_safety_valve", + "value": null + }, + { + "desc": "The home directory of the system user on the local filesystem. This setting must reflect the system's configured value - only changing it here will not change the actual home directory.", + "display_name": "System User's Home Directory", + "name": "hdfs_user_home_dir", + "value": "/var/lib/hadoop-hdfs" + }, + { + "desc": "SASL protection mode for secured connections to the DataNodes when reading or writing data.", + "display_name": "DataNode Data Transfer Protection", + "name": "dfs_data_transfer_protection", + "value": null + }, + { + "desc": "When computing the overall HDFS cluster health, consider the health of the standby NameNode.", + "display_name": "Standby NameNode Health Check", + "name": "hdfs_standby_namenodes_health_enabled", + "value": "true" + }, + { + "desc": "Comma-delimited list of hosts where you want to allow the flume user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", + "display_name": "Flume Proxy User Hosts", + "name": "flume_proxy_user_hosts_list", + "value": "*" + }, + { + "desc": "The URL for the LDAP server to use for resolving user groups when using LdapGroupsMapping.", + "display_name": "Hadoop User Group Mapping LDAP URL", + "name": "hadoop_group_mapping_ldap_url", + "value": "" + }, + { + "desc": "SSH connection timeout, in milliseconds, to use with the built-in sshfence fencer.", + "display_name": "Timeout for SSH Fencing Strategy", + "name": "dfs_ha_fencing_ssh_connect_timeout", + "value": "30000" + }, + { + "desc": "Maximum number of rolled over audit logs to retain. The logs will not be deleted if they contain audit events that have not yet been propagated to Audit Server.", + "display_name": "Number of Audit Logs to Retain", + "name": "navigator_audit_log_max_backup_index", + "value": "10" + }, + { + "desc": "Enable automatic synchronization of HDFS ACLs with Sentry privileges. HDFS Access Control Lists and Check HDFS Permissions must be enabled when this feature is enabled. Use Sentry Synchronization Path Prefixes to define the HDFS regions where authorization is enforced using Sentry information. For more information, see Synchronizing HDFS ACLs and Sentry Authorization.", + "display_name": "Enable Sentry Synchronization", + "name": "hdfs_sentry_sync_enable", + "value": "false" + }, + { + "desc": "Comma-delimited list of hosts where you allow the HttpFS user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", + "display_name": "HttpFS Proxy User Hosts", + "name": "httpfs_proxy_user_hosts_list", + "value": "*" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Name of the ZooKeeper service that this HDFS service instance depends on", + "display_name": "ZooKeeper Service", + "name": "zookeeper_service", + "value": null + }, + { + "desc": "The amount of time to wait for HDFS filesystem image transfer from NameNode to complete.", + "display_name": "FsImage Transfer Timeout", + "name": "dfs_image_transfer_timeout", + "value": "60000" + }, + { + "desc": "Kerberos principal used by all roles of this service.", + "display_name": "Kerberos Principal", + "name": "kerberos_princ_name", + "value": "hdfs" + }, + { + "desc": "The group that this service's processes should run as (except the HttpFS server, which has its own group)", + "display_name": "System Group", + "name": "process_groupname", + "value": "hdfs" + }, + { + "desc": "Comma-delimited list of hosts where you want to allow the Hive user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", + "display_name": "Hive Proxy User Hosts", + "name": "hive_proxy_user_hosts_list", + "value": "*" + }, + { + "desc": "Comma-delimited list of hosts that you want to allow the YARN user to impersonate. The default '*' allows all hosts. To disable entirely, use a string that does not correspond to a host name, such as '_no_host'.", + "display_name": "YARN Proxy User Hosts", + "name": "yarn_proxy_user_hosts_list", + "value": "*" + }, + { + "desc": "The frequency in which the log4j event publication appender will retry sending undelivered log events to the Event server, in seconds", + "display_name": "Log Event Retry Frequency", + "name": "log_event_retry_frequency", + "value": "30" + }, + { + "desc": "Default umask for file and directory creation, specified in an octal value (with a leading 0)", + "display_name": "Default Umask", + "name": "dfs_umaskmode", + "value": "022" + }, + { + "desc": "The health check thresholds of free space in HDFS. Specified as a percentage of total HDFS capacity.", + "display_name": "HDFS Free Space Monitoring Thresholds", + "name": "hdfs_free_space_thresholds", + "value": "{\"critical\":\"10.0\",\"warning\":\"20.0\"}" + }, + { + "desc": "Enables the health check that a client can create, read, write, and delete files", + "display_name": "HDFS Canary Health Check", + "name": "hdfs_canary_health_enabled", + "value": "true" + }, + { + "desc": "The health check thresholds of the number of blocks that have at least one corrupt replica. Specified as a percentage of the total number of blocks.", + "display_name": "Blocks With Corrupt Replicas Monitoring Thresholds", + "name": "hdfs_blocks_with_corrupt_replicas_thresholds", + "value": "{\"critical\":\"1.0\",\"warning\":\"0.5\"}" + }, + { + "desc": "Comma-separated list of groups authorized to perform admin operations on Hadoop. This is emitted only if authorization is enabled.", + "display_name": "Authorized Admin Groups", + "name": "hadoop_authorized_admin_groups", + "value": "" + }, + { + "desc": "An additional filter to use when searching for groups.", + "display_name": "Hadoop User Group Mapping LDAP Group Search Filter", + "name": "hadoop_group_mapping_ldap_group_filter", + "value": "(objectClass=group)" + }, + { + "desc": "A list of path prefixes that define the HDFS regions where authorization is enforced using Sentry information. Only relevant when Sentry Synchronization is enabled.", + "display_name": "Sentry Synchronization Path Prefixes", + "name": "hdfs_sentry_sync_path_prefixes", + "value": "/user/hive/warehouse" + }, + { + "desc": "For advanced use only, a string to be inserted into the client configuration for navigator.client.properties.", + "display_name": "HDFS Client Advanced Configuration Snippet (Safety Valve) for navigator.client.properties", + "name": "navigator_client_config_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds of the overall DataNode health. The check returns \"Concerning\" health if the percentage of \"Healthy\" DataNodes falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" DataNodes falls below the critical threshold.", + "display_name": "Healthy DataNode Monitoring Thresholds", + "name": "hdfs_datanodes_healthy_thresholds", + "value": "{\"critical\":\"90.0\",\"warning\":\"95.0\"}" + }, + { + "desc": "The SSH private key files to use with the built-in sshfence fencer. These are to be accessible to the hdfs user on the machines running the NameNodes.", + "display_name": "Private Keys for SSH Fencing Strategy", + "name": "dfs_ha_fencing_ssh_private_key_files", + "value": null + }, + { + "desc": "Whether or not to use SSL when connecting to the LDAP server.", + "display_name": "Hadoop User Group Mapping LDAP SSL Enabled", + "name": "hadoop_group_mapping_ldap_use_ssl", + "value": "false" + }, + { + "desc": "For advanced use only, a string to be inserted into ssl-client.xml. Applies cluster-wide, but can be overridden by individual services.", + "display_name": "HDFS Advanced Configuration Snippet (Safety Valve) for ssl-client.xml", + "name": "hdfs_ssl_client_safety_valve", + "value": null + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hive-gateway.json b/sahara/plugins/cdh/v5_4_0/resources/hive-gateway.json new file mode 100644 index 00000000..02580f09 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hive-gateway.json @@ -0,0 +1,62 @@ +[ + { + "desc": "For advanced use only, a string to be inserted into the client configuration for hive-site.xml.", + "display_name": "Hive Client Advanced Configuration Snippet (Safety Valve) for hive-site.xml", + "name": "hive_client_config_safety_valve", + "value": null + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "Gateway Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "These are Java command line arguments. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Client Java Configuration Options", + "name": "hive_client_java_opts", + "value": "-Djava.net.preferIPv4Stack=true" + }, + { + "desc": "Timeout for requests to the Hive Metastore Server. Consider increasing this if you have tables with a lot of metadata and see timeout errors. Used by most Hive Metastore clients such as Hive CLI and HiveServer2, but not by Impala. Impala has a separately configured timeout.", + "display_name": "Hive Metastore Connection Timeout", + "name": "hive_metastore_timeout", + "value": "300" + }, + { + "desc": "The directory where the client configs will be deployed", + "display_name": "Deploy Directory", + "name": "client_config_root_dir", + "value": "/etc/hive" + }, + { + "desc": "Maximum size in bytes for the Java process heap memory. Passed to Java -Xmx.", + "display_name": "Client Java Heap Size in Bytes", + "name": "hive_client_java_heapsize", + "value": "1073741824" + }, + { + "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", + "display_name": "Alternatives Priority", + "name": "client_config_priority", + "value": "90" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the client configuration for hive-env.sh", + "display_name": "Gateway Client Environment Advanced Configuration Snippet for hive-env.sh (Safety Valve)", + "name": "hive_client_env_safety_valve", + "value": null + }, + { + "desc": "The minimum log level for Gateway logs", + "display_name": "Gateway Logging Threshold", + "name": "log_threshold", + "value": "INFO" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hive-hivemetastore.json b/sahara/plugins/cdh/v5_4_0/resources/hive-hivemetastore.json new file mode 100644 index 00000000..0dd6c462 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hive-hivemetastore.json @@ -0,0 +1,248 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for Hive Metastore Server", + "name": "hive_metastore_java_opts", + "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Maximum number of worker threads in the Hive Metastore Server's thread pool", + "display_name": "Max Hive Metastore Server Threads", + "name": "hive_metastore_max_threads", + "value": "100000" + }, + { + "desc": "Enables the health test that checks that basic Hive Metastore operations succeed", + "display_name": "Hive Metastore Canary Health Test", + "name": "metastore_canary_health_enabled", + "value": "true" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "The maximum number of rolled log files to keep for Hive Metastore Server logs. Typically used by log4j or logback.", + "display_name": "Hive Metastore Server Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "Minimum number of worker threads in the Hive Metastore Server's thread pool", + "display_name": "Min Hive Metastore Server Threads", + "name": "hive_metastore_min_threads", + "value": "200" + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "When computing the overall Hive Metastore Server health, consider the host's health.", + "display_name": "Hive Metastore Server Host Health Test", + "name": "hivemetastore_host_health_enabled", + "value": "true" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "Hive Metastore Server Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of Hive Metastore Server in Bytes", + "name": "hive_metastore_java_heapsize", + "value": "1073741824" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "hivemetastore_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "Port on which Hive Metastore Server will listen for connections.", + "display_name": "Hive Metastore Server Port", + "name": "hive_metastore_port", + "value": "9083" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", + "display_name": "Hive Metastore Server Advanced Configuration Snippet (Safety Valve) for hive-site.xml", + "name": "hive_metastore_config_safety_valve", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The maximum size, in megabytes, per log file for Hive Metastore Server logs. Typically used by log4j or logback.", + "display_name": "Hive Metastore Server Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "Enables the health test that the Hive Metastore Server's process state is consistent with the role configuration", + "display_name": "Hive Metastore Server Process Health Test", + "name": "hivemetastore_scm_health_enabled", + "value": "true" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "Directory where Hive Metastore Server will place its log files.", + "display_name": "Hive Metastore Server Log Directory", + "name": "hive_log_dir", + "value": "/var/log/hive" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "The delegation token store implementation class. Use DBTokenStore for Highly Available Metastore Configuration.", + "display_name": "Hive Metastore Delegation Token Store", + "name": "hive_metastore_delegation_token_store", + "value": "org.apache.hadoop.hive.thrift.MemoryTokenStore" + }, + { + "desc": "The minimum log level for Hive Metastore Server logs", + "display_name": "Hive Metastore Server Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "Hive Metastore Server Environment Advanced Configuration Snippet (Safety Valve)", + "name": "hive_metastore_env_safety_valve", + "value": null + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hive-hiveserver2.json b/sahara/plugins/cdh/v5_4_0/resources/hive-hiveserver2.json new file mode 100644 index 00000000..2c8634c3 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hive-hiveserver2.json @@ -0,0 +1,500 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The check interval for session/operation timeout, in milliseconds, which can be disabled by setting to zero or a negative value. ", + "display_name": "Session Check Interval", + "name": "hiveserver2_session_check_interval", + "value": "0" + }, + { + "desc": "Number of cores per Spark executor.", + "display_name": "Spark Executor Cores", + "name": "hiveserver2_spark_executor_cores", + "value": "1" + }, + { + "desc": "Ratio between 0.0 and 1.0 of entries in the vectorized group-by aggregation hash that is flushed when the memory threshold is exceeded.", + "display_name": "Vectorized GroupBy Flush Ratio", + "name": "hiveserver2_vectorized_groupby_flush_ratio", + "value": "0.1" + }, + { + "desc": "Operation will be closed when not accessed for this duration of time, in milliseconds; disable by setting to zero. For a positive value, checked for operations in terminal state only (FINISHED, CANCELED, CLOSED, ERROR). For a negative value, checked for all of the operations regardless of state.", + "display_name": "Idle Operation Timeout", + "name": "hiveserver2_idle_operation_timeout", + "value": "0" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "Address of the load balancer used for HiveServer2 roles, specified in host:port format. If port is not specified, the port used by HiveServer2 is used. Note: Changing this property regenerates Kerberos keytabs for all HiveServer2 roles.", + "display_name": "HiveServer2 Load Balancer", + "name": "hiverserver2_load_balancer", + "value": null + }, + { + "desc": "Initial number of executors used by the application at any given time. This is required if the dynamic executor allocation feature is enabled.", + "display_name": "Initial Number of Executors", + "name": "hiveserver2_spark_dynamic_allocation_initial_executors", + "value": "1" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Directory in HDFS where Hive writes intermediate data between MapReduce jobs. If not specified, Hive uses a default location..", + "display_name": "Hive HDFS Scratch Directory Free Space Monitoring Absolute Thresholds", + "name": "hiveserver2_exec_scratch_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "HiveServer2 will impersonate the beeline client user when talking to other services such as Mapreduce and Hdfs.", + "display_name": "HiveServer2 Enable Impersonation", + "name": "hiveserver2_enable_impersonation", + "value": "true" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Enable optimization that checks if a query can be answered using statistics. If so, answers the query using only statistics stored in metastore.", + "display_name": "Enable Stats Optimization", + "name": "hiveserver2_compute_query_using_stats", + "value": "true" + }, + { + "desc": "Lower bound on the number of executors used by the application at any given time. This is used by dynamic executor allocation", + "display_name": "Lower Bound on Number of Executors", + "name": "hiveserver2_spark_dynamic_allocation_min_executors", + "value": "1" + }, + { + "desc": "When enabled, Spark will add and remove executors dynamically to Hive jobs. This is done based on the workload.", + "display_name": "Enable Dynamic Executor Allocation", + "name": "hiveserver2_spark_dynamic_allocation_enabled", + "value": "true" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "Port on which HiveServer2 will listen for connections.", + "display_name": "HiveServer2 Port", + "name": "hs2_thrift_address_port", + "value": "10000" + }, + { + "desc": "Enable optimization that converts common join into MapJoin based on input file size.", + "display_name": "Enable MapJoin Optimization", + "name": "hiveserver2_enable_mapjoin", + "value": "true" + }, + { + "desc": "The desired file size after merging. This should be larger than hive.merge.smallfiles.avgsize.", + "display_name": "Desired File Size After Merging", + "name": "hiveserver2_merge_size_per_task", + "value": "268435456" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Merge small files at the end of a Spark job. When enabled, a map-only job is created to merge the files in the destination table/partitions.", + "display_name": "Enable Merging Small Files - Spark Job", + "name": "hiveserver2_merge_sparkfiles", + "value": "true" + }, + { + "desc": "When the average output file size of a job is less than the value of this property, Hive will start an additional map-only job to merge the output files into bigger files. This is only done for map-only jobs if hive.merge.mapfiles is true, for map-reduce jobs if hive.merge.mapredfiles is true, and for Spark jobs if hive.merge.sparkfiles is true.", + "display_name": "Small File Average Size Merge Threshold", + "name": "hiveserver2_merge_smallfiles_avgsize", + "value": "16777216" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Local Directory where Hive stores jars and data when performing a MapJoin optimization. If not specified, Hive uses a default location..", + "display_name": "Hive Local Scratch Directory Free Space Monitoring Absolute Thresholds", + "name": "hiveserver2_exec_local_scratch_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", + "display_name": "HiveServer2 Advanced Configuration Snippet (Safety Valve) for hive-site.xml", + "name": "hive_hs2_config_safety_valve", + "value": null + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "Local directory where Hive stores jars downloaded for remote file systems (HDFS). If not specified, Hive uses a default location.", + "display_name": "Hive Downloaded Resources Directory", + "name": "hiveserver2_downloaded_resources_dir", + "value": null + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "The max percentage of heap to be used for hash in ReduceSink operator for Top-K selection. A 0 means the optimization is disabled. Values accepted are between 0 and 1.", + "display_name": "Maximum ReduceSink Top-K Memory Usage", + "name": "hiveserver2_limit_pushdown_memory_usage", + "value": "0.1" + }, + { + "desc": "Whether to try sorted merge bucket (SMB) join.", + "display_name": "Hive Optimize Sorted Merge Bucket Join", + "name": "hiveserver2_optimize_bucketmapjoin_sortedmerge", + "value": "false" + }, + { + "desc": "When computing the overall HiveServer2 health, consider the host's health.", + "display_name": "HiveServer2 Host Health Test", + "name": "hiveserver2_host_health_enabled", + "value": "true" + }, + { + "desc": "Maximum size of each Spark executor's Java heap memory when Hive is running on Spark.", + "display_name": "Spark Executor Maximum Java Heap Size", + "name": "hiveserver2_spark_executor_memory", + "value": "268435456" + }, + { + "desc": "Maximum size of each Spark driver's Java heap memory when Hive is running on Spark.", + "display_name": "Spark Driver Maximum Java Heap Size", + "name": "hiveserver2_spark_driver_memory", + "value": "268435456" + }, + { + "desc": "If Hive auto convert join is on, and the sum of the size for n-1 of the tables/partitions for a n-way join is smaller than the specified size, the join is directly converted to a MapJoin (there is no conditional task).", + "display_name": "Hive Auto Convert Join Noconditional Size", + "name": "hiveserver2_auto_convert_join_noconditionaltask_size", + "value": "20971520" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Local Directory where Hive stores jars and data when performing a MapJoin optimization. If not specified, Hive uses a default location.. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Local Directory where Hive stores jars and data when performing a MapJoin optimization. If not specified, Hive uses a default location. Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Hive Local Scratch Directory Free Space Monitoring Percentage Thresholds", + "name": "hiveserver2_exec_local_scratch_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Directory in HDFS where Hive writes intermediate data between MapReduce jobs. If not specified, Hive uses a default location.. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Directory in HDFS where Hive writes intermediate data between MapReduce jobs. If not specified, Hive uses a default location. Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Hive HDFS Scratch Directory Free Space Monitoring Percentage Thresholds", + "name": "hiveserver2_exec_scratch_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "HiveServer2 Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "HiveServer2 Environment Advanced Configuration Snippet (Safety Valve)", + "name": "hive_hs2_env_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Directory in HDFS where Hive writes intermediate data between MapReduce jobs. If not specified, Hive uses a default location.", + "display_name": "Hive HDFS Scratch Directory", + "name": "hiveserver2_exec_scratchdir", + "value": null + }, + { + "desc": "Some select queries can be converted to a single FETCH task instead of MapReduce task, minimizing latency. A value of none disables all conversion, minimal converts simple queries such as SELECT * and filter on partition columns, and more will convert SELECT queries including FILTERS.", + "display_name": "Fetch Task Query Conversion", + "name": "hiveserver2_fetch_task_conversion", + "value": "minimal" + }, + { + "desc": "Remove extra map-reduce jobs if the data is already clustered by the same key, eliminating the need to repartition the dataset again.", + "display_name": "Enable ReduceDeDuplication Optimization", + "name": "hiveserver2_optimize_reducededuplication", + "value": "true" + }, + { + "desc": "In vectorized group-by, the number of row entries added to the hash table before re-checking average variable size for memory usage estimation.", + "display_name": "Vectorized GroupBy Check Interval", + "name": "hiveserver2_vectorized_groupby_checkinterval", + "value": "4096" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "Session will be closed when not accessed for this duration of time, in milliseconds; disable by setting to zero or a negative value.", + "display_name": "Idle Session Timeout", + "name": "hiveserver2_idle_session_timeout", + "value": "0" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The maximum size, in megabytes, per log file for HiveServer2 logs. Typically used by log4j or logback.", + "display_name": "HiveServer2 Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "Whether to vectorize the reduce side of query execution.", + "display_name": "Enable Reduce-Side Vectorization", + "name": "hiveserver2_vectorized_reduce_enabled", + "value": "false" + }, + { + "desc": "Merge small files at the end of a map-reduce job. When enabled, a map-only job is created to merge the files in the destination table/partitions.", + "display_name": "Enable Merging Small Files - Map-Reduce Job", + "name": "hiveserver2_merge_mapredfiles", + "value": "false" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "Minimum number of worker threads in HiveServer2's thread pool", + "display_name": "Min HiveServer2 Threads", + "name": "hiveserver2_min_threads", + "value": "5" + }, + { + "desc": "Enabled the Calcite based Cost Based Optimizer for HiveServer2.", + "display_name": "Enable Cost Based Optimizer for Hive", + "name": "hiveserver2_enable_cbo", + "value": "false" + }, + { + "desc": "Portion of total memory used in map-side partial aggregation. When exceeded, the partially aggregated results will be flushed from the map task to the reducers.", + "display_name": "Ratio of Memory Usage for Map-Side Aggregation", + "name": "hiveserver2_map_aggr_hash_memory_ratio", + "value": "0.5" + }, + { + "desc": "Enable map-side partial aggregation, which cause the mapper to generate fewer rows. This reduces the data to be sorted and distributed to reducers.", + "display_name": "Enable Map-Side Aggregation", + "name": "hiveserver2_map_aggr", + "value": "true" + }, + { + "desc": "Number of Spark executors assigned to each application. This should not be set when Dynamic Executor Allocation is enabled.", + "display_name": "Spark Executors Per Application", + "name": "hiveserver2_spark_executor_instances", + "value": null + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for HiveServer2", + "name": "hiveserver2_java_opts", + "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" + }, + { + "desc": "The minimum log level for HiveServer2 logs", + "display_name": "HiveServer2 Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "Local Directory where Hive stores jars and data when performing a MapJoin optimization. If not specified, Hive uses a default location.", + "display_name": "Hive Local Scratch Directory", + "name": "hiveserver2_exec_local_scratchdir", + "value": null + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "When dynamic partition is enabled, reducers keep only one record writer at all times, which lowers the memory pressure on reducers.", + "display_name": "Enable Sorted Dynamic Partition Optimizer", + "name": "hiveserver2_optimize_sort_dynamic_partition", + "value": "false" + }, + { + "desc": "The maximum number of rolled log files to keep for HiveServer2 logs. Typically used by log4j or logback.", + "display_name": "HiveServer2 Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "Enables the health test that the HiveServer2's process state is consistent with the role configuration", + "display_name": "HiveServer2 Process Health Test", + "name": "hiveserver2_scm_health_enabled", + "value": "true" + }, + { + "desc": "Merge small files at the end of a map-only job. When enabled, a map-only job is created to merge the files in the destination table/partitions.", + "display_name": "Enable Merging Small Files - Map-Only Job", + "name": "hiveserver2_merge_mapfiles", + "value": "true" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of HiveServer2 in Bytes", + "name": "hiveserver2_java_heapsize", + "value": "268435456" + }, + { + "desc": "Above this size, queries will not be converted to fetch tasks.", + "display_name": "Fetch Task Query Conversion Threshold", + "name": "hiveserver2_fetch_task_conversion_threshold", + "value": "268435456" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "Directory where HiveServer2 will place its log files.", + "display_name": "HiveServer2 Log Directory", + "name": "hive_log_dir", + "value": "/var/log/hive" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "This is the amount of extra off-heap memory that can be requested from YARN, per driver. This, together with spark.driver.memory, is the total memory that YARN can use to create JVM for a driver process.", + "display_name": "Spark Driver Memory Overhead", + "name": "hiveserver2_spark_yarn_driver_memory_overhead", + "value": "26" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Maximum number of worker threads in HiveServer2's thread pool", + "display_name": "Max HiveServer2 Threads", + "name": "hiveserver2_max_threads", + "value": "100" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "hiveserver2_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "When the number of ReduceSink operators after merging is less than this number, the ReduceDeDuplication optimization will be disabled.", + "display_name": "Mininum Reducers for ReduceDeDuplication Optimization", + "name": "hiveserver2_optimize_reducededuplication_min_reducer", + "value": "4" + }, + { + "desc": "This is the amount of extra off-heap memory that can be requested from YARN, per executor process. This, together with spark.executor.memory, is the total memory that YARN can use to create JVM for an executor process.", + "display_name": "Spark Executor Memory Overhead", + "name": "hiveserver2_spark_yarn_executor_memory_overhead", + "value": "26" + }, + { + "desc": "Enable optimization that vectorizes query execution by streamlining operations by processing a block of 1024 rows at a time.", + "display_name": "Enable Vectorization Optimization", + "name": "hiveserver2_vectorized_enabled", + "value": "true" + }, + { + "desc": "The number of rows with the same key value to be cached in memory per SMB-joined table.", + "display_name": "Hive SMB Join Cache Rows", + "name": "hiveserver2_smbjoin_cache_rows", + "value": "10000" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hive-service.json b/sahara/plugins/cdh/v5_4_0/resources/hive-service.json new file mode 100644 index 00000000..795db0a0 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hive-service.json @@ -0,0 +1,380 @@ +[ + { + "desc": "The user that this service's processes should run as.", + "display_name": "System User", + "name": "process_username", + "value": "hive" + }, + { + "desc": "Directory containing auxiliary JARs used by Hive. This should be a directory location and not a classpath containing one or more JARs. This directory must be created and managed manually on Hive CLI or HiveServer2 host.", + "display_name": "Hive Auxiliary JARs Directory", + "name": "hive_aux_jars_path_dir", + "value": null + }, + { + "desc": "Password for Hive Metastore database", + "display_name": "Hive Metastore Database Password", + "name": "hive_metastore_database_password", + "value": "" + }, + { + "desc": "The password for the HiveServer2 TLS/SSL Certificate Trust Store File. Note that this password is not required to access the trust store: this field can be left blank. This password provides optional integrity checking of the file. The contents of trust stores are certificates, and certificates are public information.", + "display_name": "HiveServer2 TLS/SSL Certificate Trust Store Password", + "name": "hiveserver2_truststore_password", + "value": null + }, + { + "desc": "Name of the Sentry service that this Hive service instance depends on. If selected, Hive uses this Sentry service to look up authorization privileges. Before enabling Sentry, read the requirements and configuration steps in Setting Up The Sentry Service.", + "display_name": "Sentry Service", + "name": "sentry_service", + "value": null + }, + { + "desc": "Directory name where Hive Metastore's database is stored (only for Derby)", + "display_name": "Hive Metastore Derby Path", + "name": "hive_metastore_derby_path", + "value": "/var/lib/hive/cloudera_manager/derby/metastore_db" + }, + { + "desc": "Hive warehouse directory is the location in HDFS where Hive's tables are stored. Note that Hive's default value for its warehouse directory is '/user/hive/warehouse'.", + "display_name": "Hive Warehouse Directory", + "name": "hive_warehouse_directory", + "value": "/user/hive/warehouse" + }, + { + "desc": "The password for the HiveServer2 JKS keystore file.", + "display_name": "HiveServer2 TLS/SSL Server JKS Keystore File Password", + "name": "hiveserver2_keystore_password", + "value": null + }, + { + "desc": "

\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n

\n\n

\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n

\n\n

Event trackers are defined in a JSON object like the following:

\n\n
\n{\n  \"timeToLive\" : [integer],\n  \"fields\" : [\n    {\n      \"type\" : [string],\n      \"name\" : [string]\n    }\n  ]\n}\n
\n\n

\nWhere:\n

\n\n\n\n

\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n

\n\n\n\n

\nThe following is the list of fields that can be used to compare Hive events:\n

\n\n\n\n", + "display_name": "Audit Event Tracker", + "name": "navigator_event_tracker", + "value": null + }, + { + "desc": "Action to take when the audit event queue is full. Drop the event or shutdown the affected process.", + "display_name": "Audit Queue Policy", + "name": "navigator_audit_queue_policy", + "value": "DROP" + }, + { + "desc": "HDFS path to the global policy file for Sentry authorization. This should be a relative path (and not a full HDFS URL). The global policy file must be in Sentry policy file format.", + "display_name": "Sentry Global Policy File", + "name": "hive_sentry_provider_resource", + "value": "/user/hive/sentry/sentry-provider.ini" + }, + { + "desc": "The location on disk of the trust store, in .jks format, used to confirm the authenticity of TLS/SSL servers that HiveServer2 might connect to. This is used when HiveServer2 is the client in a TLS/SSL connection. This trust store must contain the certificate(s) used to sign the service(s) being connected to. If this parameter is not provided, the default list of well-known certificate authorities is used instead.", + "display_name": "HiveServer2 TLS/SSL Certificate Trust Store File", + "name": "hiveserver2_truststore_file", + "value": null + }, + { + "desc": "Max number of reducers to use. If the configuration parameter Hive Reduce Tasks is negative, Hive will limit the number of reducers to the value of this parameter.", + "display_name": "Hive Max Reducers", + "name": "hive_max_reducers", + "value": "1099" + }, + { + "desc": "Prevent Metastore operations in the event of schema version incompatibility. Consider setting this to true to reduce probability of schema corruption during Metastore operations. Note that setting this property to true will also set datanucleus.autoCreateSchema property to false and datanucleus.fixedDatastore property to true. Any values set in Cloudera Manager for these properties will be overridden.", + "display_name": "Strict Hive Metastore Schema Validation", + "name": "hive_metastore_schema_verification", + "value": "true" + }, + { + "desc": "Encrypt communication between clients and HiveServer2 using Transport Layer Security (TLS) (formerly known as Secure Socket Layer (SSL)).", + "display_name": "Enable TLS/SSL for HiveServer2", + "name": "hiveserver2_enable_ssl", + "value": "false" + }, + { + "desc": "Type of Hive Metastore database. Note that Derby is not recommended and Cloudera Impala does not support Derby.", + "display_name": "Hive Metastore Database Type", + "name": "hive_metastore_database_type", + "value": "mysql" + }, + { + "desc": "Maximum size of audit log file in MB before it is rolled over.", + "display_name": "Maximum Audit Log File Size", + "name": "navigator_audit_log_max_file_size", + "value": "100" + }, + { + "desc": "For advanced use only, a string to be inserted into hive-site.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Hive Service Advanced Configuration Snippet (Safety Valve) for hive-site.xml", + "name": "hive_service_config_safety_valve", + "value": null + }, + { + "desc": "Maximum number of rolled over audit logs to retain. The logs will not be deleted if they contain audit events that have not yet been propagated to Audit Server.", + "display_name": "Number of Audit Logs to Retain", + "name": "navigator_audit_log_max_backup_index", + "value": "10" + }, + { + "desc": "The URI of the LDAP server to use if LDAP authentication is enabled. The URI must be prefixed with ldap:// or ldaps://. Usernames and passwords will go over the wire in the clear unless an \"ldaps://\" URI is specified. The URI can optionally specify the port, for example: ldaps://ldap_server.example.com:636.", + "display_name": "LDAP URI", + "name": "hiveserver2_ldap_uri", + "value": null + }, + { + "desc": "Port number of Hive Metastore database", + "display_name": "Hive Metastore Database Port", + "name": "hive_metastore_database_port", + "value": "3306" + }, + { + "desc": "Name of the ZooKeeper service that this Hive service instance depends on.", + "display_name": "ZooKeeper Service", + "name": "zookeeper_service", + "value": null + }, + { + "desc": "Host name of Hive Metastore database", + "display_name": "Hive Metastore Database Host", + "name": "hive_metastore_database_host", + "value": "localhost" + }, + { + "desc": "List of users that are allowed to bypass Sentry Authorization in the Hive metastore. These are usually service users that already ensure that all activity has been authorized, such as hive and impala. Only applies when Hive is using Sentry Service.", + "display_name": "Bypass Sentry Authorization Users", + "name": "sentry_metastore_service_users", + "value": "hive,impala,hue,hdfs" + }, + { + "desc": "When set, this value will be appended to all usernames before authenticating with the LDAP server. For example, if this parameter is set to \"my.domain.com\" and the user authenticating is \"mike\", then \"mike@my.domain.com\" will be passed to the LDAP server. If this field is unset, the username will remain unaltered before being passed to the LDAP server. LDAP Domain is most useful when authenticating against an Active Directory server. This parameter is mutually exclusive with LDAP BaseDN.", + "display_name": "LDAP Domain", + "name": "hiveserver2_ldap_domain", + "value": null + }, + { + "desc": "The path to the TLS/SSL keystore file containing the server certificate and private key used for TLS/SSL. Used when HiveServer2 is acting as a TLS/SSL server. The keystore must be in JKS format.", + "display_name": "HiveServer2 TLS/SSL Server JKS Keystore File Location", + "name": "hiveserver2_keystore_path", + "value": null + }, + { + "desc": "Automatically create or upgrade tables in the Hive Metastore database when needed. Consider setting this to false and managing the schema manually.", + "display_name": "Auto Create and Upgrade Hive Metastore Database Schema", + "name": "hive_metastore_database_auto_create_schema", + "value": "false" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Hive Service Environment Advanced Configuration Snippet (Safety Valve)", + "name": "hive_service_env_safety_valve", + "value": null + }, + { + "desc": "Path to the directory where audit logs will be written. The directory will be created if it doesn't exist.", + "display_name": "Audit Log Directory", + "name": "audit_event_log_dir", + "value": "/var/log/hive/audit" + }, + { + "desc": "Kerberos principal used by all roles of this service.", + "display_name": "Kerberos Principal", + "name": "kerberos_princ_name", + "value": "hive" + }, + { + "desc": "When checked, LDAP-based authentication for users is enabled. This option is incompatible with Kerberos authentication for Hive Server 2 at this time.", + "display_name": "Enable LDAP Authentication", + "name": "hiveserver2_enable_ldap_auth", + "value": "false" + }, + { + "desc": "Name of Hive Metastore database", + "display_name": "Hive Metastore Database Name", + "name": "hive_metastore_database_name", + "value": "metastore" + }, + { + "desc": "The class to use in Sentry authorization for user to group mapping. Sentry authorization may be configured to use either Hadoop user to group mapping or local groups defined in the policy file. Hadoop user to group mapping may be configured in the Cloudera Manager HDFS service configuration page under the Security section.", + "display_name": "Sentry User to Group Mapping Class", + "name": "hive_sentry_provider", + "value": "org.apache.sentry.provider.file.HadoopGroupResourceAuthorizationProvider" + }, + { + "desc": "Use Sentry to enable role-based, fine-grained authorization. This configuration enables Sentry using policy files. To enable Sentry using the Sentry service instead, add the Sentry service as a dependency to the Hive service. The Sentry service provides concurrent and secure access to authorization policy metadata and is the recommended option for enabling Sentry. Sentry is supported only on CDH 4.4 or later deployments. Before enabling Sentry, read the requirements and configuration steps in Setting Up Hive Authorization with Sentry.", + "display_name": "Enable Sentry Authorization using Policy Files", + "name": "sentry_enabled", + "value": "false" + }, + { + "desc": "This configuration overrides the value set for Hive Proxy User Groups configuration in HDFS service for use by Hive Metastore Server. Specify a comma-delimited list of groups that you want to allow access to Hive Metastore metadata and allow the Hive user to impersonate. A value of '*' allows all groups. The default value of empty inherits the value set for Hive Proxy User Groups configuration in the HDFS service.", + "display_name": "Hive Metastore Access Control and Proxy User Groups Override", + "name": "hive_proxy_user_groups_list", + "value": null + }, + { + "desc": "Default number of reduce tasks per job. Usually set to a prime number close to the number of available hosts. Ignored when mapred.job.tracker is \"local\". Hadoop sets this to 1 by default, while Hive uses -1 as the default. When set to -1, Hive will automatically determine an appropriate number of reducers for each job.", + "display_name": "Hive Reduce Tasks", + "name": "hive_reduce_tasks", + "value": "-1" + }, + { + "desc": "Let the table directories inherit the permission of the Warehouse or Database directory instead of being created with the permissions derived from dfs umask. This allows Impala to insert into tables created via Hive.", + "display_name": "Hive Warehouse Subdirectories Inherit Permissions", + "name": "hive_warehouse_subdir_inherit_perms", + "value": "true" + }, + { + "desc": "For advanced use only, a string to be inserted into core-site.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Hive Service Advanced Configuration Snippet (Safety Valve) for core-site.xml", + "name": "hive_core_site_safety_valve", + "value": null + }, + { + "desc": "The group that this service's processes should run as.", + "display_name": "System Group", + "name": "process_groupname", + "value": "hive" + }, + { + "desc": "

Event filters are defined in a JSON object like the following:

\n\n
\n{\n  \"defaultAction\" : (\"accept\", \"discard\"),\n  \"rules\" : [\n    {\n      \"action\" : (\"accept\", \"discard\"),\n      \"fields\" : [\n        {\n          \"name\" : \"fieldName\",\n          \"match\" : \"regex\"\n        }\n      ]\n    }\n  ]\n}\n
\n\n

\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n

\n\n

\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n

\n\n

\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n

\n\n

\nThe following is the list of fields that can be filtered for Hive events:\n

\n\n\n", + "display_name": "Audit Event Filter", + "name": "navigator_audit_event_filter", + "value": "{\n \"comment\" : [\n \"Default filter for Hive services.\",\n \"Discards events generated by Hive MR jobs in /tmp directory\"\n ],\n \"defaultAction\" : \"accept\",\n \"rules\" : [\n {\n \"action\" : \"discard\",\n \"fields\" : [\n { \"name\" : \"operation\", \"match\" : \"QUERY\" },\n { \"name\" : \"objectType\", \"match\" : \"DFS_DIR\"},\n { \"name\" : \"resourcePath\", \"match\" : \"/tmp/hive-(?:.+)?/hive_(?:.+)?/-mr-.*\" }\n ]\n }\n ]\n}\n" + }, + { + "desc": "The health test thresholds of the overall Hive Metastore Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Hive Metastore Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Hive Metastore Servers falls below the critical threshold.", + "display_name": "Healthy Hive Metastore Server Monitoring Thresholds", + "name": "hive_hivemetastores_healthy_thresholds", + "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Service Level Health Alerts", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The server name used when defining privilege rules in Sentry authorization. Sentry uses this name as an alias for the Hive service. It does not correspond to any physical server name.", + "display_name": "Server Name for Sentry Authorization", + "name": "hive_sentry_server", + "value": "server1" + }, + { + "desc": "Name of the Spark on YARN service that this Hive service instance depends on. If selected and Enable Hive on Spark is set to true, Hive jobs can use the Spark execution engine instead of MapReduce2. Requires that Hive depends on YARN.", + "display_name": "Spark On YARN Service", + "name": "spark_on_yarn_service", + "value": null + }, + { + "desc": "The health test thresholds of the overall HiveServer2 health. The check returns \"Concerning\" health if the percentage of \"Healthy\" HiveServer2s falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" HiveServer2s falls below the critical threshold.", + "display_name": "Healthy HiveServer2 Monitoring Thresholds", + "name": "hive_hiveserver2s_healthy_thresholds", + "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" + }, + { + "desc": "For advanced use only, a list of configuration properties that will be used by the Service Monitor instead of the current client configuration for the service.", + "display_name": "Service Monitor Client Config Overrides", + "name": "smon_client_config_overrides", + "value": "hive.metastore.client.socket.timeout60" + }, + { + "desc": "

The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n  \"streamThreshold\": 10, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Service Triggers", + "name": "service_triggers", + "value": "[]" + }, + { + "desc": "Instead of talking to Hive Metastore Server for Metastore information, Hive clients will talk directly to the Metastore database.", + "display_name": "Bypass Hive Metastore Server", + "name": "hive_bypass_metastore_server", + "value": "false" + }, + { + "desc": "Smaller than this size, Hive uses a single-threaded copy; larger than this size, Hive uses DistCp.", + "display_name": "Hive Copy Large File Size", + "name": "hive_exec_copyfile_maxsize", + "value": "33554432" + }, + { + "desc": "User for Hive Metastore database", + "display_name": "Hive Metastore Database User", + "name": "hive_metastore_database_user", + "value": "hive" + }, + { + "desc": "Perform DataNucleus validation of metadata during startup. Note: when enabled, Hive will log DataNucleus warnings even though Hive will function normally.", + "display_name": "Hive Metastore Database DataNucleus Metadata Validation", + "name": "hive_metastore_database_datanucleus_metadata_validation", + "value": "false" + }, + { + "desc": "Cloudera does not support Hive on Spark in CDH 5.4. Enable Hive to use Spark for execution even though it is not supported. For evaluation purposes only.", + "display_name": "Enable Hive on Spark (Unsupported)", + "name": "enable_hive_on_spark", + "value": "false" + }, + { + "desc": "For advanced use only, a string to be inserted into the client configuration for navigator.client.properties.", + "display_name": "Hive Client Advanced Configuration Snippet (Safety Valve) for navigator.client.properties", + "name": "navigator_client_config_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds of the overall WebHCat Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" WebHCat Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" WebHCat Servers falls below the critical threshold.", + "display_name": "Healthy WebHCat Server Monitoring Thresholds", + "name": "hive_webhcats_healthy_thresholds", + "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" + }, + { + "desc": "For advanced use only, a string to be inserted into sentry-site.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Hive Service Advanced Configuration Snippet (Safety Valve) for sentry-site.xml", + "name": "hive_server2_sentry_safety_valve", + "value": null + }, + { + "desc": "Size per reducer. If the input size is 10GiB and this is set to 1GiB, Hive will use 10 reducers.", + "display_name": "Hive Bytes Per Reducer", + "name": "hive_bytes_per_reducer", + "value": "67108864" + }, + { + "desc": "Enable collection of audit events from the service's roles.", + "display_name": "Enable Audit Collection", + "name": "navigator_audit_enabled", + "value": "true" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Allows URIs when defining privileges in per-database policy files. Warning: Typically, this configuration should be disabled. Enabling it would allow database policy file owner (which is generally not Hive admin user) to grant load privileges to any directory with read access to Hive admin user, including databases controlled by other database policy files.", + "display_name": "Allow URIs in Database Policy File", + "name": "sentry_allow_uri_db_policyfile", + "value": "false" + }, + { + "desc": "In unsecure mode, setting this property to true will cause the Metastore Server to execute DFS operations using the client's reported user and group permissions. Cloudera Manager will set this for all clients and servers.", + "display_name": "Set User and Group Information", + "name": "hive_set_ugi", + "value": "true" + }, + { + "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", + "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", + "name": "smon_derived_configs_safety_valve", + "value": null + }, + { + "desc": "When set, this parameter is used to convert the username into the LDAP Distinguished Name (DN), so that the resulting DN looks like uid=username,X. For example, if this parameter is set to \"ou=People,dc=cloudera,dc=com\", and the username passed in is \"mike\", the resulting authentication passed to the LDAP server will look like \"uid=mike,ou=People,dc=cloudera,dc=com\". This parameter is generally most useful when authenticating against an OpenLDAP server. This parameter is mutually exclusive with LDAP Domain.", + "display_name": "LDAP BaseDN", + "name": "hiveserver2_ldap_basedn", + "value": null + }, + { + "desc": "MapReduce jobs are run against this service.", + "display_name": "MapReduce Service", + "name": "mapreduce_yarn_service", + "value": null + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hive-site.xml b/sahara/plugins/cdh/v5_4_0/resources/hive-site.xml new file mode 100644 index 00000000..7e70626f --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hive-site.xml @@ -0,0 +1,61 @@ + + + + + + hive.metastore.local + false + + + hive.metastore.uris + thrift://test-master-001.novalocal:9083 + + + hive.metastore.client.socket.timeout + 300 + + + hive.metastore.warehouse.dir + /user/hive/warehouse + + + hive.warehouse.subdir.inherit.perms + true + + + mapred.reduce.tasks + -1 + + + hive.exec.reducers.bytes.per.reducer + 1073741824 + + + hive.exec.copyfile.maxsize + 33554432 + + + hive.exec.reducers.max + 999 + + + hive.metastore.execute.setugi + true + + + hive.cluster.delegation.token.store.class + org.apache.hadoop.hive.thrift.MemoryTokenStore + + + hive.server2.enable.doAs + true + + + fs.hdfs.impl.disable.cache + true + + + hive.server2.use.SSL + false + + diff --git a/sahara/plugins/cdh/v5_4_0/resources/hive-webhcat.json b/sahara/plugins/cdh/v5_4_0/resources/hive-webhcat.json new file mode 100644 index 00000000..145d161e --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hive-webhcat.json @@ -0,0 +1,230 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Port on which WebHCat Server will listen for connections.", + "display_name": "WebHCat Server Port", + "name": "hive_webhcat_address_port", + "value": "50111" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "The maximum number of rolled log files to keep for WebHCat Server logs. Typically used by log4j or logback.", + "display_name": "WebHCat Server Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of WebHCat Server in Bytes", + "name": "hive_webhcat_java_heapsize", + "value": "268435456" + }, + { + "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", + "display_name": "WebHCat Server Advanced Configuration Snippet (Safety Valve) for hive-site.xml", + "name": "hive_webhcat_hive_config_safety_valve", + "value": null + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "webhcat_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "WebHCat Server Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "WebHCat Server Environment Advanced Configuration Snippet (Safety Valve)", + "name": "hive_webhcat_env_safety_valve", + "value": null + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for WebHCat Server", + "name": "hive_webhcat_java_opts", + "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The maximum size, in megabytes, per log file for WebHCat Server logs. Typically used by log4j or logback.", + "display_name": "WebHCat Server Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "Enables the health test that the WebHCat Server's process state is consistent with the role configuration", + "display_name": "WebHCat Server Process Health Test", + "name": "webhcat_scm_health_enabled", + "value": "true" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "For advanced use only, a string to be inserted into webhcat-site.xml for this role only.", + "display_name": "WebHCat Server Advanced Configuration Snippet (Safety Valve) for webhcat-site.xml", + "name": "hive_webhcat_config_safety_valve", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "When computing the overall WebHCat Server health, consider the host's health.", + "display_name": "WebHCat Server Host Health Test", + "name": "webhcat_host_health_enabled", + "value": "true" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "The minimum log level for WebHCat Server logs", + "display_name": "WebHCat Server Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "Directory where WebHCat Server will place its log files.", + "display_name": "WebHCat Server Log Directory", + "name": "hcatalog_log_dir", + "value": "/var/log/hcatalog" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hue-hue_server.json b/sahara/plugins/cdh/v5_4_0/resources/hue-hue_server.json new file mode 100644 index 00000000..aa1b426e --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hue-hue_server.json @@ -0,0 +1,188 @@ +[ + { + "desc": "For advanced use only, a string to be inserted into hue_safety_valve_server.ini for this role only.", + "display_name": "Hue Server Advanced Configuration Snippet (Safety Valve) for hue_safety_valve_server.ini", + "name": "hue_server_hue_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Enable HTTPS for the Hue web server.", + "display_name": "Enable HTTPS", + "name": "ssl_enable", + "value": "false" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "hue_server_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "Directory where Hue Server will place its log files.", + "display_name": "Hue Server Log Directory", + "name": "hue_server_log_dir", + "value": "/var/log/hue" + }, + { + "desc": "Timeout in seconds for Thrift calls to HiveServer2 and Impala.", + "display_name": "HiveServer2 and Impala Thrift Connection Timeout", + "name": "hs2_conn_timeout", + "value": "120" + }, + { + "desc": "An optional, custom one-line HTML code to display as a banner on top of all Hue Server web pages. Useful in displaying cluster identity of the Hue Server.", + "display_name": "Top Banner Custom HTML", + "name": "banner_html", + "value": null + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "Location on HDFS where the jobsub examples and templates are stored.", + "display_name": "Jobsub Examples and Templates Directory", + "name": "hue_server_remote_data_dir", + "value": "/user/hue/jobsub" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "When computing the overall Hue Server health, consider the host's health.", + "display_name": "Hue Server Host Health Test", + "name": "hue_server_host_health_enabled", + "value": "true" + }, + { + "desc": "Port to use to connect to the Hue server.", + "display_name": "Hue HTTP Port", + "name": "hue_http_port", + "value": "8888" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "Hue Server Environment Advanced Configuration Snippet (Safety Valve)", + "name": "HUE_SERVER_role_env_safety_valve", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "Path to the SSL private key on the host running the Hue web server. This file must be readable by the Hue system user. Hue only supports a key without a passphrase.", + "display_name": "Local Path to SSL Private Key", + "name": "ssl_private_key", + "value": null + }, + { + "desc": "Path to the SSL certificate on the host running the Hue web server. This file must be readable by the Hue system user.", + "display_name": "Local Path to SSL Certificate", + "name": "ssl_certificate", + "value": null + }, + { + "desc": "If enabled, the Hue server binds to the wildcard address (\"0.0.0.0\") for its ports.", + "display_name": "Bind Hue Server to Wildcard Address", + "name": "hue_server_bind_wildcard", + "value": "false" + }, + { + "desc": "Enables the health test that the Hue Server's process state is consistent with the role configuration", + "display_name": "Hue Server Process Health Test", + "name": "hue_server_scm_health_enabled", + "value": "true" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Random string used for secure hashing in the session store.", + "display_name": "Secret Key", + "name": "secret_key", + "value": null + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hue-kt_renewer.json b/sahara/plugins/cdh/v5_4_0/resources/hue-kt_renewer.json new file mode 100644 index 00000000..7fbee5cd --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hue-kt_renewer.json @@ -0,0 +1,134 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "Interval in seconds with which Hue's Kerberos ticket will get renewed.", + "display_name": "Hue Keytab Renewal Interval", + "name": "keytab_reinit_frequency", + "value": "3600" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "Enables the health test that the Kerberos Ticket Renewer's process state is consistent with the role configuration", + "display_name": "Kerberos Ticket Renewer Process Health Test", + "name": "kt_renewer_scm_health_enabled", + "value": "true" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "Kerberos Ticket Renewer Environment Advanced Configuration Snippet (Safety Valve)", + "name": "KT_RENEWER_role_env_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "When computing the overall Kerberos Ticket Renewer health, consider the host's health.", + "display_name": "Kerberos Ticket Renewer Host Health Test", + "name": "kt_renewer_host_health_enabled", + "value": "true" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "kt_renewer_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "Directory where Kerberos Ticket Renewer will place its log files.", + "display_name": "Kerberos Ticket Renewer Log Directory", + "name": "kt_renewer_log_dir", + "value": "/var/log/hue" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hue-service.json b/sahara/plugins/cdh/v5_4_0/resources/hue-service.json new file mode 100644 index 00000000..7c2a7b8e --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/hue-service.json @@ -0,0 +1,350 @@ +[ + { + "desc": "Create users in Hue when they try to login with their LDAP credentials. For use when using LdapBackend for Hue authentication.", + "display_name": "Create LDAP users on login", + "name": "create_users_on_login", + "value": "true" + }, + { + "desc": "Number of threads used by the Hue web server.", + "display_name": "Hue Web Server Threads", + "name": "cherrypy_server_threads", + "value": "10" + }, + { + "desc": "Name of the HBase service that this Hue service instance depends on", + "display_name": "HBase Service", + "name": "hbase_service", + "value": null + }, + { + "desc": "

The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n  \"streamThreshold\": 10, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Service Triggers", + "name": "service_triggers", + "value": "[]" + }, + { + "desc": "Name of the Oozie service that this Hue service instance depends on", + "display_name": "Oozie Service", + "name": "oozie_service", + "value": null + }, + { + "desc": "The distinguished name to use as a search base for finding users and groups. This should be similar to 'dc=hadoop,dc=mycompany,dc=com'.", + "display_name": "LDAP Search Base", + "name": "base_dn", + "value": null + }, + { + "desc": "Comma-separated list of regular expressions, which match 'host:port' of requested proxy target.", + "display_name": "Whitelist", + "name": "whitelist", + "value": "(localhost|127\\.0\\.0\\.1):(50030|50070|50060|50075)" + }, + { + "desc": "Name of the Sentry service that this Hue service instance depends on", + "display_name": "Sentry Service", + "name": "sentry_service", + "value": null + }, + { + "desc": "Thrift server to use for HBase app.", + "display_name": "HBase Thrift Server", + "name": "hue_hbase_thrift", + "value": null + }, + { + "desc": "Base filter for searching for groups. For Active Directory, this is typically '(objectClass=group)'.", + "display_name": "LDAP Group Filter", + "name": "group_filter", + "value": null + }, + { + "desc": "Time zone name.", + "display_name": "Time Zone", + "name": "time_zone", + "value": "America/Los_Angeles" + }, + { + "desc": "The health test thresholds of the overall Kerberos Ticket Renewer health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Kerberos Ticket Renewers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Kerberos Ticket Renewers falls below the critical threshold.", + "display_name": "Healthy Kerberos Ticket Renewer Monitoring Thresholds", + "name": "hue_kt_renewers_healthy_thresholds", + "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" + }, + { + "desc": "The user that this service's processes should run as.", + "display_name": "System User", + "name": "process_username", + "value": "hue" + }, + { + "desc": "The health test thresholds of the overall Hue Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Hue Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Hue Servers falls below the critical threshold.", + "display_name": "Healthy Hue Server Monitoring Thresholds", + "name": "hue_hue_servers_healthy_thresholds", + "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "The group name attribute in the LDAP schema. For Active Directory, this is typically 'cn'.", + "display_name": "LDAP Group Name Attribute", + "name": "group_name_attr", + "value": null + }, + { + "desc": "HDFS directory used for storing temporary files.", + "display_name": "HDFS Temporary Directory", + "name": "hdfs_tmp_dir", + "value": "/tmp" + }, + { + "desc": "Name of the Solr service that this Hue service instance depends on", + "display_name": "Solr Service", + "name": "solr_service", + "value": null + }, + { + "desc": "If the database is SQLite3, this is the filename of the database to use, and the directory of this file must be writable by the 'hue' user.", + "display_name": "Hue Database Directory", + "name": "database_dir", + "value": "/var/lib/hue/desktop.db" + }, + { + "desc": "The name of a default group that users will be added to at creation time.", + "display_name": "Default User Group", + "name": "default_user_group", + "value": null + }, + { + "desc": "When you enable anonymous usage data collection Hue tracks anonymised pages and application versions in order to gather information about each application's usage levels. The data collected does not include any hostnames or IDs. Data collection option is available on CDH 4.4 and later deployments.", + "display_name": "Enable Usage Data Collection", + "name": "usage_data_collection_enable", + "value": "true" + }, + { + "desc": "Name of the ZooKeeper service that this Hue service instance depends on", + "display_name": "ZooKeeper Service", + "name": "zookeeper_service", + "value": null + }, + { + "desc": "Distinguished name of the user to bind as. Not required if the LDAP server supports anonymous searches. For Active Directory, this would be similar to 'hadoop-admin@mycompany.com'.", + "display_name": "LDAP Bind User", + "name": "bind_dn", + "value": null + }, + { + "desc": "The URL of the LDAP Server; similar to 'ldap://auth.mycompany.com' or 'ldaps://auth.mycompany.com'.", + "display_name": "LDAP URL", + "name": "ldap_url", + "value": null + }, + { + "desc": "Whether to use StartTLS (as opposed to ldaps) to communicate securely with the LDAP server. This is only effective when the LDAP certificate is specified.", + "display_name": "Use StartTLS", + "name": "use_start_tls", + "value": "true" + }, + { + "desc": "The password of the bind user.", + "display_name": "LDAP Bind Password", + "name": "bind_password", + "value": null + }, + { + "desc": "Only applies to Active Directory. The Active Directory Domain will be similar to 'MYCOMPANY.COM'.", + "display_name": "NT Domain", + "name": "nt_domain", + "value": null + }, + { + "desc": "Kerberos principal used by all roles of this service.", + "display_name": "Kerberos Principal", + "name": "kerberos_princ_name", + "value": "hue" + }, + { + "desc": "Class that defines extra accessor methods for user objects.", + "display_name": "User Augmentor", + "name": "user_augmentor", + "value": "desktop.auth.backend.DefaultUserAugmentor" + }, + { + "desc": "File where the database gets dumped to or loaded from.", + "display_name": "Database Dump File", + "name": "database_dump_file", + "value": "/tmp/hue_database_dump.json" + }, + { + "desc": "LDAP Username Pattern for use with non-Active Directory LDAP implementations. Must contain the special '<username>' string for replacement during authentication.", + "display_name": "LDAP Username Pattern", + "name": "ldap_username_pattern", + "value": null + }, + { + "desc": "The group that this service's processes should run as.", + "display_name": "System Group", + "name": "process_groupname", + "value": "hue" + }, + { + "desc": "In debug mode, Django displays a detailed traceback when an exception occurs. Debugging information may contain sensitive data. Django remembers every SQL query it executes in debug mode, which will rapidly consume memory.", + "display_name": "Enable Django Debug Mode", + "name": "django_debug_enable", + "value": "false" + }, + { + "desc": "LDAP certificate for authentication over TLS", + "display_name": "LDAP Certificate", + "name": "ldap_cert", + "value": null + }, + { + "desc": "Type of database used for Hue", + "display_name": "Hue Database Type", + "name": "database_type", + "value": "sqlite3" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Service Level Health Alerts", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "When computing the overall HUE health, consider Beeswax Server's health", + "display_name": "Beeswax Server Role Health Test", + "name": "hue_beeswax_server_health_enabled", + "value": "true" + }, + { + "desc": "HTTPFS role or Namenode (if webhdfs is enabled) that hue can use to communicate with HDFS.", + "display_name": "HDFS Web Interface Role", + "name": "hue_webhdfs", + "value": null + }, + { + "desc": "Mode of authenticating login credentials. Select desktop.auth.backend.LdapBackend to use LDAP to authenticate login credentials. LDAP requires you to also set the LDAP URL, NT Domain, and optionally LDAP certificate if you are using secure LDAP. Select desktop.auth.backend.PamBackend to use PAM to authenticate login credentials.", + "display_name": "Authentication Backend", + "name": "auth_backend", + "value": "desktop.auth.backend.AllowFirstUserDjangoBackend" + }, + { + "desc": "Name of the Sqoop service that this Hue service instance depends on", + "display_name": "Sqoop Service", + "name": "sqoop_service", + "value": null + }, + { + "desc": "The username to use to log into the Hue database. Not necessary for SQLite3.", + "display_name": "Hue Database Username", + "name": "database_user", + "value": "hue" + }, + { + "desc": "Enable debug output in HTTP Internal Server Error (status 500) responses. Debugging information may contain sensitive data. If Enable Django Debug Mode is set, this is automatically enabled.", + "display_name": "Enable Debugging of Internal Server Error Responses", + "name": "http_500_debug_enable", + "value": "false" + }, + { + "desc": "Comma-separated list of regular expressions, which match any prefix of 'host:port/path' of requested proxy target. This does not support matching GET parameters.", + "display_name": "Blacklist", + "name": "blacklist", + "value": "()" + }, + { + "desc": "For advanced use only, a string to be inserted into sentry-site.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Hue Service Advanced Configuration Snippet (Safety Valve) for sentry-site.xml", + "name": "hue_sentry_safety_valve", + "value": null + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Hue Service Environment Advanced Configuration Snippet (Safety Valve)", + "name": "hue_service_env_safety_valve", + "value": null + }, + { + "desc": "Name of the Impala service that this Hue service instance depends on", + "display_name": "Impala Service", + "name": "impala_service", + "value": null + }, + { + "desc": "Name of Hue database.", + "display_name": "Hue Database Name", + "name": "database_name", + "value": "hue" + }, + { + "desc": "The attribute of the group object that identifies the members of the group. For Active Directory, this is typically 'member'.", + "display_name": "LDAP Group Membership Attribute", + "name": "group_member_attr", + "value": null + }, + { + "desc": "Name of the Hive service that this Hue service instance depends on", + "display_name": "Hive Service", + "name": "hive_service", + "value": null + }, + { + "desc": "The username attribute in the LDAP schema. For Active Directory, this is typically 'sAMAccountName'.", + "display_name": "LDAP Username Attribute", + "name": "user_name_attr", + "value": null + }, + { + "desc": "Default encoding for site data.", + "display_name": "Default Site Encoding", + "name": "default_site_encoding", + "value": "utf" + }, + { + "desc": "Search Bind Authentication connects to the LDAP server using credentials provided in the 'bind_dn' and 'bind_password' configurations. If these configurations are not set, then an anonymous search is performed.", + "display_name": "Use Search Bind Authentication", + "name": "search_bind_authentication", + "value": "false" + }, + { + "desc": "For advanced use only, a string to be inserted into hue_safety_valve.ini. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Hue Service Advanced Configuration Snippet (Safety Valve) for hue_safety_valve.ini", + "name": "hue_service_safety_valve", + "value": null + }, + { + "desc": "Port on host where the Hue database is running. Not necessary for SQLite3.", + "display_name": "Hue Database Port", + "name": "database_port", + "value": "3306" + }, + { + "desc": "Password for Hue database. Not necessary for SQLite3.", + "display_name": "Hue Database Password", + "name": "database_password", + "value": "" + }, + { + "desc": "The base filter for searching for users. For Active Directory, this is typically '(objectClass=user)'.", + "display_name": "LDAP User Filter", + "name": "user_filter", + "value": null + }, + { + "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", + "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", + "name": "smon_derived_configs_safety_valve", + "value": null + }, + { + "desc": "Name of host where the Hue database is running. Not necessary for SQLite3.", + "display_name": "Hue Database Hostname", + "name": "database_host", + "value": "localhost" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/impala-catalogserver.json b/sahara/plugins/cdh/v5_4_0/resources/impala-catalogserver.json new file mode 100644 index 00000000..e586377c --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/impala-catalogserver.json @@ -0,0 +1,266 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", + "display_name": "Health Check Startup Tolerance", + "name": "catalogserver_startup_tolerance", + "value": "5" + }, + { + "desc": "Port where Catalog Server debug web server runs.", + "display_name": "Catalog Server HTTP Server Port", + "name": "catalogserver_webserver_port", + "value": "25020" + }, + { + "desc": "The health test thresholds on the duration of the metrics request to the web server.", + "display_name": "Web Metric Collection Duration", + "name": "catalogserver_web_metric_collection_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "catalogserver_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "Port where Catalog Server is exported.", + "display_name": "Catalog Server Service Port", + "name": "catalog_service_port", + "value": "26000" + }, + { + "desc": "Buffer log messages logged at this level or lower (-1 means don't buffer; 0 means buffer INFO only; 1 means buffer WARNING only, ...)", + "display_name": "Catalog Server Log Buffer Level", + "name": "logbuflevel", + "value": "0" + }, + { + "desc": "If true, loads catalog metadata in the background. If false, metadata is loaded lazily (on access). Only effective in CDH 5 and Impala 1.2.4 and higher.", + "display_name": "Load Catalog in Background", + "name": "load_catalog_in_background", + "value": "true" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "Local path to the certificate presented by the Catalogd debug webserver. This file must be in .pem format. If empty, webserver SSL/TLS support is not enabled.", + "display_name": "SSL/TLS Certificate for Catalog Server Webserver", + "name": "webserver_certificate_file", + "value": null + }, + { + "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", + "display_name": "Web Metric Collection", + "name": "catalogserver_web_metric_collection_enabled", + "value": "true" + }, + { + "desc": "The amount of time to wait for the Catalog Server to fully start up and connect to the StateStore before enforcing the connectivity check.", + "display_name": "Catalog Server Connectivity Tolerance at Startup", + "name": "catalogserver_connectivity_tolerance", + "value": "180" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "true" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Timeout for requests to the Hive Metastore Server from Catalog Server. Consider increasing this if you have tables with a lot of metadata and see timeout errors.", + "display_name": "Catalog Server Hive Metastore Connection Timeout", + "name": "hive_metastore_timeout", + "value": "3600" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "Username for Catalog Server web server authentication.", + "display_name": "Catalog Server Web Server Username", + "name": "webserver_htpassword_user", + "value": null + }, + { + "desc": "Password for Catalog Server web server authentication.", + "display_name": "Catalog Server Web Server User Password", + "name": "webserver_htpassword_password", + "value": null + }, + { + "desc": "Enables the health test that the Impala Catalog Server's process state is consistent with the role configuration", + "display_name": "Impala Catalog Server Process Health Test", + "name": "catalogserver_scm_health_enabled", + "value": "true" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be added (verbatim) to Catalog Server command line flags. Key names should begin with a hyphen(-). For example: -log_filename=foo.log", + "display_name": "Catalog Server Command Line Argument Advanced Configuration Snippet (Safety Valve)", + "name": "catalogd_cmd_args_safety_valve", + "value": null + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", + "display_name": "Catalog Server HDFS Advanced Configuration Snippet (Safety Valve)", + "name": "catalogd_hdfs_site_conf_safety_valve", + "value": null + }, + { + "desc": "Directory where Catalog Server will place its log files.", + "display_name": "Catalog Server Log Directory", + "name": "log_dir", + "value": "/var/log/catalogd" + }, + { + "desc": "Enables the health test that verifies the Catalog Server is connected to the StateStore", + "display_name": "Catalog Server Connectivity Health Test", + "name": "catalogserver_connectivity_health_enabled", + "value": "true" + }, + { + "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", + "display_name": "Catalog Server HBase Advanced Configuration Snippet (Safety Valve)", + "name": "catalogd_hbase_conf_safety_valve", + "value": null + }, + { + "desc": "Enable/Disable Catalog Server web server. This web server contains useful information about Catalog Server daemon.", + "display_name": "Enable Catalog Server Web Server", + "name": "catalogd_enable_webserver", + "value": "true" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "Impala Catalog Server Environment Advanced Configuration Snippet (Safety Valve)", + "name": "CATALOGSERVER_role_env_safety_valve", + "value": null + }, + { + "desc": "When computing the overall Impala Catalog Server health, consider the host's health.", + "display_name": "Impala Catalog Server Host Health Test", + "name": "catalogserver_host_health_enabled", + "value": "true" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The maximum size, in megabytes, per log file for Impala Catalog Server logs. Typically used by log4j or logback.", + "display_name": "Impala Catalog Server Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", + "display_name": "Catalog Server Hive Advanced Configuration Snippet (Safety Valve)", + "name": "catalogd_hive_conf_safety_valve", + "value": null + }, + { + "desc": "Verbose logging level for the GLog logger. These messages are always logged at 'INFO' log level, so this setting has no effect if Logging Threshold is set to 'WARN' or above.", + "display_name": "Catalog Server Verbose Log Level", + "name": "log_verbose_level", + "value": "1" + }, + { + "desc": "The health test thresholds on the resident size of the process.", + "display_name": "Resident Set Size Thresholds", + "name": "process_resident_set_size_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "The minimum log level for Impala Catalog Server logs", + "display_name": "Impala Catalog Server Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "The number of log files that are kept for each severity level before all older log files are removed. The number has to be greater than 1 to keep at least the current log file open. If set to 0, all log files are retained and log rotation is effectively disabled.", + "display_name": "Catalog Server Maximum Log Files", + "name": "max_log_files", + "value": "10" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/impala-impalad.json b/sahara/plugins/cdh/v5_4_0/resources/impala-impalad.json new file mode 100644 index 00000000..6220f199 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/impala-impalad.json @@ -0,0 +1,440 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Directories where Impala Daemon will write data such as spilling information to disk to free up memory. This can potentially be large amounts of data.. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Directories where Impala Daemon will write data such as spilling information to disk to free up memory. This can potentially be large amounts of data. Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Impala Daemon Scratch Directories Free Space Monitoring Percentage Thresholds", + "name": "impalad_scratch_directories_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "User-defined function (UDF) libraries are copied from HDFS into this local directory.", + "display_name": "Local UDF Library Dir", + "name": "local_library_dir", + "value": "/var/lib/impala/udfs" + }, + { + "desc": "Encrypt and verify the integrity of all data spilled to disk as part of a query. This feature is only supported for Impala 2.0 and higher and CDH 5.2 and higher (which includes Impala 2.0).", + "display_name": "Disk Spill Encryption", + "name": "disk_spill_encryption", + "value": "false" + }, + { + "desc": "Maximum number of query results a client may request to be cached on a per-query basis to support restarting fetches. This option guards against unreasonably large result caches requested by clients. Requests exceeding this maximum will be rejected.", + "display_name": "Result Cache Maximum Size", + "name": "impalad_result_cache_max_size", + "value": "100000" + }, + { + "desc": "Number of seconds to wait between attempts during Llama registration.", + "display_name": "Llama Registration Wait Seconds", + "name": "llama_registration_wait_secs", + "value": "3" + }, + { + "desc": "Abort Impala startup if there are improper configs or running on unsupported hardware.", + "display_name": "Abort on Config Error", + "name": "abort_on_config_error", + "value": "true" + }, + { + "desc": "The timeout used by the Cloudera Manager Agent's query monitor when communicating with the Impala Daemon web server, specified in seconds.", + "display_name": "Query Monitoring Timeout", + "name": "executing_queries_timeout_seconds", + "value": "5.0" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "The maximum size (in entries) of the Impala daemon lineage log file before a new one is created.", + "display_name": "Impala Daemon Maximum Lineage Log File Size", + "name": "max_lineage_log_file_size", + "value": "5000" + }, + { + "desc": "Directories where Impala Daemon will write data such as spilling information to disk to free up memory. This can potentially be large amounts of data.", + "display_name": "Impala Daemon Scratch Directories", + "name": "scratch_dirs", + "value": null + }, + { + "desc": "Buffer log messages logged at this level or lower (-1 means don't buffer; 0 means buffer INFO only; 1 means buffer WARNING only, ...)", + "display_name": "Impala Daemon Log Buffer Level", + "name": "logbuflevel", + "value": "0" + }, + { + "desc": "Enables lineage generation by Impala daemons. The lineage log file is placed in the directory specified by the 'Impala Daemon Lineage Log Directory' parameter.", + "display_name": "Enable Impala Lineage Generation", + "name": "enable_lineage_log", + "value": "true" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be added (verbatim) to Impala Daemon command-line flags. Key names should begin with a hyphen(-). For example: -log_filename=foo.log", + "display_name": "Impala Daemon Command Line Argument Advanced Configuration Snippet (Safety Valve)", + "name": "impalad_cmd_args_safety_valve", + "value": null + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "true" + }, + { + "desc": "Enable or disable the Impala Daemon web server. This web server contains useful information about Impala Daemon.", + "display_name": "Enable Impala Daemon Web Server", + "name": "impalad_enable_webserver", + "value": "true" + }, + { + "desc": "The location on disk of the certificate, in .pem format, used to confirm the authenticity of the LDAP server certificate. This is the Certificate Authority (CA) certificate, and it was used to sign the LDAP server certificate. If not set, Impala by default trusts all certificates supplied by the LDAP server, which means that an attacker could potentially intercept otherwise encrypted usernames and passwords.", + "display_name": "LDAP Server CA Certificate", + "name": "impalad_ldap_ca_certificate", + "value": null + }, + { + "desc": "Port on which HiveServer2 client requests are served by Impala Daemons.", + "display_name": "Impala Daemon HiveServer2 Port", + "name": "hs2_port", + "value": "21050" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "Username for Impala Daemon webserver authentication.", + "display_name": "Impala Daemon Web Server Username", + "name": "webserver_htpassword_user", + "value": null + }, + { + "desc": "Enables audit event generation by Impala daemons. The audit log file will be placed in the directory specified by 'Impala Daemon Audit Log Directory' parameter.", + "display_name": "Enable Impala Audit Event Generation", + "name": "enable_audit_event_log", + "value": "false" + }, + { + "desc": "Maximum number of seconds that Impala attempts to register or re-register with Llama. If registration is unsuccessful, Impala cancels the action with an error, which could result in an impalad startup failure or a cancelled query. A setting of -1 seconds means try indefinitely.", + "display_name": "Llama Registration Timeout Seconds", + "name": "llama_registration_timeout_secs", + "value": "30" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"any\"}" + }, + { + "desc": "When computing the overall Impala Daemon health, consider the host's health.", + "display_name": "Impala Daemon Host Health Test", + "name": "impalad_host_health_enabled", + "value": "true" + }, + { + "desc": "The directory in which Impala daemon audit event log files are written. If \"Impala Audit Event Generation\" property is enabled, Impala will generate its audit logs in this directory.", + "display_name": "Impala Daemon Audit Log Directory", + "name": "audit_event_log_dir", + "value": "/var/log/impalad/audit" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "An XML string to use verbatim for the contents of fair-scheduler.xml for Impala Daemons. This configuration only has effect on Impala versions 1.3 or greater.", + "display_name": "Impala Daemon Fair Scheduler Advanced Configuration", + "name": "impalad_fair_scheduler_safety_valve", + "value": null + }, + { + "desc": "A list of key-value pairs of additional query options to pass to the Impala Daemon command line, separated by ','.", + "display_name": "Impala Daemon Query Options Advanced Configuration Snippet (Safety Valve)", + "name": "default_query_options", + "value": "" + }, + { + "desc": "Port on which Beeswax client requests are served by Impala Daemons.", + "display_name": "Impala Daemon Beeswax Port", + "name": "beeswax_port", + "value": "21000" + }, + { + "desc": "Port where StateStoreSubscriberService is running.", + "display_name": "StateStoreSubscriber Service Port", + "name": "state_store_subscriber_port", + "value": "23000" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "Impala Daemon Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "Enables the health check that determines if the Impala daemon is ready to process queries.", + "display_name": "Impala Daemon Ready Status Health Check", + "name": "impalad_ready_status_check_enabled", + "value": "true" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Port on which ImpalaBackendService is exported.", + "display_name": "Impala Daemon Backend Port", + "name": "be_port", + "value": "22000" + }, + { + "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", + "display_name": "Web Metric Collection", + "name": "impalad_web_metric_collection_enabled", + "value": "true" + }, + { + "desc": "An XML snippet to append to llama-site.xml for Impala Daemons. This configuration only has effect on Impala versions 1.3 or greater.", + "display_name": "Impala Daemon Llama Site Advanced Configuration", + "name": "impala_llama_site_conf_safety_valve", + "value": null + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "Address of the load balancer used for Impala daemons. Should be specified in host:port format. If this is specified and Kerberos is enabled, Cloudera Manager adds a principal for 'impala/<load_balancer_host>@<realm>' to the keytab for all Impala daemons.", + "display_name": "Impala Daemons Load Balancer", + "name": "impalad_load_balancer", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "Maximum number of times a request to reserve, expand, or release resources is attempted until the request is cancelled.", + "display_name": "Llama Maximum Request Attempts", + "name": "llama_max_request_attempts", + "value": "5" + }, + { + "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", + "display_name": "Impala Daemon HBase Advanced Configuration Snippet (Safety Valve)", + "name": "impalad_hbase_conf_safety_valve", + "value": null + }, + { + "desc": "The maximum size, in megabytes, per log file for Impala Daemon logs. Typically used by log4j or logback.", + "display_name": "Impala Daemon Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "Impala Daemon Environment Advanced Configuration Snippet (Safety Valve)", + "name": "IMPALAD_role_env_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds on the duration of the metrics request to the web server.", + "display_name": "Web Metric Collection Duration", + "name": "impalad_web_metric_collection_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" + }, + { + "desc": "The polling period of the Impala query monitor in the Cloudera Manager Agent, specified in seconds. If set to zero, query monitoring is disabled.", + "display_name": "Query Monitoring Period", + "name": "query_monitoring_period_seconds", + "value": "1.0" + }, + { + "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", + "display_name": "Impala Daemon Hive Advanced Configuration Snippet (Safety Valve)", + "name": "impala_hive_conf_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "Directory where Impala Daemon will place its log files.", + "display_name": "Impala Daemon Log Directory", + "name": "log_dir", + "value": "/var/log/impalad" + }, + { + "desc": "The amount of time at Impala Daemon startup allowed for the Impala Daemon to start accepting new queries for processing.", + "display_name": "Impala Daemon Ready Status Startup Tolerance", + "name": "impalad_ready_status_check_startup_tolerance", + "value": "180" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Enables the health test that verifies the Impala Daemon is connected to the StateStore.", + "display_name": "Impala Daemon Connectivity Health Test", + "name": "impalad_connectivity_health_enabled", + "value": "true" + }, + { + "desc": "Port where Impala debug web server runs.", + "display_name": "Impala Daemon HTTP Server Port", + "name": "impalad_webserver_port", + "value": "25000" + }, + { + "desc": "Verbose logging level for the GLog logger. These messages are always logged at 'INFO' log level, so this setting has no effect if Logging Threshold is set to 'WARN' or above. ", + "display_name": "Impala Daemon Verbose Log Level", + "name": "log_verbose_level", + "value": "1" + }, + { + "desc": "The health test thresholds on the resident size of the process.", + "display_name": "Resident Set Size Thresholds", + "name": "process_resident_set_size_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Enables the health test that the Impala Daemon's process state is consistent with the role configuration", + "display_name": "Impala Daemon Process Health Test", + "name": "impalad_scm_health_enabled", + "value": "true" + }, + { + "desc": "The maximum size (in queries) of the Impala Daemon audit event log file before a new one is created.", + "display_name": "Impala Daemon Maximum Audit Log File Size", + "name": "max_audit_event_log_file_size", + "value": "5000" + }, + { + "desc": "Timeout for requests to the Hive Metastore Server from Impala. Consider increasing this if you have tables with a lot of metadata and see timeout errors.", + "display_name": "Impala Daemon Hive Metastore Connection Timeout", + "name": "hive_metastore_timeout", + "value": "3600" + }, + { + "desc": "Memory limit in bytes for Impala Daemon, enforced by the daemon itself. If reached, queries running on the Impala Daemon may be killed. Leave it blank to let Impala pick its own limit. Use a value of -1 B to specify no limit.", + "display_name": "Impala Daemon Memory Limit", + "name": "impalad_memory_limit", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The amount of time to wait for the Impala Daemon to fully start up and connect to the StateStore before enforcing the connectivity check.", + "display_name": "Impala Daemon Connectivity Tolerance at Startup", + "name": "impalad_connectivity_tolerance", + "value": "180" + }, + { + "desc": "Port where Llama notification callback should be started", + "display_name": "Llama Callback Port", + "name": "llama_callback_port", + "value": "28000" + }, + { + "desc": "Local path to the certificate presented by the Impala daemon debug webserver. This file must be in .pem format. If empty, webserver SSL/TLS support is not enabled.", + "display_name": "SSL/TLS Certificate for Impala Daemon Webserver", + "name": "webserver_certificate_file", + "value": null + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "impalad_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Directories where Impala Daemon will write data such as spilling information to disk to free up memory. This can potentially be large amounts of data..", + "display_name": "Impala Daemon Scratch Directories Free Space Monitoring Absolute Thresholds", + "name": "impalad_scratch_directories_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", + "display_name": "Impala Daemon HDFS Advanced Configuration Snippet (Safety Valve)", + "name": "impala_hdfs_site_conf_safety_valve", + "value": null + }, + { + "desc": "The directory in which Impala daemon lineage log files are written. If \"Impala Lineage Generation\" property is enabled, Impala generates its lineage logs in this directory.", + "display_name": "Impala Daemon Lineage Log Directory", + "name": "lineage_event_log_dir", + "value": "/var/log/impalad/lineage" + }, + { + "desc": "Password for Impala Daemon webserver authentication.", + "display_name": "Impala Daemon Web Server User Password", + "name": "webserver_htpassword_password", + "value": null + }, + { + "desc": "The minimum log level for Impala Daemon logs", + "display_name": "Impala Daemon Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "The number of log files that are kept for each severity level before all older log files are removed. The number has to be greater than 1 to keep at least the current log file open. If set to 0, all log files are retained and log rotation is effectively disabled.", + "display_name": "Impala Maximum Log Files", + "name": "max_log_files", + "value": "10" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/impala-llama.json b/sahara/plugins/cdh/v5_4_0/resources/impala-llama.json new file mode 100644 index 00000000..30cf4039 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/impala-llama.json @@ -0,0 +1,368 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Kerberos principal used by the Impala Llama ApplicationMaster roles.", + "display_name": "Role-Specific Kerberos Principal", + "name": "kerberos_role_princ_name", + "value": "llama" + }, + { + "desc": "Whether Llama should cache allocated resources on release.", + "display_name": "Enable Resource Caching", + "name": "llama_am_cache_enabled", + "value": "true" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "Directory where Llama will place its log files.", + "display_name": "Llama Log Directory", + "name": "llama_log_dir", + "value": "/var/log/impala-llama" + }, + { + "desc": "Queues Llama ApplicationMaster should connect to at start up.", + "display_name": "Core Queues", + "name": "llama_am_core_queues", + "value": "" + }, + { + "desc": "Maximum amount of time the backed off reservations will be in 'backed off' state. The actual amount time is a random value between the minimum and the maximum.", + "display_name": "Anti-Deadlock Maximum Delay", + "name": "llama_am_gang_anti_deadlock_max_delay_ms", + "value": "30000" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "ACL for Impala ApplicationMaster clients. The ACL is a comma-separated list of user and group names. The user and group list is separated by a blank. For e.g. \"alice,bob users,wheel\". A special value of \"*\" means all users are allowed. These take effect only if security is enabled.", + "display_name": "Client ACLs", + "name": "llama_am_server_thrift_client_acl", + "value": "*" + }, + { + "desc": "For advanced use only, a string to be inserted into llama-site.xml for this role only.", + "display_name": "Impala Llama ApplicationMaster Advanced Configuration Snippet (Safety Valve) for llama-site.xml", + "name": "llama_config_valve", + "value": null + }, + { + "desc": "When computing the overall Impala Llama ApplicationMaster health, consider the host's health.", + "display_name": "Impala Llama ApplicationMaster Host Health Test", + "name": "llama_host_health_enabled", + "value": "true" + }, + { + "desc": "Port on which the Llama ApplicationMaster listens to HTTP requests.", + "display_name": "Llama HTTP Port", + "name": "llama_http_port", + "value": "15001" + }, + { + "desc": "Minimum number of threads used by the Llama ApplicationMaster auxiliary service uses for serving client requests.", + "display_name": "Thrift Server Minimum Threads", + "name": "llama_am_server_thrift_server_min_threads", + "value": "10" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "Enter an XML string that will be inserted verbatim into the Fair Scheduler allocations file. Overrides the configuration set using the Pools configuration UI. This configuration only has effect on Impala versions 1.3 or greater.", + "display_name": "Fair Scheduler XML Advanced Configuration Snippet (Safety Valve)", + "name": "llama_fair_scheduler_safety_valve", + "value": null + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "true" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Client notification retry interval, in milliseconds.", + "display_name": "Client Notification Retry Interval", + "name": "llama_am_server_thrift_client_notifier_retry_interval_ms", + "value": "5000" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "The group that the Llama processes should run as.", + "display_name": "Llama System Group", + "name": "process_groupname", + "value": "llama" + }, + { + "desc": "Timeout policy for resources being cached.", + "display_name": "Resource Caching Idle Timeout", + "name": "llama_am_cache_eviction_timeout_policy_idle_timeout_ms", + "value": "30000" + }, + { + "desc": "Port on which the Llama ApplicationMaster listens to administrative requests on its administrative Thrift interface.", + "display_name": "Llama Thrift Admin Port", + "name": "llama_am_server_thrift_admin_address", + "value": "15002" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of Impala Llama ApplicationMaster in Bytes", + "name": "llama_java_heapsize", + "value": "268435456" + }, + { + "desc": "Interval of time without any new allocation that will trigger the Impala ApplicationMaster anti-deadlock logic.", + "display_name": "Anti-Deadlock No Allocation Limit Interval", + "name": "llama_am_gang_anti_deadlock_no_allocation_limit_ms", + "value": "30000" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "Maximum number of threads used by the Llama ApplicationMaster auxiliary service uses for serving client requests.", + "display_name": "Thrift Server Maximum Threads", + "name": "llama_am_server_thrift_server_max_threads", + "value": "50" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "Impala Llama ApplicationMaster Environment Advanced Configuration Snippet (Safety Valve)", + "name": "LLAMA_role_env_safety_valve", + "value": null + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Time in milliseconds after which Llama will discard its AM for a queue that has been empty of reservations. Does not apply to queues specified with the Core Queues property.", + "display_name": "Queue Expiration Age", + "name": "llama_am_queue_expire_ms", + "value": "300000" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "Impala Llama ApplicationMaster Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "Minimum amount of time the backed off reservations will be in 'backed off' state. The actual amount time is a random value between the minimum and the maximum.", + "display_name": "Anti-Deadlock Minimum Delay", + "name": "llama_am_gang_anti_deadlock_min_delay_ms", + "value": "10000" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "llama_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "If enabled, the Impala Llama ApplicationMaster binds to the wildcard address (\"0.0.0.0\") on all of its ports.", + "display_name": "Bind Impala Llama ApplicationMaster to Wildcard Address", + "name": "llama_bind_wildcard", + "value": "false" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "The maximum size, in megabytes, per log file for Impala Llama ApplicationMaster logs. Typically used by log4j or logback.", + "display_name": "Impala Llama ApplicationMaster Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for Llama Server", + "name": "llama_java_opts", + "value": "" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "Llama ApplicationMaster heartbeat interval, in milliseconds. On each heartbeat the ApplicationMaster submits new reservations to YARN ResourceManager and gets updates from it.", + "display_name": "AM Heartbeat Interval", + "name": "llama_am_server_thrift_client_notifier_heartbeat_ms", + "value": "5000" + }, + { + "desc": "The user that the Llama process should run as.", + "display_name": "Llama System User", + "name": "process_username", + "value": "llama" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Percentage of resources that will be backed off by the Impala ApplicationMaster anti-deadlock logic. Random reservations will be backed off until the percentage of backed off resources reaches this percentage.", + "display_name": "Anti-Deadlock Backoff Percentage", + "name": "llama_am_gang_anti_deadlock_backoff_percent", + "value": "30" + }, + { + "desc": "Maximum number of retries for a client notification. After the maximum number of client notification retries has been reached without success the client is considered lost and all its reservations are released. A successful client notification resets the retries count.", + "display_name": "Maximum Client Notification Retries", + "name": "llama_am_server_thrift_client_notifier_max_retries", + "value": "5" + }, + { + "desc": "Port on which the Llama ApplicationMaster serves its Thrift interface.", + "display_name": "Llama Thrift Port", + "name": "llama_port", + "value": "15000" + }, + { + "desc": "Whether to break resource requests into smaller requests of standard size before the resource cache. The sizes are taken from Yarn settings Container Memory Increment and Container Virtual CPU Cores Increment.", + "display_name": "Enable Resource Cache Normalization", + "name": "llama_am_resource_normalizing_enabled", + "value": "true" + }, + { + "desc": "The maximum number of rolled log files to keep for Impala Llama ApplicationMaster logs. Typically used by log4j or logback.", + "display_name": "Impala Llama ApplicationMaster Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "Socket time, in milliseconds, used Llama ApplicationMaster auxiliary service for all its server and client Thrift connections.", + "display_name": "Thrift Transport Timeout", + "name": "llama_am_server_thrift_transport_timeout_ms", + "value": "60000" + }, + { + "desc": "Enables the health test that the Impala Llama ApplicationMaster's process state is consistent with the role configuration", + "display_name": "Impala Llama ApplicationMaster Process Health Test", + "name": "llama_scm_health_enabled", + "value": "true" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "ACL for Impala ApplicationMaster admins. The ACL is a comma-separated list of user and group names. The user and group list is separated by a blank. For e.g. \"alice,bob users,wheel\". A special value of \"*\" means all users are allowed. These take effect only if security is enabled.", + "display_name": "Administrative Interface ACLs", + "name": "llama_am_server_thrift_admin_acl", + "value": "*" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "The minimum log level for Impala Llama ApplicationMaster logs", + "display_name": "Impala Llama ApplicationMaster Logging Threshold", + "name": "log_threshold", + "value": "INFO" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/impala-service.json b/sahara/plugins/cdh/v5_4_0/resources/impala-service.json new file mode 100644 index 00000000..0c160685 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/impala-service.json @@ -0,0 +1,356 @@ +[ + { + "desc": "The health test thresholds for the assignment locality health test. Specified as a percentage of total assignments.", + "display_name": "Assignment Locality Ratio Thresholds", + "name": "impala_assignment_locality_thresholds", + "value": "{\"critical\":\"5.0\",\"warning\":\"80.0\"}" + }, + { + "desc": "Name of the HBase service that this Impala service instance depends on", + "display_name": "HBase Service", + "name": "hbase_service", + "value": null + }, + { + "desc": "When set, this parameter is used to convert the username into the LDAP Distinguished Name (DN), so that the resulting DN looks like uid=username,X. For example, if this parameter is set to \"ou=People,dc=cloudera,dc=com\", and the username passed in is \"mike\", the resulting authentication passed to the LDAP server looks like \"uid=mike,ou=People,dc=cloudera,dc=com\". This parameter is frequently useful when authenticating against an OpenLDAP server. This parameter is mutually exclusive with LDAP Domain and LDAP Pattern.", + "display_name": "LDAP BaseDN", + "name": "ldap_baseDN", + "value": null + }, + { + "desc": "The user that this Impala's processes should run as (except Llama, which has its own user).", + "display_name": "Impala System User (except Llama)", + "name": "process_username", + "value": "impala" + }, + { + "desc": "JSON representation of all the configurations that the Fair Scheduler can take on across all schedules. Typically edited using the Pools configuration UI. This configuration only has effect on Impala versions 1.3 or greater.", + "display_name": "Fair Scheduler Allocations", + "name": "impala_scheduled_allocations", + "value": "{\"defaultMinSharePreemptionTimeout\":null,\"defaultQueueSchedulingPolicy\":null,\"fairSharePreemptionTimeout\":null,\"queueMaxAMShareDefault\":null,\"queueMaxAppsDefault\":null,\"queuePlacementRules\":null,\"queues\":[{\"aclAdministerApps\":null,\"aclSubmitApps\":null,\"minSharePreemptionTimeout\":null,\"name\":\"root\",\"queues\":[{\"aclAdministerApps\":null,\"aclSubmitApps\":null,\"minSharePreemptionTimeout\":null,\"name\":\"default\",\"queues\":[],\"schedulablePropertiesList\":[{\"impalaMaxMemory\":null,\"impalaMaxQueuedQueries\":null,\"impalaMaxRunningQueries\":null,\"maxAMShare\":null,\"maxResources\":null,\"maxRunningApps\":null,\"minResources\":null,\"scheduleName\":\"default\",\"weight\":null}],\"schedulingPolicy\":null}],\"schedulablePropertiesList\":[{\"impalaMaxMemory\":null,\"impalaMaxQueuedQueries\":null,\"impalaMaxRunningQueries\":null,\"maxAMShare\":null,\"maxResources\":null,\"maxRunningApps\":null,\"minResources\":null,\"scheduleName\":\"default\",\"weight\":null}],\"schedulingPolicy\":null}],\"userMaxAppsDefault\":null,\"users\":[]}" + }, + { + "desc": "

Event filters are defined in a JSON object like the following:

\n\n
\n{\n  \"defaultAction\" : (\"accept\", \"discard\"),\n  \"rules\" : [\n    {\n      \"action\" : (\"accept\", \"discard\"),\n      \"fields\" : [\n        {\n          \"name\" : \"fieldName\",\n          \"match\" : \"regex\"\n        }\n      ]\n    }\n  ]\n}\n
\n\n

\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n

\n\n

\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n

\n\n

\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n

\n\n

\nThe following is the list of fields that can be filtered for Impala events:\n

\n\n\n", + "display_name": "Audit Event Filter", + "name": "navigator_audit_event_filter", + "value": null + }, + { + "desc": "

\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n

\n\n

\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n

\n\n

Event trackers are defined in a JSON object like the following:

\n\n
\n{\n  \"timeToLive\" : [integer],\n  \"fields\" : [\n    {\n      \"type\" : [string],\n      \"name\" : [string]\n    }\n  ]\n}\n
\n\n

\nWhere:\n

\n\n\n\n

\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n

\n\n\n\n

\nThe following is the list of fields that can be used to compare Impala events:\n

\n\n\n\n", + "display_name": "Audit Event Tracker", + "name": "navigator_event_tracker", + "value": null + }, + { + "desc": "For advanced use only, a string to be inserted into sentry-site.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Impala Service Advanced Configuration Snippet (Safety Valve) for sentry-site.xml", + "name": "impalad_sentry_safety_valve", + "value": null + }, + { + "desc": "Use Impala Admission Control to throttle Impala requests. Unless 'Enable Dynamic Resource Pools' is enabled, Impala uses a single, default pool that is configured using the Single Pool configurations below. These features are only supported in Impala 1.3 or higher deployments.", + "display_name": "Enable Impala Admission Control", + "name": "all_admission_control_enabled", + "value": "false" + }, + { + "desc": "For advanced use only, a string to be inserted into the client configuration for navigator.client.properties.", + "display_name": "Impala Client Advanced Configuration Snippet (Safety Valve) for navigator.client.properties", + "name": "navigator_client_config_safety_valve", + "value": null + }, + { + "desc": "Configures the maximum number of queued queries for admission control when using a single pool. -1 or 0 disables queuing, i.e. incoming requests are rejected if they can not be executed immediately. Ignored when Dynamic Resource Pools for Admission Control is enabled.", + "display_name": "Single Pool Max Queued Queries", + "name": "admission_control_single_pool_max_queued", + "value": "200" + }, + { + "desc": "Name of the Sentry service that this Impala service instance depends on. If selected, Impala uses this Sentry service to look up authorization privileges. Before enabling Sentry, read the requirements and configuration steps in Setting Up The Sentry Service.", + "display_name": "Sentry Service", + "name": "sentry_service", + "value": null + }, + { + "desc": "When set, this value is appended to all usernames before authenticating with the LDAP server. For example, if this parameter is set to \"my.domain.com\", and the user authenticating to the Impala daemon is \"mike\", then \"mike@my.domain.com\" is passed to the LDAP server. If this field is unset, the username remains unaltered before being passed to the LDAP server. This parameter is mutually exclusive with LDAP BaseDN and LDAP Pattern.", + "display_name": "LDAP Domain", + "name": "ldap_domain", + "value": null + }, + { + "desc": "The health test thresholds of the overall Impala Daemon health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Impala Daemons falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Impala Daemons falls below the critical threshold.", + "display_name": "Healthy Impala Daemon Monitoring Thresholds", + "name": "impala_impalads_healthy_thresholds", + "value": "{\"critical\":\"90.0\",\"warning\":\"95.0\"}" + }, + { + "desc": "Name of the HDFS service that this Impala service instance depends on", + "display_name": "HDFS Service", + "name": "hdfs_service", + "value": null + }, + { + "desc": "When computing the overall IMPALA health, consider Impala StateStore's health", + "display_name": "Impala StateStore Role Health Test", + "name": "impala_statestore_health_enabled", + "value": "true" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Impala Service Environment Advanced Configuration Snippet (Safety Valve)", + "name": "impala_service_env_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds of the overall Impala Llama ApplicationMaster health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Impala Llama ApplicationMasters falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Impala Llama ApplicationMasters falls below the critical threshold.", + "display_name": "Healthy Impala Llama ApplicationMaster Monitoring Thresholds", + "name": "impala_llamas_healthy_thresholds", + "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" + }, + { + "desc": "Maximum amount of time (in milliseconds) that a request waits to be admitted before timing out. Must be a positive integer.", + "display_name": "Admission Control Queue Timeout", + "name": "admission_control_queue_timeout", + "value": "60000" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Use Dynamic Resource Pools to configure Impala admission control or RM for this Impala service. These features are only supported in Impala 1.3 or higher deployments.", + "display_name": "Enable Dynamic Resource Pools", + "name": "admission_control_enabled", + "value": "false" + }, + { + "desc": "Name of the Zookeeper service to use for leader election and fencing when Llama is configured for High Availability. This service dependency is required when more than one Llama role is present.", + "display_name": "Zookeeper Service for Llama HA", + "name": "zookeeper_service", + "value": null + }, + { + "desc": "Number of minutes between reestablishing our ticket with the Kerberos server.", + "display_name": "Kerberos Re-init Interval", + "name": "kerberos_reinit_interval", + "value": "60" + }, + { + "desc": "Time in seconds before Impala Daemon or Catalog Server times out with the StateStore.", + "display_name": "StateStoreSubscriber Timeout", + "name": "statestore_subscriber_timeout", + "value": "30" + }, + { + "desc": "The time period over which to compute the assignment locality ratio. Specified in minutes.", + "display_name": "Assignment Locality Monitoring Period", + "name": "impala_assignment_locality_window", + "value": "15" + }, + { + "desc": "Kerberos principal used by all roles of this service.", + "display_name": "Kerberos Principal", + "name": "kerberos_princ_name", + "value": "impala" + }, + { + "desc": "Configures the max memory of running queries for admission control when using a single pool. -1 or 0 indicates no limit. Ignored when Dynamic Resource Pools for Admission Control is enabled.", + "display_name": "Single Pool Mem Limit", + "name": "admission_control_single_pool_mem_limit", + "value": "-1" + }, + { + "desc": "Use Sentry to enable role-based, fine-grained authorization. This configuration enables Sentry using policy files. To enable Sentry using Sentry service instead, add Sentry service as a dependency to Impala service. Sentry service provides concurrent and secure access to authorization policy metadata and is the recommended option for enabling Sentry. Sentry is supported only on Impala 1.1 or later deployments.", + "display_name": "Enable Sentry Authorization using Policy Files", + "name": "sentry_enabled", + "value": "false" + }, + { + "desc": "When set, this parameter allows arbitrary mapping from usernames into a Distinguished Name (DN). The string specified must have a placeholder named \"#UID\" inside it, and that #UID is replaced with the username. For example, you could mimic the behavior of LDAP BaseDN by specifying \"uid=#UID,ou=People,dc=cloudera,dc=com\". When the username of \"mike\" comes in, it replaces the #UID and the result is \"uid=mike,ou=People,dc=cloudera,dc=com\". This option should be used when more control over the DN is needed. This parameter is mutually exclusive with LDAP Domain and LDAP BaseDN.", + "display_name": "LDAP Pattern", + "name": "ldap_bind_pattern", + "value": null + }, + { + "desc": "The URI of the LDAP server to use if LDAP is enabled. The URI must be prefixed with ldap:// or ldaps://. The URI can optionally specify the port, for example: ldap://ldap_server.example.com:389. ldaps:// is only supported in Impala 1.4 or CDH 5.1 or higher, and usually requires that you specify a port.", + "display_name": "LDAP URI", + "name": "impala_ldap_uri", + "value": null + }, + { + "desc": "Maximum number of HBase client retries for Impala. Used as a maximum for all operations such as fetching of the root region from the root RegionServer, getting a cell's value, and starting a row update. Overrides configuration in the HBase service.", + "display_name": "Maximum HBase Client Retries", + "name": "hbase_client_retries_number", + "value": "3" + }, + { + "desc": "Controls which queries a non-admin user can see in the queries list view", + "display_name": "Non-Admin Users Query List Visibility Settings", + "name": "user_query_list_settings", + "value": "ALL" + }, + { + "desc": "The group that this Impala's processes should run as (except Llama, which has its own group).", + "display_name": "Impala System Group (except Llama)", + "name": "process_groupname", + "value": "impala" + }, + { + "desc": "Specifies the set of authorized proxy users (users who can impersonate other users during authorization) and whom they are allowed to impersonate. Input is a semicolon-separated list of key=value pairs of authorized proxy users to the user(s) they can impersonate. These users are specified as a comma separated list of short usernames, or '*' to indicate all users. For example: joe=alice,bob;hue=*;admin=*. Only valid when Sentry is enabled.", + "display_name": "Proxy User Configuration", + "name": "impala_authorized_proxy_user_config", + "value": "hue=*" + }, + { + "desc": "Controls the aggregate metrics generated for Impala queries. The structure is a JSON list of the attributes to aggregate and the entities to aggregate to. For example, if the attributeName is 'hdfs_bytes_read' and the aggregationTargets is ['USER'] then the Service Monitor will create the metric 'impala_query_hdfs_bytes_read_rate' and, every ten minutes, will record the total hdfs bytes read for each user across all their Impala queries. By default it will also record the number of queries issues ('num_impala_queries_rate') for both users and pool. For a full list of the supported attributes see the Impala search page. Note that the valid aggregation targets are USER, YARN_POOL, and IMPALA (the service), and that these aggregate metrics can be viewed on both the reports and charts search pages.", + "display_name": "Impala Query Aggregates", + "name": "impala_query_aggregates", + "value": "[\n {\n \"attributeName\": \"hdfs_bytes_read\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"IMPALA\"]\n },\n {\n \"attributeName\": \"hdfs_bytes_written\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"IMPALA\"]\n },\n {\n \"attributeName\": \"thread_cpu_time\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"IMPALA\"]\n },\n {\n \"attributeName\": \"bytes_streamed\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"IMPALA\"]\n },\n {\n \"attributeName\": \"cm_cpu_milliseconds\",\n \"aggregationTargets\": [\"USER\"]\n }\n]\n" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Service Level Health Alerts", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be added (verbatim) to Impala Daemon command-line flags. Applies to all roles in this service. Key names should begin with a hyphen(-). For example: -log_filename=foo.log", + "display_name": "Impala Command Line Argument Advanced Configuration Snippet (Safety Valve)", + "name": "impala_cmd_args_safety_valve", + "value": null + }, + { + "desc": "Used to generate a core dump to get more information about an Impala crash. Unless otherwise configured systemwide using /proc/sys/kernel/core_pattern, the dump is generated in the 'current directory' of the Impala process (usually a subdirectory of the /var/run/cloudera-scm-agent/process directory). The core file can be very large.", + "display_name": "Enable Core Dump", + "name": "enable_core_dump", + "value": "false" + }, + { + "desc": "Use debug build of Impala binaries when starting roles. Useful when performing diagnostic activities to get more information in the stacktrace or core dump.", + "display_name": "Use Debug Build", + "name": "use_debug_build", + "value": "false" + }, + { + "desc": "Enable HDFS short circuit read. This allows a client co-located with the DataNode to read HDFS file blocks directly. This gives a performance boost to distributed clients that are aware of locality.", + "display_name": "Enable HDFS Short Circuit Read", + "name": "dfs_client_read_shortcircuit", + "value": "true" + }, + { + "desc": "Action to take when the audit event queue is full. Drop the event or shutdown the affected process.", + "display_name": "Audit Queue Policy", + "name": "navigator_audit_queue_policy", + "value": "DROP" + }, + { + "desc": "Local path to the private key that matches the certificate specified in the Certificate for Clients. This file must be in PEM format, and is required if the SSL/TLS Certificate for Clients is supplied.", + "display_name": "SSL/TLS Private Key for Clients", + "name": "ssl_private_key", + "value": null + }, + { + "desc": "

The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n  \"streamThreshold\": 10, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Service Triggers", + "name": "service_triggers", + "value": "[]" + }, + { + "desc": "If true, attempts to establish a TLS (Transport Layer Security) connection with the LDAP server. Only supported in Impala 1.4 or CDH 5.1 or higher. Not required when using an LDAP URI with prefix ldaps://, because that already specifies TLS.", + "display_name": "Enable LDAP TLS", + "name": "enable_ldap_tls", + "value": "false" + }, + { + "desc": "Controls which queries admin users can see in the queries list view", + "display_name": "Admin Users Query List Visibility Settings", + "name": "admin_query_list_settings", + "value": "ALL" + }, + { + "desc": "Encrypt communication between clients (like ODBC, JDBC, and the Impala shell) and the Impala daemon using Transport Layer Security (TLS) (formerly known as Secure Socket Layer (SSL)).", + "display_name": "Enable TLS/SSL for Impala Client Services", + "name": "client_services_ssl_enabled", + "value": "false" + }, + { + "desc": "The minimum number of assignments that must occur during the test time period before the threshold values will be checked. Until this number of assignments have been observed in the test time period the health test will be disabled.", + "display_name": "Assignment Locality Minimum Assignments", + "name": "impala_assignment_locality_minimum", + "value": "10" + }, + { + "desc": "Enable collection of lineage from the service's roles.", + "display_name": "Enable Lineage Collection", + "name": "navigator_lineage_enabled", + "value": "true" + }, + { + "desc": "Name of YARN service to use for resource management integration between Impala and YARN. This service dependency and the existence of a Llama role is required for using said integration.", + "display_name": "YARN Service for Resource Management", + "name": "yarn_service", + "value": null + }, + { + "desc": "For advanced use only, a string to be inserted into the client configuration for navigator.lineage.client.properties.", + "display_name": "Impala Client Advanced Configuration Snippet (Safety Valve) for navigator.lineage.client.properties", + "name": "navigator_lineage_client_config_safety_valve", + "value": null + }, + { + "desc": "Local path to the X509 certificate that will identify the Impala daemon to clients during SSL/TLS connections. This file must be in PEM format.", + "display_name": "SSL/TLS Certificate for Clients", + "name": "ssl_server_certificate", + "value": null + }, + { + "desc": "Timeout in milliseconds for all HBase RPCs made by Impala. Overrides configuration in HBase service.", + "display_name": "HBase RPC Timeout", + "name": "hbase_rpc_timeout", + "value": "3000" + }, + { + "desc": "Name of the Hive service that this Impala service instance depends on", + "display_name": "Hive Service", + "name": "hive_service", + "value": null + }, + { + "desc": "Enable collection of audit events from the service's roles.", + "display_name": "Enable Audit Collection", + "name": "navigator_audit_enabled", + "value": "true" + }, + { + "desc": "A list specifying the rules to run to determine which Fair Scheduler configuration to use. Typically edited using the Rules configuration UI. This configuration only has effect on Impala versions 1.3 or greater.", + "display_name": "Fair Scheduler Configuration Rules", + "name": "impala_schedule_rules", + "value": "[]" + }, + { + "desc": "When checked, LDAP-based authentication for users is enabled. Usernames and passwords are transmitted in the clear unless encryption is turned on. To encrypt the network traffic from the Impala daemon to the LDAP server, use either an ldaps:// URI or select 'Enable LDAP TLS'. To encrypt network traffic from clients to the Impala daemon, specify 'Enable TLS/SSL for Impala Client Services'.", + "display_name": "Enable LDAP Authentication", + "name": "enable_ldap_auth", + "value": "false" + }, + { + "desc": "When computing the overall IMPALA health, consider Impala Catalog Server's health", + "display_name": "Impala Catalog Server Role Health Test", + "name": "impala_catalogserver_health_enabled", + "value": "true" + }, + { + "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", + "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", + "name": "smon_derived_configs_safety_valve", + "value": null + }, + { + "desc": "Configures the maximum number of concurrently running queries for admission control when using a single pool. -1 indicates no limit and 0 indicates all incoming requests will be rejected. Ignored when Dynamic Resource Pools for Admission Control is enabled.", + "display_name": "Single Pool Max Running Queries", + "name": "admission_control_single_pool_max_requests", + "value": "200" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/impala-statestore.json b/sahara/plugins/cdh/v5_4_0/resources/impala-statestore.json new file mode 100644 index 00000000..3f650226 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/impala-statestore.json @@ -0,0 +1,236 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", + "display_name": "Web Metric Collection", + "name": "statestore_web_metric_collection_enabled", + "value": "true" + }, + { + "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", + "display_name": "Health Check Startup Tolerance", + "name": "statestore_startup_tolerance", + "value": "5" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be added (verbatim) to StateStore command line flags.", + "display_name": "Statestore Command Line Argument Advanced Configuration Snippet (Safety Valve)", + "name": "statestore_cmd_args_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "statestore_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "Enables the health test that the Impala StateStore's process state is consistent with the role configuration", + "display_name": "Impala StateStore Process Health Test", + "name": "statestore_scm_health_enabled", + "value": "true" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Port where StateStoreService is exported.", + "display_name": "StateStore Service Port", + "name": "state_store_port", + "value": "24000" + }, + { + "desc": "When computing the overall Impala StateStore health, consider the host's health.", + "display_name": "Impala StateStore Host Health Test", + "name": "statestore_host_health_enabled", + "value": "true" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "true" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "Username for Statestore webserver authentication.", + "display_name": "Statestore Web Server Username", + "name": "webserver_htpassword_user", + "value": null + }, + { + "desc": "Local path to the certificate presented by the StateStore debug webserver. This file must be in .pem format. If empty, webserver SSL/TLS support is not enabled.", + "display_name": "SSL/TLS Certificate for Statestore Webserver", + "name": "webserver_certificate_file", + "value": null + }, + { + "desc": "Maximum number of tasks allowed to be pending at the thread manager underlying the StateStore Thrift server (0 allows infinitely many pending tasks)", + "display_name": "Maximum StateStore Pending Tasks", + "name": "state_store_pending_task_count_max", + "value": "0" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "Port where StateStore debug web server runs.", + "display_name": "StateStore HTTP Server Port", + "name": "statestore_webserver_port", + "value": "25010" + }, + { + "desc": "Directory where StateStore will place its log files.", + "display_name": "StateStore Log Directory", + "name": "log_dir", + "value": "/var/log/statestore" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The maximum size, in megabytes, per log file for Impala StateStore logs. Typically used by log4j or logback.", + "display_name": "Impala StateStore Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "Number of worker threads for the thread manager underlying the StateStore Thrift server.", + "display_name": "StateStore Worker Threads", + "name": "state_store_num_server_worker_threads", + "value": "4" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Verbose logging level for the GLog logger. These messages are always logged at 'INFO' log level, so this setting has no effect if Logging Threshold is set to 'WARN' or above. ", + "display_name": "StateStore Verbose Log Level", + "name": "log_verbose_level", + "value": "1" + }, + { + "desc": "The health test thresholds on the resident size of the process.", + "display_name": "Resident Set Size Thresholds", + "name": "process_resident_set_size_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Buffer log messages logged at this level or lower (-1 means don't buffer; 0 means buffer INFO only; 1 means buffer WARNING only, ...)", + "display_name": "StateStore Log Buffer Level", + "name": "logbuflevel", + "value": "0" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Enable/Disable StateStore web server. This web server contains useful information about StateStore daemon.", + "display_name": "Enable StateStore Web Server", + "name": "statestore_enable_webserver", + "value": "true" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "Impala StateStore Environment Advanced Configuration Snippet (Safety Valve)", + "name": "STATESTORE_role_env_safety_valve", + "value": null + }, + { + "desc": "Password for Statestore webserver authentication.", + "display_name": "Statestore Web Server User Password", + "name": "webserver_htpassword_password", + "value": null + }, + { + "desc": "The minimum log level for Impala StateStore logs", + "display_name": "Impala StateStore Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "The number of log files that are kept for each severity level before all older log files are removed. The number has to be greater than 1 to keep at least the current log file open. If set to 0, all log files are retained and log rotation is effectively disabled.", + "display_name": "StateStore Maximum Log Files", + "name": "max_log_files", + "value": "10" + }, + { + "desc": "The health test thresholds on the duration of the metrics request to the web server.", + "display_name": "Web Metric Collection Duration", + "name": "statestore_web_metric_collection_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/ks_indexer-hbase_indexer.json b/sahara/plugins/cdh/v5_4_0/resources/ks_indexer-hbase_indexer.json new file mode 100644 index 00000000..50bca7d8 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/ks_indexer-hbase_indexer.json @@ -0,0 +1,242 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Kerberos principal used by the Lily HBase Indexer roles.", + "display_name": "Role-Specific Kerberos Principal", + "name": "kerberos_role_princ_name", + "value": "hbase" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "HTTP port used by HBase Indexer.", + "display_name": "HBase Indexer HTTP Port", + "name": "hbase_indexer_http_port", + "value": "11060" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "hbase_indexer_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "The maximum number of rolled log files to keep for Lily HBase Indexer logs. Typically used by log4j or logback.", + "display_name": "Lily HBase Indexer Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "The group that the HBase Indexer process should run as.", + "display_name": "System Group", + "name": "hbase_indexer_process_groupname", + "value": "hbase" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for Lily HBase Indexer", + "name": "hbase_indexer_java_opts", + "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "For advanced use only, a string to be inserted into hbase-indexer-site.xml for this role only.", + "display_name": "Lily HBase Indexer Advanced Configuration Snippet (Safety Valve) for hbase-indexer-site.xml", + "name": "hbase_indexer_config_safety_valve", + "value": null + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "Lily HBase Indexer Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "Lily HBase Indexer Environment Advanced Configuration Snippet (Safety Valve)", + "name": "HBASE_INDEXER_role_env_safety_valve", + "value": null + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The maximum size, in megabytes, per log file for Lily HBase Indexer logs. Typically used by log4j or logback.", + "display_name": "Lily HBase Indexer Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "Directory where HBase Indexer will place its log files.", + "display_name": "HBase Indexer Log Directory", + "name": "hbase_indexer_log_dir", + "value": "/var/log/hbase-solr" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "When computing the overall Lily HBase Indexer health, consider the host's health.", + "display_name": "Lily HBase Indexer Host Health Test", + "name": "hbase_indexer_host_health_enabled", + "value": "true" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "Enables the health test that the Lily HBase Indexer's process state is consistent with the role configuration", + "display_name": "Lily HBase Indexer Process Health Test", + "name": "hbase_indexer_scm_health_enabled", + "value": "true" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "The minimum log level for Lily HBase Indexer logs", + "display_name": "Lily HBase Indexer Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "The user that the HBase Indexer process should run as.", + "display_name": "System User", + "name": "hbase_indexer_process_username", + "value": "hbase" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of Lily HBase Indexer in Bytes", + "name": "hbase_indexer_java_heapsize", + "value": "1073741824" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/ks_indexer-service.json b/sahara/plugins/cdh/v5_4_0/resources/ks_indexer-service.json new file mode 100644 index 00000000..4adb518f --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/ks_indexer-service.json @@ -0,0 +1,110 @@ +[ + { + "desc": "Name of the Solr service that this Key-Value Store Indexer service instance depends on", + "display_name": "Solr Service", + "name": "solr_service", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Service Level Health Alerts", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "Name of the HBase service that this Key-Value Store Indexer service instance depends on", + "display_name": "HBase Service", + "name": "hbase_service", + "value": null + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Key-Value Store Indexer Service Environment Advanced Configuration Snippet (Safety Valve)", + "name": "ks_indexer_env_safety_valve", + "value": null + }, + { + "desc": "Text that goes verbatim into custom-mimetypes.xml file used by HBase Indexers.", + "display_name": "Custom Mime-types File", + "name": "custom_mimetypes_file", + "value": "\n\n\n\n\n \n \n \n\n \n \n \n \n \n \n\n \n \n \n \n \n \n \n \n \n \n \n \n" + }, + { + "desc": "Text that goes into morphlines.conf file used by HBase Indexers. The text goes verbatim into the config file except that $ZK_HOST is replaced by the ZooKeeper quorum of the Solr service.", + "display_name": "Morphlines File", + "name": "morphlines_conf_file", + "value": "SOLR_LOCATOR : {\n # Name of solr collection\n collection : collection\n \n # ZooKeeper ensemble\n zkHost : \"$ZK_HOST\" \n}\n\n\nmorphlines : [\n{\nid : morphline\nimportCommands : [\"org.kitesdk.**\", \"com.ngdata.**\"]\n\ncommands : [ \n {\n extractHBaseCells {\n mappings : [\n {\n inputColumn : \"data:*\"\n outputField : \"data\" \n type : string \n source : value\n }\n ]\n }\n }\n\n\n { logDebug { format : \"output record: {}\", args : [\"@{}\"] } }\n]\n}\n]\n" + }, + { + "desc": "The class to use in Sentry authorization for user to group mapping. Sentry authorization may be configured to use either Hadoop groups or local groups defined in the policy file. When configured with Hadoop groups, Sentry will ask the HDFS Namenode for group mapping for a given user to determine authorization access.", + "display_name": "Sentry User to Group Mapping Class", + "name": "keystore_indexer_sentry_provider", + "value": "org.apache.sentry.provider.file.HadoopGroupResourceAuthorizationProvider" + }, + { + "desc": "The password for the HBase Indexer TLS/SSL Certificate Trust Store File. Note that this password is not required to access the trust store: this field can be left blank. This password provides optional integrity checking of the file. The contents of trust stores are certificates, and certificates are public information.", + "display_name": "HBase Indexer TLS/SSL Certificate Trust Store Password", + "name": "keystore_indexer_truststore_password", + "value": null + }, + { + "desc": "Authentication mechanism used by HBase Indexer.", + "display_name": "HBase Indexer Secure Authentication", + "name": "hbase_indexer_security_authentication", + "value": "simple" + }, + { + "desc": "The location on disk of the trust store, in .jks format, used to confirm the authenticity of TLS/SSL servers that HBase Indexer might connect to. This is used when HBase Indexer is the client in a TLS/SSL connection. This trust store must contain the certificate(s) used to sign the service(s) being connected to. If this parameter is not provided, the default list of well-known certificate authorities is used instead.", + "display_name": "HBase Indexer TLS/SSL Certificate Trust Store File", + "name": "keystore_indexer_truststore_file", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "HDFS path to the global policy file for Sentry authorization. This should be a relative path (and not a full HDFS URL). The global policy file must be in Sentry policy file format.", + "display_name": "Sentry Global Policy File", + "name": "keystore_indexer_provider_resource", + "value": "/user/hbaseindexer/sentry/sentry-provider.ini" + }, + { + "desc": "Use Sentry to enable role-based, fine-grained authorization. This configuration enables Sentry using policy files. Sentry is supported only on CDH 5.4 or later deployments and requires authentication to be turned on for HBase Indexer..", + "display_name": "Enable Sentry Authorization using Policy Files", + "name": "keystore_indexer_sentry_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds of the overall Lily HBase Indexer health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Lily HBase Indexers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Lily HBase Indexers falls below the critical threshold.", + "display_name": "Healthy Lily HBase Indexer Monitoring Thresholds", + "name": "ks_indexer_indexers_healthy_thresholds", + "value": "{\"critical\":\"90.0\",\"warning\":\"95.0\"}" + }, + { + "desc": "Text that goes verbatim into grok-dictionary.conf file used by HBase Indexers.", + "display_name": "Grok Dictionary File", + "name": "grok_dictionary_conf_file", + "value": "USERNAME [a-zA-Z0-9._-]+\nUSER %{USERNAME}\nINT (?:[+-]?(?:[0-9]+))\nBASE10NUM (?[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+)))\nNUMBER (?:%{BASE10NUM})\nBASE16NUM (?(?\"(?>\\\\.|[^\\\\\"]+)+\"|\"\"|(?>'(?>\\\\.|[^\\\\']+)+')|''|(?>`(?>\\\\.|[^\\\\`]+)+`)|``))\nUUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}\n\n# Networking\nMAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})\nCISCOMAC (?:(?:[A-Fa-f0-9]{4}\\.){2}[A-Fa-f0-9]{4})\nWINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})\nCOMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})\nIP (?/(?>[\\w_%!$@:.,-]+|\\\\.)*)+\n#UNIXPATH (?/dev/pts/%{NONNEGINT})\nBSDTTY (?>/dev/tty[pq][a-z0-9])\nTTY (?:%{BSDTTY}|%{LINUXTTY})\nWINPATH (?>[A-Za-z]+:|\\\\)(?:\\\\[^\\\\?*]*)+\nURIPROTO [A-Za-z]+(\\+[A-Za-z+]+)?\nURIHOST %{IPORHOST}(?::%{POSINT:port})?\n# uripath comes loosely from RFC1738, but mostly from what Firefox\n# doesn't turn into %XX\nURIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=#%_\\-]*)+\n#URIPARAM \\?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?\nURIPARAM \\?[A-Za-z0-9$.+!*'|(){},~#%&/=:;_?\\-\\[\\]]*\nURIPATHPARAM %{URIPATH}(?:%{URIPARAM})?\nURI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?\n\n# Months: January, Feb, 3, 03, 12, December\nMONTH \\b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\\b\nMONTHNUM (?:0?[1-9]|1[0-2])\nMONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])\n\n# Days: Monday, Tue, Thu, etc...\nDAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)\n\n# Years?\nYEAR (?>\\d\\d){1,2}\n# Time: HH:MM:SS\n#TIME \\d{2}:\\d{2}(?::\\d{2}(?:\\.\\d+)?)?\n# I'm still on the fence about using grok to perform the time match,\n# since it's probably slower.\n# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\\.%{POSINT})?)?\nHOUR (?:2[0123]|[01]?[0-9])\nMINUTE (?:[0-5][0-9])\n# '60' is a leap second in most time standards and thus is valid.\nSECOND (?:(?:[0-5][0-9]|60)(?:[:.,][0-9]+)?)\nTIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])\n# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)\nDATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}\nDATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}\nISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))\nISO8601_SECOND (?:%{SECOND}|60)\nTIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?\nDATE %{DATE_US}|%{DATE_EU}\nDATESTAMP %{DATE}[- ]%{TIME}\nTZ (?:[PMCE][SD]T)\nDATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}\nDATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}\n\n# Syslog Dates: Month Day HH:MM:SS\nSYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}\nPROG (?:[\\w._/%-]+)\nSYSLOGPROG %{PROG:program}(?:\\[%{POSINT:pid}\\])?\nSYSLOGHOST %{IPORHOST}\nSYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>\nHTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}\n\n# Shortcuts\nQS %{QUOTEDSTRING}\n\n# Log formats\nSYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:\nCOMBINEDAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \\[%{HTTPDATE:timestamp}\\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent}\n\n# Log Levels\nLOGLEVEL ([T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)" + }, + { + "desc": "For advanced use only, a string to be inserted into sentry-site.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Key-Value Store Indexer Service Advanced Configuration Snippet (Safety Valve) for sentry-site.xml", + "name": "keystore_indexer_sentry_safety_valve", + "value": null + }, + { + "desc": "

The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n  \"streamThreshold\": 10, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Service Triggers", + "name": "service_triggers", + "value": "[]" + }, + { + "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", + "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", + "name": "smon_derived_configs_safety_valve", + "value": null + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/mapred-site.xml b/sahara/plugins/cdh/v5_4_0/resources/mapred-site.xml new file mode 100644 index 00000000..5228ed17 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/mapred-site.xml @@ -0,0 +1,157 @@ + + + + + + mapreduce.job.split.metainfo.maxsize + 10000000 + + + mapreduce.job.counters.max + 120 + + + mapreduce.output.fileoutputformat.compress + false + + + mapreduce.output.fileoutputformat.compress.type + BLOCK + + + mapreduce.output.fileoutputformat.compress.codec + org.apache.hadoop.io.compress.DefaultCodec + + + mapreduce.map.output.compress.codec + org.apache.hadoop.io.compress.SnappyCodec + + + mapreduce.map.output.compress + true + + + zlib.compress.level + DEFAULT_COMPRESSION + + + mapreduce.task.io.sort.factor + 64 + + + mapreduce.map.sort.spill.percent + 0.8 + + + mapreduce.reduce.shuffle.parallelcopies + 10 + + + mapreduce.task.timeout + 600000 + + + mapreduce.client.submit.file.replication + 10 + + + mapreduce.job.reduces + 1 + + + mapreduce.task.io.sort.mb + 256 + + + mapreduce.map.speculative + false + + + mapreduce.reduce.speculative + false + + + mapreduce.job.reduce.slowstart.completedmaps + 0.8 + + + mapreduce.jobhistory.address + test-master-001.novalocal:10020 + + + mapreduce.jobhistory.webapp.address + test-master-001.novalocal:19888 + + + mapreduce.jobhistory.webapp.https.address + test-master-001.novalocal:19890 + + + mapreduce.jobhistory.admin.address + test-master-001.novalocal:10033 + + + mapreduce.framework.name + yarn + + + yarn.app.mapreduce.am.staging-dir + /user + + + yarn.app.mapreduce.am.resource.mb + 1024 + + + yarn.app.mapreduce.am.resource.cpu-vcores + 1 + + + mapreduce.job.ubertask.enable + false + + + yarn.app.mapreduce.am.command-opts + -Djava.net.preferIPv4Stack=true -Xmx825955249 + + + mapreduce.map.java.opts + -Djava.net.preferIPv4Stack=true -Xmx825955249 + + + mapreduce.reduce.java.opts + -Djava.net.preferIPv4Stack=true -Xmx825955249 + + + yarn.app.mapreduce.am.admin.user.env + LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH + + + mapreduce.map.memory.mb + 1024 + + + mapreduce.map.cpu.vcores + 1 + + + mapreduce.reduce.memory.mb + 1024 + + + mapreduce.reduce.cpu.vcores + 1 + + + mapreduce.application.classpath + $HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$MR2_CLASSPATH + + + mapreduce.admin.user.env + LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH + + + mapreduce.shuffle.max.connections + 80 + + diff --git a/sahara/plugins/cdh/v5_4_0/resources/oozie-oozie_server.json b/sahara/plugins/cdh/v5_4_0/resources/oozie-oozie_server.json new file mode 100644 index 00000000..cb5359e6 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/oozie-oozie_server.json @@ -0,0 +1,446 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Comma-separated list of Oozie plug-ins to be activated. If one plugin cannot be loaded, all the plugins are ignored.", + "display_name": "Oozie Server Plugins", + "name": "oozie_plugins_list", + "value": "" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The period over which to compute the moving average of the callable queue size.", + "display_name": "Oozie Server Callable Queue Monitoring Period", + "name": "oozie_server_callable_queue_window", + "value": "5" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "Enable SMTP authentication for Oozie email action", + "display_name": "Oozie Email Action SMTP Authentication Enabled", + "name": "oozie_email_smtp_auth", + "value": "false" + }, + { + "desc": "The health test thresholds for the weighted average extra time the pause monitor spent paused. Specified as a percentage of elapsed wall clock time.", + "display_name": "Pause Duration Thresholds", + "name": "oozie_server_pause_duration_thresholds", + "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" + }, + { + "desc": "When computing the overall Oozie Server health, consider the host's health.", + "display_name": "Oozie Server Host Health Test", + "name": "oozie_server_host_health_enabled", + "value": "true" + }, + { + "desc": "Password for connecting to the database used by Oozie Server. Does not apply if you are using Derby as the database type.", + "display_name": "Oozie Server Database Password", + "name": "oozie_database_password", + "value": "" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "SMTP username for Oozie email action", + "display_name": "Oozie Email Action SMTP Authentication Username", + "name": "oozie_email_smtp_username", + "value": null + }, + { + "desc": "The password for the Oozie TLS/SSL Certificate Trust Store File. Note that this password is not required to access the trust store: this field can be left blank. This password provides optional integrity checking of the file. The contents of trust stores are certificates, and certificates are public information.", + "display_name": "Oozie TLS/SSL Certificate Trust Store Password", + "name": "oozie_https_truststore_password", + "value": null + }, + { + "desc": "Enables the health test that the Oozie Server's process state is consistent with the role configuration", + "display_name": "Oozie Server Process Health Test", + "name": "oozie_server_scm_health_enabled", + "value": "true" + }, + { + "desc": "Number of threads used for executing callables", + "display_name": "Number Threads For Executing Callables", + "name": "oozie_service_callablequeueservice_threads", + "value": "10" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "SMTP password for Oozie email action", + "display_name": "Oozie Email Action SMTP Authentication Password", + "name": "oozie_email_smtp_password", + "value": null + }, + { + "desc": "Maximum concurrency for a given callable type. Each command is a callable type: submit, start, run, etc. Each action type is a callable type: MapReduce, SSH, sub-workflow, etc. All commands that use action executors (action-start, action-end. etc.) use the action type as the callable type.", + "display_name": "Maximum concurrency for a given callable type", + "name": "oozie_service_callablequeueservice_callable_concurrency", + "value": "3" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for Oozie Server", + "name": "oozie_java_opts", + "value": "" + }, + { + "desc": "The admin port Oozie server runs.", + "display_name": "Oozie Admin Port", + "name": "oozie_admin_port", + "value": "11001" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "The period to review when computing the moving average of extra time the pause monitor spent paused.", + "display_name": "Pause Duration Monitoring Period", + "name": "oozie_server_pause_duration_window", + "value": "5" + }, + { + "desc": "Type of the database used by Oozie Server.", + "display_name": "Oozie Server Database Type", + "name": "oozie_database_type", + "value": "derby" + }, + { + "desc": "If true, enables the Oozie Server web console. ExtJS 2.2 zip archive must be extracted to /var/lib/oozie on the same host as the Oozie Server.", + "display_name": "Enable Oozie Server Web Console", + "name": "oozie_web_console", + "value": "false" + }, + { + "desc": "The health test thresholds on the duration of the metrics request to the web server.", + "display_name": "Web Metric Collection Duration", + "name": "oozie_server_web_metric_collection_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" + }, + { + "desc": "Username for connecting to the database used by Oozie Server. Does not apply if you are using Derby as the database type.", + "display_name": "Oozie Server Database User", + "name": "oozie_database_user", + "value": "sa" + }, + { + "desc": "The location on disk of the trust store, in .jks format, used to confirm the authenticity of TLS/SSL servers that Oozie might connect to. This is used when Oozie is the client in a TLS/SSL connection. This trust store must contain the certificate(s) used to sign the service(s) being connected to. If this parameter is not provided, the default list of well-known certificate authorities is used instead.", + "display_name": "Oozie TLS/SSL Certificate Trust Store File", + "name": "oozie_https_truststore_file", + "value": null + }, + { + "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", + "display_name": "Web Metric Collection", + "name": "oozie_server_web_metric_collection_enabled", + "value": "true" + }, + { + "desc": "The health test thresholds of the weighted average size of the Oozie Server callable queue over a recent period. See also Oozie Server Callable Queue Monitoring Period.", + "display_name": "Oozie Server Callable Queue Monitoring Threshold", + "name": "oozie_server_callable_queue_threshold", + "value": "{\"critical\":\"95.0\",\"warning\":\"80.0\"}" + }, + { + "desc": "Directory where the Oozie Server places its data. Only applicable when using Derby as the database type.", + "display_name": "Oozie Server Data Directory", + "name": "oozie_data_dir", + "value": "/var/lib/oozie/data" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The password for the Oozie JKS keystore file.", + "display_name": "Oozie TLS/SSL Server JKS Keystore File Password", + "name": "oozie_https_keystore_password", + "value": null + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "Completed workflow jobs older than this value, in days, will be purged by the PurgeService.", + "display_name": "Days to Keep Completed Workflow Jobs", + "name": "purgeservice_older_than", + "value": "30" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "oozie_server_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "Oozie Server Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The path to the TLS/SSL keystore file containing the server certificate and private key used for TLS/SSL. Used when Oozie is acting as a TLS/SSL server. The keystore must be in JKS format.", + "display_name": "Oozie TLS/SSL Server JKS Keystore File Location", + "name": "oozie_https_keystore_file", + "value": "/var/lib/oozie/.keystore" + }, + { + "desc": "The from address to be used for mailing all emails for Oozie email action", + "display_name": "Oozie Email Action From Address", + "name": "oozie_email_from_address", + "value": "oozie@localhost" + }, + { + "desc": "Port of the Oozie Server while using SSL.", + "display_name": "Oozie HTTPS Port", + "name": "oozie_https_port", + "value": "11443" + }, + { + "desc": "Comma-separated list of SchemaService workflow extension schemas for additional action types. From CDH 5.4 and higher, this parameter is used only to specify additional schemas for workflows. All schemas included in that release will be added automatically and do not need to be specified.", + "display_name": "Oozie SchemaService Workflow Extension Schemas", + "name": "oozie_workflow_extension_schemas", + "value": "" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The SMTP server port to use for Oozie email action", + "display_name": "Oozie Email Action SMTP Port", + "name": "oozie_email_smtp_prt", + "value": "25" + }, + { + "desc": "Port of Oozie Server", + "display_name": "Oozie HTTP Port", + "name": "oozie_http_port", + "value": "11000" + }, + { + "desc": "The maximum number of rolled log files to keep for Oozie Server logs. Typically used by log4j or logback.", + "display_name": "Oozie Server Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "720" + }, + { + "desc": "Workflow Status metrics collection interval.", + "display_name": "Workflow Status Metrics Collection Interval", + "name": "oozie_job_metric_collection_interval", + "value": "1" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "Name of the database used by Oozie Server.", + "display_name": "Oozie Server Database Name", + "name": "oozie_database_name", + "value": "oozie" + }, + { + "desc": "Directory where Oozie Server will place its log files.", + "display_name": "Oozie Server Log Directory", + "name": "oozie_log_dir", + "value": "/var/log/oozie" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of Oozie Server in Bytes", + "name": "oozie_java_heapsize", + "value": "1073741824" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Whether to use the Codehale-based metrics for instrumentation. Enabling this disables the 'instrumentation' REST endpoint and enables the 'metrics' REST endpoint (<hostname:port>/v2/admin/metrics).", + "display_name": "Enable The Metrics Instrumentation Service", + "name": "oozie_use_metric_instrumentation", + "value": "true" + }, + { + "desc": "If true, enables version check for Oozie Server and installed shared libraries.", + "display_name": "Enable Oozie Server Shared Libraries Version Check", + "name": "oozie_server_shared_lib_version_check_enabled", + "value": "true" + }, + { + "desc": "Comma-separated list of ActionService executor extension classes. Only action types with associated executors can be used in workflows. For CDH 5.4 and higher, this parameter is used only to specify additional classes for workflows. All executor extension classes included in that release will be added automatically and do not need to be specified.", + "display_name": "Oozie ActionService Executor Extension Classes", + "name": "oozie_executor_extension_classes", + "value": "" + }, + { + "desc": "Whether to purge completed workflows and their corresponding coordinator actions of long-running coordinator jobs if the completed workflow jobs are older than the value specified in oozie.service.PurgeService.older.than.", + "display_name": "Enable Purge for Long-Running Coordinator Jobs", + "name": "purgeservice_purge_old_coord_action", + "value": "false" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "Completed bundle jobs older than this value, in days, will be purged by the PurgeService.", + "display_name": "Days to Keep Completed Bundle Jobs", + "name": "purgeservice_bundle_older_than", + "value": "7" + }, + { + "desc": "Maximum callable queue size", + "display_name": "Maximum Callable Queue Size", + "name": "oozie_service_callablequeueservice_queue_size", + "value": "10000" + }, + { + "desc": "Completed coordinator jobs older than this value, in days, will be purged by the PurgeService.", + "display_name": "Days to Keep Completed Coordinator Jobs", + "name": "purgeservice_coord_older_than", + "value": "7" + }, + { + "desc": "For advanced use only, a string to be inserted into oozie-site.xml for this role only.", + "display_name": "Oozie Server Advanced Configuration Snippet (Safety Valve) for oozie-site.xml", + "name": "oozie_config_safety_valve", + "value": null + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "The SMTP server host to use for Oozie email action", + "display_name": "Oozie Email Action SMTP Host", + "name": "oozie_email_smtp_host", + "value": "localhost" + }, + { + "desc": "The minimum log level for Oozie Server logs", + "display_name": "Oozie Server Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "Hostname of the database used by Oozie Server. If the port is non-default for your database type, use host:port notation. Does not apply if you are using Derby as the database type.", + "display_name": "Oozie Server Database Host", + "name": "oozie_database_host", + "value": "localhost" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "Oozie Server Environment Advanced Configuration Snippet (Safety Valve)", + "name": "OOZIE_SERVER_role_env_safety_valve", + "value": null + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/oozie-service.json b/sahara/plugins/cdh/v5_4_0/resources/oozie-service.json new file mode 100644 index 00000000..10533d9e --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/oozie-service.json @@ -0,0 +1,140 @@ +[ + { + "desc": "Namespace used by this Oozie service in ZooKeeper when High Availability is enabled.", + "display_name": "ZooKeeper Namespace", + "name": "oozie_zookeeper_namespace", + "value": "oozie" + }, + { + "desc": "The user that this service's processes should run as.", + "display_name": "System User", + "name": "process_username", + "value": "oozie" + }, + { + "desc": "URL of the JMS Broker used by the Oozie service in JMS integration is enabled.", + "display_name": "JMS Broker", + "name": "oozie_jms_broker", + "value": "tcp://localhost:61616" + }, + { + "desc": "Encrypt communication between clients and Oozie using Transport Layer Security (TLS) (formerly known as Secure Socket Layer (SSL)).", + "display_name": "Enable TLS/SSL for Oozie", + "name": "oozie_use_ssl", + "value": "false" + }, + { + "desc": "List of event listeners used by the Oozie service. Listeners needed for JMS or SLA integration are automatically emitted if they are enabled.", + "display_name": "Oozie Event Listeners", + "name": "oozie_event_listeners", + "value": "" + }, + { + "desc": "Name of the ZooKeeper service that this Oozie service instance depends on", + "display_name": "ZooKeeper Service", + "name": "zookeeper_service", + "value": null + }, + { + "desc": "Whether to configure Oozie properties needed for SLA integration", + "display_name": "Enable SLA Integration", + "name": "oozie_use_sla", + "value": "false" + }, + { + "desc": "Kerberos principal used by all roles of this service.", + "display_name": "Kerberos Principal", + "name": "kerberos_princ_name", + "value": "oozie" + }, + { + "desc": "Root of the directory in HDFS where the Oozie ShareLibs are stored. The libraries are stored in the share/lib subdirectory under the specified root directory.", + "display_name": "ShareLib Root Directory", + "name": "oozie_sharelib_rootdir", + "value": "/user/oozie" + }, + { + "desc": "The group that this service's processes should run as.", + "display_name": "System Group", + "name": "process_groupname", + "value": "oozie" + }, + { + "desc": "Whether to configure Oozie properties needed for JMS integration", + "display_name": "Enable JMS Integration", + "name": "oozie_use_jms", + "value": "false" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Oozie Service Environment Advanced Configuration Snippet (Safety Valve)", + "name": "oozie_env_safety_valve", + "value": null + }, + { + "desc": "A list of credential class mappings for CredentialsProvider.", + "display_name": "Oozie Credential Classes", + "name": "oozie_credential_classes", + "value": "hcat=org.apache.oozie.action.hadoop.HCatCredentials,hbase=org.apache.oozie.action.hadoop.HbaseCredentials,hive2=org.apache.oozie.action.hadoop.Hive2Credentials" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Address of the load balancer used if Oozie HA is enabled. Should be specified in host:port format.", + "display_name": "Oozie Load Balancer", + "name": "oozie_load_balancer", + "value": null + }, + { + "desc": "

The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n  \"streamThreshold\": 10, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Service Triggers", + "name": "service_triggers", + "value": "[]" + }, + { + "desc": "The health test thresholds of the overall Oozie Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Oozie Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Oozie Servers falls below the critical threshold.", + "display_name": "Healthy Oozie Server Monitoring Thresholds", + "name": "oozie_servers_healthy_thresholds", + "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Service Level Health Alerts", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "Coordinator Job Lookup trigger command is scheduled at this interval (in seconds).", + "display_name": "Coordinator Job Lookup Interval", + "name": "oozie_service_coord_lookup_interval", + "value": "300" + }, + { + "desc": "Name of the Hive service that this Oozie service instance depends on. This is used to configure Oozie HCat integration.", + "display_name": "Hive Service", + "name": "hive_service", + "value": null + }, + { + "desc": "Service to run MapReduce jobs against", + "display_name": "MapReduce Service", + "name": "mapreduce_yarn_service", + "value": null + }, + { + "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", + "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", + "name": "smon_derived_configs_safety_valve", + "value": null + }, + { + "desc": "Use ACLs on Znode while a secure ZooKeeper is used for Oozie High Availability. Note: This config is not emitted if ZooKeeper is not secure.", + "display_name": "Use ACLs on Znode", + "name": "oozie_zk_secure", + "value": "true" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/priority-one-confs.json b/sahara/plugins/cdh/v5_4_0/resources/priority-one-confs.json new file mode 100644 index 00000000..646b4aef --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/priority-one-confs.json @@ -0,0 +1,34 @@ +[ + "dfs_block_size", + "dfs_umaskmode", + "dfs_webhdfs_enabled", + "dfs_permissions", + "io_compression_codecs", + "dfs_datanode_du_reserved", + "dfs_datanode_failed_volumes_tolerated", + "dfs_name_dir_restore", + "fs_trash_interval", + "dfs_safemode_min_datanodes", + "dfs_safemode_extension", + "dfs_access_time_precision", + "yarn_acl_enable", + "yarn_admin_acl", + "yarn_log_aggregation_enable", + "yarn_log_aggregation_retain_seconds", + "mapreduce_jobhistory_max_age_ms", + "mapreduce_jobhistory_cleaner_interval", + "yarn_nodemanager_container_manager_thread_count", + "yarn_nodemanager_delete_thread_count", + "yarn_nodemanager_heartbeat_interval_ms", + "yarn_nodemanager_localizer_cache_cleanup_interval_ms", + "yarn_nodemanager_localizer_client_thread_count", + "yarn_nodemanager_localizer_cache_target_size_mb", + "yarn_nodemanager_localizer_fetch_thread_count", + "yarn_nodemanager_log_retain_seconds", + "yarn_resourcemanager_client_thread_count", + "yarn_resourcemanager_scheduler_client_thread_count", + "yarn_resourcemanager_admin_client_thread_count", + "yarn_resourcemanager_amliveliness_monitor_interval_ms", + "yarn_am_liveness_monitor_expiry_interval_ms", + "yarn_resourcemanager_am_max_retries" +] diff --git a/sahara/plugins/cdh/v5_4_0/resources/sentry-sentry_server.json b/sahara/plugins/cdh/v5_4_0/resources/sentry-sentry_server.json new file mode 100644 index 00000000..6045586b --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/sentry-sentry_server.json @@ -0,0 +1,218 @@ +[ + { + "desc": "When computing the overall Sentry Server health, consider the host's health.", + "display_name": "Sentry Server Host Health Test", + "name": "sentry_server_host_health_enabled", + "value": "true" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "The maximum size, in megabytes, per log file for Sentry Server logs. Typically used by log4j or logback.", + "display_name": "Sentry Server Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "The maximum number of rolled log files to keep for Sentry Server logs. Typically used by log4j or logback.", + "display_name": "Sentry Server Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for Sentry Server", + "name": "sentry_server_java_opts", + "value": "" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "Directory where Sentry Server will place its log files.", + "display_name": "Sentry Server Log Directory", + "name": "sentry_server_log_dir", + "value": "/var/log/sentry" + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "Sentry Server Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of Sentry Server in Bytes", + "name": "sentry_server_java_heapsize", + "value": "1073741824" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "RPC port number of Sentry Server.", + "display_name": "Sentry Server RPC Port", + "name": "sentry_service_server_rpc_port", + "value": "8038" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "sentry_server_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "Sentry Server Environment Advanced Configuration Snippet (Safety Valve)", + "name": "sentry_env_safety_valve", + "value": null + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "The minimum log level for Sentry Server logs", + "display_name": "Sentry Server Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "Enables the health test that the Sentry Server's process state is consistent with the role configuration", + "display_name": "Sentry Server Process Health Test", + "name": "sentry_server_scm_health_enabled", + "value": "true" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/sentry-service.json b/sahara/plugins/cdh/v5_4_0/resources/sentry-service.json new file mode 100644 index 00000000..f8c51988 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/sentry-service.json @@ -0,0 +1,158 @@ +[ + { + "desc": "Password for Sentry Server database.", + "display_name": "Sentry Server Database Password", + "name": "sentry_server_database_password", + "value": "" + }, + { + "desc": "

\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n

\n\n

\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n

\n\n

Event trackers are defined in a JSON object like the following:

\n\n
\n{\n  \"timeToLive\" : [integer],\n  \"fields\" : [\n    {\n      \"type\" : [string],\n      \"name\" : [string]\n    }\n  ]\n}\n
\n\n

\nWhere:\n

\n\n\n\n

\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n

\n\n\n\n

\nThe following is the list of fields that can be used to compare Sentry events:\n

\n\n\n", + "display_name": "Audit Event Tracker", + "name": "navigator_event_tracker", + "value": null + }, + { + "desc": "The user that this service's processes should run as.", + "display_name": "System User", + "name": "process_username", + "value": "sentry" + }, + { + "desc": "User for Sentry Server database.", + "display_name": "Sentry Server Database User", + "name": "sentry_server_database_user", + "value": "sentry" + }, + { + "desc": "Name of the HDFS service that this Sentry service instance depends on", + "display_name": "HDFS Service", + "name": "hdfs_service", + "value": null + }, + { + "desc": "Type of Sentry Server database.", + "display_name": "Sentry Server Database Type", + "name": "sentry_server_database_type", + "value": "mysql" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Sentry Service Environment Advanced Configuration Snippet (Safety Valve)", + "name": "SENTRY_service_env_safety_valve", + "value": null + }, + { + "desc": "Name of the ZooKeeper service that this Sentry service instance depends on", + "display_name": "ZooKeeper Service", + "name": "zookeeper_service", + "value": null + }, + { + "desc": "Maximum size of audit log file in MB before it is rolled over.", + "display_name": "Maximum Audit Log File Size", + "name": "navigator_audit_log_max_file_size", + "value": "100" + }, + { + "desc": "Host name of Sentry Server database.", + "display_name": "Sentry Server Database Host", + "name": "sentry_server_database_host", + "value": "localhost" + }, + { + "desc": "The health test thresholds of the overall Sentry Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Sentry Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Sentry Servers falls below the critical threshold.", + "display_name": "Healthy Sentry Server Monitoring Thresholds", + "name": "sentry_sentry_server_healthy_thresholds", + "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" + }, + { + "desc": "Name of Sentry Server database.", + "display_name": "Sentry Server Database Name", + "name": "sentry_server_database_name", + "value": "sentry" + }, + { + "desc": "Kerberos principal used by all roles of this service.", + "display_name": "Kerberos Principal", + "name": "kerberos_princ_name", + "value": "sentry" + }, + { + "desc": "List of users allowed to connect to the Sentry Server. These are usually service users such as hive and impala, and the list does not usually need to include end users.", + "display_name": "Allowed Connecting Users", + "name": "sentry_service_allow_connect", + "value": "hive,impala,hue,hdfs" + }, + { + "desc": "The group that this service's processes should run as.", + "display_name": "System Group", + "name": "process_groupname", + "value": "sentry" + }, + { + "desc": "For advanced use only, a string to be inserted into sentry-site.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Sentry Service Advanced Configuration Snippet (Safety Valve) for sentry-site.xml", + "name": "sentry_server_config_safety_valve", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Path to the directory where audit logs will be written. The directory will be created if it doesn't exist.", + "display_name": "Audit Log Directory", + "name": "audit_event_log_dir", + "value": "/var/log/sentry/audit" + }, + { + "desc": "Action to take when the audit event queue is full. Drop the event or shutdown the affected process.", + "display_name": "Audit Queue Policy", + "name": "navigator_audit_queue_policy", + "value": "DROP" + }, + { + "desc": "

The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n  \"streamThreshold\": 10, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Service Triggers", + "name": "service_triggers", + "value": "[]" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Service Level Health Alerts", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "Port number of Sentry Server database.", + "display_name": "Sentry Server Database Port", + "name": "sentry_server_database_port", + "value": "3306" + }, + { + "desc": "If an end user is in one of these admin groups, that user has administrative privileges on the Sentry Server.", + "display_name": "Admin Groups", + "name": "sentry_service_admin_group", + "value": "hive,impala,hue" + }, + { + "desc": "For advanced use only, a string to be inserted into the client configuration for navigator.client.properties.", + "display_name": "Sentry Client Advanced Configuration Snippet (Safety Valve) for navigator.client.properties", + "name": "navigator_client_config_safety_valve", + "value": null + }, + { + "desc": "Enable collection of audit events from the service's roles.", + "display_name": "Enable Audit Collection", + "name": "navigator_audit_enabled", + "value": "true" + }, + { + "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", + "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", + "name": "smon_derived_configs_safety_valve", + "value": null + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/solr-gateway.json b/sahara/plugins/cdh/v5_4_0/resources/solr-gateway.json new file mode 100644 index 00000000..ae9e4dc5 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/solr-gateway.json @@ -0,0 +1,32 @@ +[ + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "The minimum log level for Gateway logs", + "display_name": "Gateway Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "The directory where the client configs will be deployed", + "display_name": "Deploy Directory", + "name": "client_config_root_dir", + "value": "/etc/solr" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "Gateway Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", + "display_name": "Alternatives Priority", + "name": "client_config_priority", + "value": "90" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/solr-service.json b/sahara/plugins/cdh/v5_4_0/resources/solr-service.json new file mode 100644 index 00000000..76c64862 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/solr-service.json @@ -0,0 +1,188 @@ +[ + { + "desc": "Choose the authentication mechanism used by Solr.", + "display_name": "Solr Secure Authentication", + "name": "solr_security_authentication", + "value": "simple" + }, + { + "desc": "

\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n

\n\n

\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n

\n\n

Event trackers are defined in a JSON object like the following:

\n\n
\n{\n  \"timeToLive\" : [integer],\n  \"fields\" : [\n    {\n      \"type\" : [string],\n      \"name\" : [string]\n    }\n  ]\n}\n
\n\n

\nWhere:\n

\n\n\n\n

\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n

\n\n\n\n

\nThe following is the list of fields that can be used to compare Sentry events:\n

\n\n\n", + "display_name": "Audit Event Tracker", + "name": "navigator_event_tracker", + "value": null + }, + { + "desc": "For advanced use only, a string to be inserted into sentry-site.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Solr Service Advanced Configuration Snippet (Safety Valve) for sentry-site.xml", + "name": "solr_sentry_safety_valve", + "value": null + }, + { + "desc": "For advanced use only, a string to be inserted into the client configuration for navigator.client.properties.", + "display_name": "SOLR Client Advanced Configuration Snippet (Safety Valve) for navigator.client.properties", + "name": "navigator_client_config_safety_valve", + "value": null + }, + { + "desc": "The user that this service's processes should run as.", + "display_name": "System User", + "name": "process_username", + "value": "solr" + }, + { + "desc": "The location on disk of the trust store, in .jks format, used to confirm the authenticity of TLS/SSL servers that Solr might connect to. This is used when Solr is the client in a TLS/SSL connection. This trust store must contain the certificate(s) used to sign the service(s) being connected to. If this parameter is not provided, the default list of well-known certificate authorities is used instead.", + "display_name": "Solr TLS/SSL Certificate Trust Store File", + "name": "solr_https_truststore_file", + "value": null + }, + { + "desc": "The password for the Solr TLS/SSL Certificate Trust Store File. Note that this password is not required to access the trust store: this field can be left blank. This password provides optional integrity checking of the file. The contents of trust stores are certificates, and certificates are public information.", + "display_name": "Solr TLS/SSL Certificate Trust Store Password", + "name": "solr_https_truststore_password", + "value": null + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Solr Service Environment Advanced Configuration Snippet (Safety Valve)", + "name": "solr_env_safety_valve", + "value": null + }, + { + "desc": "Name of the HDFS service that this Search service instance depends on", + "display_name": "HDFS Service", + "name": "hdfs_service", + "value": null + }, + { + "desc": "Name of the ZooKeeper service that this Search service instance depends on", + "display_name": "ZooKeeper Service", + "name": "zookeeper_service", + "value": null + }, + { + "desc": "Maximum size of audit log file in MB before it is rolled over.", + "display_name": "Maximum Audit Log File Size", + "name": "navigator_audit_log_max_file_size", + "value": "100" + }, + { + "desc": "The password for the Solr JKS keystore file.", + "display_name": "Solr TLS/SSL Server JKS Keystore File Password", + "name": "solr_https_keystore_password", + "value": null + }, + { + "desc": "HDFS directory used for storage by this Solr service.", + "display_name": "HDFS Data Directory", + "name": "hdfs_data_dir", + "value": "/solr" + }, + { + "desc": "HDFS path to the global policy file for Sentry authorization. This should be a relative path (and not a full HDFS URL). The global policy file must be in Sentry policy file format.", + "display_name": "Sentry Global Policy File", + "name": "sentry_solr_provider_resource", + "value": "/user/solr/sentry/sentry-provider.ini" + }, + { + "desc": "Path to the directory where audit logs will be written. The directory will be created if it doesn't exist.", + "display_name": "Audit Log Directory", + "name": "audit_event_log_dir", + "value": "/var/log/solr/audit" + }, + { + "desc": "Kerberos principal used by all roles of this service.", + "display_name": "Kerberos Principal", + "name": "kerberos_princ_name", + "value": "solr" + }, + { + "desc": "Encrypt communication between clients and Solr using Transport Layer Security (TLS) (formerly known as Secure Socket Layer (SSL)). Note that there are additional manual steps to be performed: see Enabling TLS/SSL for Solr.", + "display_name": "Enable TLS/SSL for Solr", + "name": "solr_use_ssl", + "value": "false" + }, + { + "desc": "The group that this service's processes should run as.", + "display_name": "System Group", + "name": "process_groupname", + "value": "solr" + }, + { + "desc": "The class to use in Sentry authorization for user to group mapping. Sentry authorization may be configured to use either Hadoop groups or local groups defined in the policy file. When configured with Hadoop groups, Sentry will ask the HDFS Namenode for group mapping for a given user to determine authorization access.", + "display_name": "Sentry User to Group Mapping Class", + "name": "solr_sentry_provider", + "value": "org.apache.sentry.provider.file.HadoopGroupResourceAuthorizationProvider" + }, + { + "desc": "Use Sentry to enable role-based, fine-grained authorization. Sentry is supported only on Search 1.1 or later and CDH 5 or later deployments and requires authentication to be turned on for Solr..", + "display_name": "Enable Sentry Authorization", + "name": "solr_sentry_enabled", + "value": "false" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Service Level Health Alerts", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "Action to take when the audit event queue is full. Drop the event or shutdown the affected process.", + "display_name": "Audit Queue Policy", + "name": "navigator_audit_queue_policy", + "value": "DROP" + }, + { + "desc": "Enable the background watchdog thread that can kill Catalina process if Solr is not responsive.", + "display_name": "Enable Solrd Watchdog", + "name": "solrd_enable_watchdog", + "value": "true" + }, + { + "desc": "

The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n  \"streamThreshold\": 10, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Service Triggers", + "name": "service_triggers", + "value": "[]" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "The path to the TLS/SSL keystore file containing the server certificate and private key used for TLS/SSL. Used when Solr is acting as a TLS/SSL server. The keystore must be in JKS format.", + "display_name": "Solr TLS/SSL Server JKS Keystore File Location", + "name": "solr_https_keystore_file", + "value": null + }, + { + "desc": "ZooKeeper znode used to store information about this Solr service.", + "display_name": "ZooKeeper Znode", + "name": "zookeeper_znode", + "value": "/solr" + }, + { + "desc": "If Solr does not respond on its web URL within this time interval, the Catalina process is killed.", + "display_name": "Solrd Watchdog Timeout", + "name": "solrd_watchdog_timeout", + "value": "30" + }, + { + "desc": "Enable collection of audit events from the service's roles.", + "display_name": "Enable Audit Collection", + "name": "navigator_audit_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds of the overall Solr Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Solr Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Solr Servers falls below the critical threshold.", + "display_name": "Healthy Solr Server Monitoring Thresholds", + "name": "solr_solr_servers_healthy_thresholds", + "value": "{\"critical\":\"90.0\",\"warning\":\"95.0\"}" + }, + { + "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", + "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", + "name": "smon_derived_configs_safety_valve", + "value": null + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/solr-solr_server.json b/sahara/plugins/cdh/v5_4_0/resources/solr-solr_server.json new file mode 100644 index 00000000..69811911 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/solr-solr_server.json @@ -0,0 +1,302 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Enable caching of HDFS blocks in Solr. There is one block cache per Solr collection. configured to use off-heap memory, Maximum Off-Heap Memory must be set high enough to account for all block caches.", + "display_name": "HDFS Block Cache", + "name": "solr_hdfs_blockcache_enabled", + "value": "true" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Admin port of the Solr Server.", + "display_name": "Solr Admin Port", + "name": "solr_admin_port", + "value": "8984" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "The health test thresholds on the duration of the metrics request to the web server.", + "display_name": "Web Metric Collection Duration", + "name": "solr_server_web_metric_collection_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of Solr Server in Bytes", + "name": "solr_java_heapsize", + "value": "1073741824" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "Enables the health test that the Solr Server's process state is consistent with the role configuration", + "display_name": "Solr Server Process Health Test", + "name": "solr_server_scm_health_enabled", + "value": "true" + }, + { + "desc": "The health test thresholds on the duration of the Solr Server API request.", + "display_name": "Solr Server API Liveness Request Duration", + "name": "solr_core_status_collection_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "When computing the overall Solr Server health, consider the host's health.", + "display_name": "Solr Server Host Health Test", + "name": "solr_server_host_health_enabled", + "value": "true" + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "Directory where Solr Server will place its log files.", + "display_name": "Solr Server Log Directory", + "name": "solr_log_dir", + "value": "/var/log/solr" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "Solr Server Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "HTTPS port of Solr Server.", + "display_name": "Solr HTTPS port", + "name": "solr_https_port", + "value": "8985" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "Solr Server Environment Advanced Configuration Snippet (Safety Valve)", + "name": "SOLR_SERVER_role_env_safety_valve", + "value": null + }, + { + "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather status of Solr Cores from the Solr Server with a simple API request.", + "display_name": "Solr Server API Liveness", + "name": "solr_core_status_collection_health_enabled", + "value": "true" + }, + { + "desc": "Use off-heap memory when caching HDFS blocks in Solr.", + "display_name": "HDFS Block Cache Off-Heap Memory", + "name": "solr_hdfs_blockcache_direct_memory_allocation", + "value": "true" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", + "display_name": "Garbage Collection Duration Thresholds", + "name": "solr_server_gc_duration_thresholds", + "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The maximum size, in megabytes, per log file for Solr Server logs. Typically used by log4j or logback.", + "display_name": "Solr Server Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "Directory on local file system where Solr Server keeps the configurations for collections.", + "display_name": "Solr Data Directory", + "name": "solr_data_dir", + "value": "/var/lib/solr" + }, + { + "desc": "Maximum amount of off-heap memory in bytes that may be allocated by the Java process. Passed to Java -XX:MaxDirectMemorySize. If unset, defaults to the size of the heap.", + "display_name": "Java Direct Memory Size of Solr Server in Bytes", + "name": "solr_java_direct_memory_size", + "value": "1073741824" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Number of blocks per cache slab. The size of the cache is 8 KB (the block size) times the number of blocks per slab times the number of slabs.", + "display_name": "HDFS Block Cache Blocks per Slab", + "name": "solr_hdfs_blockcache_blocksperbank", + "value": "16384" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "The maximum number of rolled log files to keep for Solr Server logs. Typically used by log4j or logback.", + "display_name": "Solr Server Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "solr_server_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "Number of slabs per block cache. The size of the cache is 8 KB (the block size) times the number of blocks per slab times the number of slabs.", + "display_name": "HDFS Block Cache Number of Slabs", + "name": "solr_hdfs_blockcache_slab_count", + "value": "1" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "The period to review when computing the moving average of garbage collection time.", + "display_name": "Garbage Collection Duration Monitoring Period", + "name": "solr_server_gc_duration_window", + "value": "5" + }, + { + "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", + "display_name": "Web Metric Collection", + "name": "solr_server_web_metric_collection_enabled", + "value": "true" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for Solr Server", + "name": "solr_java_opts", + "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" + }, + { + "desc": "The minimum log level for Solr Server logs", + "display_name": "Solr Server Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "HTTP port of Solr Server.", + "display_name": "Solr HTTP Port", + "name": "solr_http_port", + "value": "8983" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/spark-gateway.json b/sahara/plugins/cdh/v5_4_0/resources/spark-gateway.json new file mode 100644 index 00000000..fa148de9 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/spark-gateway.json @@ -0,0 +1,62 @@ +[ + { + "desc": "Name of class implementing org.apache.spark.serializer.Serializer to use in Spark applications.", + "display_name": "Spark Data Serializer", + "name": "spark_data_serializer", + "value": "org.apache.spark.serializer.KryoSerializer" + }, + { + "desc": "For advanced use only, a string to be inserted into the client configuration for spark-conf/log4j.properties.", + "display_name": "Spark Client Advanced Configuration Snippet (Safety Valve) for spark-conf/log4j.properties", + "name": "spark-conf/log4j.properties_client_config_safety_valve", + "value": null + }, + { + "desc": "Enable Usage of External Shuffle Service.", + "display_name": "Enable Shuffle Service", + "name": "spark_shuffle_service_enabled", + "value": "true" + }, + { + "desc": "The directory where the client configs will be deployed", + "display_name": "Deploy Directory", + "name": "client_config_root_dir", + "value": "/etc/spark" + }, + { + "desc": "Write Spark application history logs to HDFS.", + "display_name": "Enable History", + "name": "spark_history_enabled", + "value": "true" + }, + { + "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", + "display_name": "Alternatives Priority", + "name": "client_config_priority", + "value": "51" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Which deploy mode to use by default. Can be overridden by users when launching applications.", + "display_name": "Default Application Deploy Mode", + "name": "spark_deploy_mode", + "value": "client" + }, + { + "desc": "For advanced use only, a string to be inserted into the client configuration for spark-conf/spark-defaults.conf.", + "display_name": "Spark Client Advanced Configuration Snippet (Safety Valve) for spark-conf/spark-defaults.conf", + "name": "spark-conf/spark-defaults.conf_client_config_safety_valve", + "value": null + }, + { + "desc": "For advanced use only, a string to be inserted into the client configuration for spark-conf/spark-env.sh.", + "display_name": "Spark Client Advanced Configuration Snippet (Safety Valve) for spark-conf/spark-env.sh", + "name": "spark-conf/spark-env.sh_client_config_safety_valve", + "value": null + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/spark-service.json b/sahara/plugins/cdh/v5_4_0/resources/spark-service.json new file mode 100644 index 00000000..2e529da3 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/spark-service.json @@ -0,0 +1,80 @@ +[ + { + "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Service Level Health Alerts", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The location of the Spark jar in HDFS. If left blank, CM will use the Spark jar installed on the cluster nodes.", + "display_name": "Spark Jar Location (HDFS)", + "name": "spark_jar_hdfs_path", + "value": "" + }, + { + "desc": "The group that this service's processes should run as.", + "display_name": "System Group", + "name": "process_groupname", + "value": "spark" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Spark Service Environment Advanced Configuration Snippet (Safety Valve)", + "name": "SPARK_ON_YARN_service_env_safety_valve", + "value": null + }, + { + "desc": "Enable whether the Spark communication protocols do authentication using a shared secret.", + "display_name": "Spark Authentication", + "name": "spark_authenticate", + "value": "false" + }, + { + "desc": "The location of Spark application history logs in HDFS. Changing this value will not move existing logs to the new location.", + "display_name": "Spark History Location (HDFS)", + "name": "spark_history_log_dir", + "value": "/user/spark/applicationHistory" + }, + { + "desc": "Name of the YARN (MR2 Included) service that this Spark service instance depends on", + "display_name": "YARN (MR2 Included) Service", + "name": "yarn_service", + "value": null + }, + { + "desc": "The port the Spark Shuffle Service listens for fetch requests.", + "display_name": "Spark Shuffle Service Port", + "name": "spark_shuffle_service_port", + "value": "7337" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "The user that this service's processes should run as.", + "display_name": "System User", + "name": "process_username", + "value": "spark" + }, + { + "desc": "Kerberos principal used by all roles of this service.", + "display_name": "Kerberos Principal", + "name": "kerberos_princ_name", + "value": "spark" + }, + { + "desc": "

The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n  \"streamThreshold\": 10, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Service Triggers", + "name": "service_triggers", + "value": "[]" + }, + { + "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", + "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", + "name": "smon_derived_configs_safety_valve", + "value": null + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/spark-spark_yarn_history_server.json b/sahara/plugins/cdh/v5_4_0/resources/spark-spark_yarn_history_server.json new file mode 100644 index 00000000..ad6340f0 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/spark-spark_yarn_history_server.json @@ -0,0 +1,146 @@ +[ + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "History Server Environment Advanced Configuration Snippet (Safety Valve)", + "name": "SPARK_YARN_HISTORY_SERVER_role_env_safety_valve", + "value": null + }, + { + "desc": "For advanced use only, a string to be inserted into config/spark-env.sh for this role only.", + "display_name": "History Server Advanced Configuration Snippet (Safety Valve) for config/spark-env.sh", + "name": "config/spark-env.sh_role_safety_valve", + "value": null + }, + { + "desc": "The port of the history server WebUI", + "display_name": "History Server WebUI Port", + "name": "history_server_web_port", + "value": "18088" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The maximum size, in megabytes, per log file for History Server logs. Typically used by log4j or logback.", + "display_name": "History Server Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "spark_yarn_history_server_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "The log directory for log files of the role History Server.", + "display_name": "History Server Log Directory", + "name": "log_dir", + "value": "/var/log/spark" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "History Server Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "Maximum size for the Java process heap memory. Passed to Java -Xmx. Measured in bytes.", + "display_name": "Java Heap Size of History Server in Bytes", + "name": "history_server_max_heapsize", + "value": "268435456" + }, + { + "desc": "When computing the overall History Server health, consider the host's health.", + "display_name": "History Server Host Health Test", + "name": "spark_yarn_history_server_host_health_enabled", + "value": "true" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The maximum number of rolled log files to keep for History Server logs. Typically used by log4j or logback.", + "display_name": "History Server Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "Enables the health test that the History Server's process state is consistent with the role configuration", + "display_name": "History Server Process Health Test", + "name": "spark_yarn_history_server_scm_health_enabled", + "value": "true" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "The minimum log level for History Server logs", + "display_name": "History Server Logging Threshold", + "name": "log_threshold", + "value": "INFO" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/sqoop-service.json b/sahara/plugins/cdh/v5_4_0/resources/sqoop-service.json new file mode 100644 index 00000000..e2f827aa --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/sqoop-service.json @@ -0,0 +1,62 @@ +[ + { + "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Service Level Health Alerts", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "When computing the overall SQOOP health, consider Sqoop 2 Server's health", + "display_name": "Sqoop 2 Server Role Health Test", + "name": "sqoop_sqoop_server_health_enabled", + "value": "true" + }, + { + "desc": "The group that this service's processes should run as.", + "display_name": "System Group", + "name": "process_groupname", + "value": "sqoop" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", + "display_name": "Sqoop 2 Service Environment Advanced Configuration Snippet (Safety Valve)", + "name": "sqoop_env_safety_valve", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "The user that this service's processes should run as.", + "display_name": "System User", + "name": "process_username", + "value": "sqoop2" + }, + { + "desc": "Kerberos principal used by all roles of this service.", + "display_name": "Kerberos Principal", + "name": "kerberos_princ_name", + "value": "sqoop2" + }, + { + "desc": "

The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n  \"streamThreshold\": 10, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Service Triggers", + "name": "service_triggers", + "value": "[]" + }, + { + "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", + "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", + "name": "smon_derived_configs_safety_valve", + "value": null + }, + { + "desc": "MapReduce jobs are run against this service.", + "display_name": "MapReduce Service", + "name": "mapreduce_yarn_service", + "value": null + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/sqoop-sqoop_server.json b/sahara/plugins/cdh/v5_4_0/resources/sqoop-sqoop_server.json new file mode 100644 index 00000000..a13d6c11 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/sqoop-sqoop_server.json @@ -0,0 +1,272 @@ +[ + { + "desc": "Directory where Sqoop 2 Server will place its log files.", + "display_name": "Sqoop 2 Server Log Directory", + "name": "sqoop_log_dir", + "value": "/var/log/sqoop2" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "Password for Sqoop repository database. Does not apply if you are using Derby as the database type.", + "display_name": "Sqoop Repository Database Password", + "name": "sqoop_repository_database_password", + "value": "" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Admin port of Sqoop 2 Server. (Note: This configuration only applies from CDH 4.3 onwards.)", + "display_name": "Sqoop 2 Admin Port", + "name": "sqoop_admin_port", + "value": "8005" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Maximum number of clients allowed to connect to the Sqoop 2 Server.", + "display_name": "Maximum Client Connections", + "name": "max_client_connections", + "value": "10" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "For advanced use only, a string to be inserted into sqoop.properties for this role only.", + "display_name": "Sqoop 2 Server Advanced Configuration Snippet (Safety Valve) for sqoop.properties", + "name": "sqoop_config_safety_valve", + "value": null + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "The maximum number of rolled log files to keep for Sqoop 2 Server logs. Typically used by log4j or logback.", + "display_name": "Sqoop 2 Server Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "HTTP port of Sqoop 2 Server. (Note: This configuration only applies from CDH 4.3 onwards.)", + "display_name": "Sqoop 2 HTTP Port", + "name": "sqoop_http_port", + "value": "12000" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "Sqoop 2 Server Environment Advanced Configuration Snippet (Safety Valve)", + "name": "SQOOP_SERVER_role_env_safety_valve", + "value": null + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "sqoop_server_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of Sqoop 2 Server in Bytes", + "name": "sqoop_java_heapsize", + "value": "1073741824" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "Sqoop 2 Server Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Type of Sqoop repository database.", + "display_name": "Sqoop Repository Database Type", + "name": "sqoop_repository_database_type", + "value": "derby" + }, + { + "desc": "User for Sqoop repository database. Does not apply if you are using Derby as the database type.", + "display_name": "Sqoop Repository Database User", + "name": "sqoop_repository_database_user", + "value": "sa" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "Name of Sqoop repository database. Does not apply if you are using Derby as the database type.", + "display_name": "Sqoop Repository Database Name", + "name": "sqoop_repository_database_name", + "value": "sqoop" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The maximum size, in megabytes, per log file for Sqoop 2 Server logs. Typically used by log4j or logback.", + "display_name": "Sqoop 2 Server Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "Directory where the Sqoop 2 Server places its metastore data. This is used only when Sqoop Repository Database Type is Derby.", + "display_name": "Sqoop 2 Server Metastore Directory", + "name": "sqoop_metastore_data_dir", + "value": "/var/lib/sqoop2" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for Sqoop 2 Server", + "name": "sqoop_java_opts", + "value": "" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "When computing the overall Sqoop 2 Server health, consider the host's health.", + "display_name": "Sqoop 2 Server Host Health Test", + "name": "sqoop_server_host_health_enabled", + "value": "true" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "Enables the health test that the Sqoop 2 Server's process state is consistent with the role configuration", + "display_name": "Sqoop 2 Server Process Health Test", + "name": "sqoop_server_scm_health_enabled", + "value": "true" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "Host name of Sqoop repository database. If the port is non-default for your database type, use host:port notation. Does not apply if you are using Derby as the database type.", + "display_name": "Sqoop Repository Database Host", + "name": "sqoop_repository_database_host", + "value": "localhost" + }, + { + "desc": "The minimum log level for Sqoop 2 Server logs", + "display_name": "Sqoop 2 Server Logging Threshold", + "name": "log_threshold", + "value": "INFO" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/yarn-gateway.json b/sahara/plugins/cdh/v5_4_0/resources/yarn-gateway.json new file mode 100644 index 00000000..2b06b736 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/yarn-gateway.json @@ -0,0 +1,344 @@ +[ + { + "desc": "For MapReduce job outputs that are compressed, specify the compression codec to use. Will be part of generated client configuration.", + "display_name": "Compression Codec of MapReduce Job Output", + "name": "mapred_output_compression_codec", + "value": "org.apache.hadoop.io.compress.DefaultCodec" + }, + { + "desc": "The default number of parallel transfers run by reduce during the copy (shuffle) phase. This number should be between sqrt(nodes*number_of_map_slots_per_node) and nodes*number_of_map_slots_per_node/2. Will be part of generated client configuration.", + "display_name": "Default Number of Parallel Transfers During Shuffle", + "name": "mapred_reduce_parallel_copies", + "value": "10" + }, + { + "desc": "Whether map tasks should attempt to use the optimized native implementation of the map-side output collector. This can improve performance of many jobs that are shuffle-intensive. Experimental in CDH 5.2.", + "display_name": "Enable Optimized Map-side Output Collector", + "name": "mapreduce_enable_native_map_output_collector", + "value": "false" + }, + { + "desc": "The directory where the client configs will be deployed", + "display_name": "Deploy Directory", + "name": "client_config_root_dir", + "value": "/etc/hadoop" + }, + { + "desc": "The number of streams to merge at the same time while sorting files. That is, the number of sort heads to use during the merge sort on the reducer side. This determines the number of open file handles. Merging more files in parallel reduces merge sort iterations and improves run time by eliminating disk I/O. Note that merging more files in parallel uses more memory. If 'io.sort.factor' is set too high or the maximum JVM heap is set too low, excessive garbage collection will occur. The Hadoop default is 10, but Cloudera recommends a higher value. Will be part of generated client configuration.", + "display_name": "I/O Sort Factor", + "name": "io_sort_factor", + "value": "64" + }, + { + "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", + "display_name": "Alternatives Priority", + "name": "client_config_priority", + "value": "92" + }, + { + "desc": "Location to store the job history files of running jobs. This is a path on the host where the JobTracker is running.", + "display_name": "Running Job History Location", + "name": "hadoop_job_history_dir", + "value": "/var/log/hadoop-mapreduce/history" + }, + { + "desc": "The number of virtual CPU cores allocated for each map task of a job. This parameter has no effect prior to CDH 4.4.", + "display_name": "Map Task CPU Virtual Cores", + "name": "mapreduce_map_cpu_vcores", + "value": "1" + }, + { + "desc": "If enabled, multiple instances of some reduce tasks may be executed in parallel.", + "display_name": "Reduce Tasks Speculative Execution", + "name": "mapred_reduce_tasks_speculative_execution", + "value": "false" + }, + { + "desc": "The application framework to run jobs with. If not set, jobs will be run with the local job runner.", + "display_name": "Application Framework", + "name": "mapreduce_framework_name", + "value": "yarn" + }, + { + "desc": "Base sleep time between failover attempts. Used only if RM HA is enabled.", + "display_name": "Client Failover Sleep Base Time", + "name": "client_failover_sleep_base", + "value": "100" + }, + { + "desc": "For advanced use only, a string to be inserted into the client configuration for yarn-site.xml.", + "display_name": "YARN Client Advanced Configuration Snippet (Safety Valve) for yarn-site.xml", + "name": "yarn_client_config_safety_valve", + "value": null + }, + { + "desc": "Size of buffer for read and write operations of SequenceFiles.", + "display_name": "SequenceFile I/O Buffer Size", + "name": "io_file_buffer_size", + "value": "65536" + }, + { + "desc": "Fraction of the number of map tasks in the job which should be completed before reduce tasks are scheduled for the job.", + "display_name": "Number of Map Tasks to Complete Before Reduce Tasks", + "name": "mapred_reduce_slowstart_completed_maps", + "value": "0.8" + }, + { + "desc": "For advanced use only, a string to be inserted into the client configuration for mapred-site.xml.", + "display_name": "MapReduce Client Advanced Configuration Snippet (Safety Valve) for mapred-site.xml", + "name": "mapreduce_client_config_safety_valve", + "value": null + }, + { + "desc": "The maximum heap size, in bytes, of the Java MapReduce ApplicationMaster. This number will be formatted and concatenated with 'ApplicationMaster Java Opts Base' to pass to Hadoop.", + "display_name": "ApplicationMaster Java Maximum Heap Size", + "name": "yarn_app_mapreduce_am_max_heap", + "value": "825955249" + }, + { + "desc": "The replication level for submitted job files.", + "display_name": "Mapreduce Submit Replication", + "name": "mapred_submit_replication", + "value": "10" + }, + { + "desc": "The total amount of memory buffer, in megabytes, to use while sorting files. Note that this memory comes out of the user JVM heap size (meaning total user JVM heap - this amount of memory = total user usable heap space. Note that Cloudera's default differs from Hadoop's default; Cloudera uses a bigger buffer by default because modern machines often have more RAM. The smallest value across all TaskTrackers will be part of generated client configuration.", + "display_name": "I/O Sort Memory Buffer (MiB)", + "name": "io_sort_mb", + "value": "256" + }, + { + "desc": "Whether to enable ubertask optimization, which runs \"sufficiently small\" jobs sequentially within a single JVM. \"Small\" is defined by the mapreduce.job.ubertask.maxmaps, mapreduce.job.ubertask.maxreduces, and mapreduce.job.ubertask.maxbytes settings.", + "display_name": "Enable Ubertask Optimization", + "name": "mapreduce_job_ubertask_enabled", + "value": "false" + }, + { + "desc": "Java command line arguments passed to the MapReduce ApplicationMaster.", + "display_name": "ApplicationMaster Java Opts Base", + "name": "yarn_app_mapreduce_am_command_opts", + "value": "-Djava.net.preferIPv4Stack=true" + }, + { + "desc": "The amount of physical memory, in MiB, allocated for each reduce task of a job. This parameter has no effect prior to CDH 4.4.", + "display_name": "Reduce Task Memory", + "name": "mapreduce_reduce_memory_mb", + "value": "1024" + }, + { + "desc": "The maximum permissible size of the split metainfo file. The JobTracker won't attempt to read split metainfo files bigger than the configured value. No limits if set to -1.", + "display_name": "JobTracker MetaInfo Maxsize", + "name": "mapreduce_jobtracker_split_metainfo_maxsize", + "value": "10000000" + }, + { + "desc": "Compression level for the codec used to compress MapReduce outputs. Default compression is a balance between speed and compression ratio.", + "display_name": "Compression Level of Codecs", + "name": "zlib_compress_level", + "value": "DEFAULT_COMPRESSION" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "Gateway Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "A shared directory for temporary files.", + "display_name": "Shared Temp Directories", + "name": "mapreduce_cluster_temp_dir", + "value": null + }, + { + "desc": "These are Java command line arguments. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Client Java Configuration Options", + "name": "mapreduce_client_java_opts", + "value": "-Djava.net.preferIPv4Stack=true" + }, + { + "desc": "The physical memory requirement, in MiB, for the ApplicationMaster.", + "display_name": "ApplicationMaster Memory", + "name": "yarn_app_mapreduce_am_resource_mb", + "value": "1024" + }, + { + "desc": "The maximum Java heap size, in bytes, of the reduce processes. This number will be formatted and concatenated with 'Reduce Task Java Opts Base' to pass to Hadoop.", + "display_name": "Reduce Task Maximum Heap Size", + "name": "mapreduce_reduce_java_opts_max_heap", + "value": "825955249" + }, + { + "desc": "Maximum size in bytes for the Java process heap memory. Passed to Java -Xmx.", + "display_name": "Client Java Heap Size in Bytes", + "name": "mapreduce_client_java_heapsize", + "value": "825955249" + }, + { + "desc": "The number of milliseconds before a task will be terminated if it neither reads an input, writes an output, nor updates its status string.", + "display_name": "Mapreduce Task Timeout", + "name": "mapred_task_timeout", + "value": "600000" + }, + { + "desc": "The virtual CPU cores requirement, for the ApplicationMaster. This parameter has no effect prior to CDH 4.4.", + "display_name": "ApplicationMaster Virtual CPU Cores", + "name": "yarn_app_mapreduce_am_resource_cpu_vcores", + "value": "1" + }, + { + "desc": "The amount of physical memory, in MiB, allocated for each map task of a job.", + "display_name": "Map Task Memory", + "name": "mapreduce_map_memory_mb", + "value": "1024" + }, + { + "desc": "Threshold for number of reduces, beyond which a job is considered too big for ubertask optimization. Note: As of CDH 5, MR2 does not support more than one reduce in an ubertask. (Zero is valid.)", + "display_name": "Ubertask Maximum Reduces", + "name": "mapreduce_job_ubertask_maxreduces", + "value": "1" + }, + { + "desc": "For MapReduce map outputs that are compressed, specify the compression codec to use. Will be part of generated client configuration.", + "display_name": "Compression Codec of MapReduce Map Output", + "name": "mapred_map_output_compression_codec", + "value": "org.apache.hadoop.io.compress.SnappyCodec" + }, + { + "desc": "Classpaths to include for MapReduce applications.", + "display_name": "MR Application Classpath", + "name": "mapreduce_application_classpath", + "value": "$HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$MR2_CLASSPATH" + }, + { + "desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.", + "display_name": "Enable Log Event Capture", + "name": "catch_events", + "value": "true" + }, + { + "desc": "Maximum sleep time between failover attempts. Used only if RM HA is enabled.", + "display_name": "Client Failover Sleep Max Time", + "name": "client_failover_sleep_max", + "value": "2000" + }, + { + "desc": "Limit on the number of counters allowed per job.", + "display_name": "Job Counters Limit", + "name": "mapreduce_job_counters_limit", + "value": "120" + }, + { + "desc": "Compress the output of MapReduce jobs. Will be part of generated client configuration.", + "display_name": "Compress MapReduce Job Output", + "name": "mapred_output_compress", + "value": "false" + }, + { + "desc": "For MapReduce job outputs that are compressed as SequenceFiles, you can select one of these compression type options: NONE, RECORD or BLOCK. Cloudera recommends BLOCK. Will be part of generated client configuration.", + "display_name": "Compression Type of MapReduce Job Output", + "name": "mapred_output_compression_type", + "value": "BLOCK" + }, + { + "desc": "Java opts for the map processes. The following symbol, if present, will be interpolated: @taskid@ is replaced by current TaskID. Any other occurrences of '@' will go unchanged. For example, to enable verbose gc logging to a file named for the taskid in /tmp pass a value of: \"-verbose:gc -Xloggc:/tmp/@taskid@.gc\". The configuration variable 'Map Task Memory' can be used to control the maximum memory of the map processes.", + "display_name": "Map Task Java Opts Base", + "name": "mapreduce_map_java_opts", + "value": "-Djava.net.preferIPv4Stack=true" + }, + { + "desc": "Additional execution environment entries for map and reduce task processes.", + "display_name": "MR Application Environment", + "name": "mapreduce_admin_user_env", + "value": "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH" + }, + { + "desc": "The soft limit in either the buffer or record collection buffers. When this limit is reached, a thread will begin to spill the contents to disk in the background. Note that this does not imply any chunking of data to the spill. A value less than 0.5 is not recommended. The syntax is in decimal units; the default is 80% and is formatted 0.8. Will be part of generated client configuration.", + "display_name": "I/O Sort Spill Percent", + "name": "io_sort_spill_percent", + "value": "0.8" + }, + { + "desc": "The default number of reduce tasks per job. Will be part of generated client configuration.", + "display_name": "Default Number of Reduce Tasks per Job", + "name": "mapred_reduce_tasks", + "value": "1" + }, + { + "desc": "Maximum allowed connections for the shuffle. Set to 0 (zero) to indicate no limit on the number of connections.", + "display_name": "Max Shuffle Connections", + "name": "mapreduce_shuffle_max_connections", + "value": "80" + }, + { + "desc": "The maximum Java heap size, in bytes, of the map processes. This number will be formatted and concatenated with 'Map Task Java Opts Base' to pass to Hadoop.", + "display_name": "Map Task Maximum Heap Size", + "name": "mapreduce_map_java_opts_max_heap", + "value": "825955249" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the client configuration for hadoop-env.sh", + "display_name": "Gateway Client Environment Advanced Configuration Snippet for hadoop-env.sh (Safety Valve)", + "name": "mapreduce_client_env_safety_valve", + "value": null + }, + { + "desc": "The number of virtual CPU cores for each reduce task of a job.", + "display_name": "Reduce Task CPU Virtual Cores", + "name": "mapreduce_reduce_cpu_vcores", + "value": "1" + }, + { + "desc": "Environment variables for the MapReduce ApplicationMaster. These settings can be overridden in the ApplicationMaster User Environment (yarn.app.mapreduce.am.env).", + "display_name": "ApplicationMaster Environment", + "name": "yarn_app_mapreduce_am_admin_user_env", + "value": "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH" + }, + { + "desc": "Java opts for the reduce processes. The following symbol, if present, will be interpolated: @taskid@ is replaced by current TaskID. Any other occurrences of '@' will go unchanged. For example, to enable verbose gc logging to a file named for the taskid in /tmp pass a value of: \"-verbose:gc -Xloggc:/tmp/@taskid@.gc\". The configuration variable 'Reduce Task Memory' can be used to control the maximum memory of the reduce processes.", + "display_name": "Reduce Task Java Opts Base", + "name": "mapreduce_reduce_java_opts", + "value": "-Djava.net.preferIPv4Stack=true" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "If enabled, uses compression on the map outputs before they are sent across the network. Will be part of generated client configuration.", + "display_name": "Use Compression on Map Outputs", + "name": "mapred_compress_map_output", + "value": "true" + }, + { + "desc": "The maximum number of application attempts for MapReduce jobs. The value of this parameter overrides ApplicationMaster Maximum Attempts for MapReduce jobs.", + "display_name": "Maximum Number of Attempts for MapReduce Jobs", + "name": "mapreduce_am_max_attempts", + "value": "2" + }, + { + "desc": "The minimum log level for Gateway logs", + "display_name": "Gateway Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "Threshold for number of maps, beyond which a job is considered too big for ubertask optimization.", + "display_name": "Ubertask Maximum Maps", + "name": "mapreduce_job_ubertask_maxmaps", + "value": "9" + }, + { + "desc": "If enabled, multiple instances of some map tasks may be executed in parallel.", + "display_name": "Map Tasks Speculative Execution", + "name": "mapred_map_tasks_speculative_execution", + "value": "false" + }, + { + "desc": "Threshold for number of input bytes, beyond which a job is considered too big for ubertask optimization. If no value is specified, dfs.block.size is used as a default.", + "display_name": "Ubertask Maximum Job Size", + "name": "mapreduce_job_ubertask_maxbytes", + "value": null + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/yarn-jobhistory.json b/sahara/plugins/cdh/v5_4_0/resources/yarn-jobhistory.json new file mode 100644 index 00000000..790b7178 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/yarn-jobhistory.json @@ -0,0 +1,320 @@ +[ + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of JobHistory Server in Bytes", + "name": "mr2_jobhistory_java_heapsize", + "value": "1073741824" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "

This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.

Each rule has some or all of the following fields:


Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}

This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.

", + "display_name": "Rules to Extract Events from Log Files", + "name": "log_event_whitelist", + "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for JobHistory Server", + "name": "mr2_jobhistory_java_opts", + "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "Kerberos principal used by the JobHistory Server roles.", + "display_name": "Role-Specific Kerberos Principal", + "name": "kerberos_role_princ_name", + "value": "mapred" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "jobhistory_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Enables the health test that the JobHistory Server's process state is consistent with the role configuration", + "display_name": "JobHistory Server Process Health Test", + "name": "jobhistory_scm_health_enabled", + "value": "true" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "JobHistory Server Environment Advanced Configuration Snippet (Safety Valve)", + "name": "JOBHISTORY_role_env_safety_valve", + "value": null + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "When computing the overall JobHistory Server health, consider the host's health.", + "display_name": "JobHistory Server Host Health Test", + "name": "jobhistory_host_health_enabled", + "value": "true" + }, + { + "desc": "The maximum size, in megabytes, per log file for JobHistory Server logs. Typically used by log4j or logback.", + "display_name": "JobHistory Server Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Directory where JobHistory Server will place its log files.", + "display_name": "JobHistory Server Log Directory", + "name": "mr2_jobhistory_log_dir", + "value": "/var/log/hadoop-mapreduce" + }, + { + "desc": "Time interval for history cleaner to check for files to delete. Files are only deleted if they are older than mapreduce.jobhistory.max-age-ms.", + "display_name": "Job History Files Cleaner Interval", + "name": "mapreduce_jobhistory_cleaner_interval", + "value": "86400000" + }, + { + "desc": "The period to review when computing the moving average of garbage collection time.", + "display_name": "Garbage Collection Duration Monitoring Period", + "name": "jobhistory_gc_duration_window", + "value": "5" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "For advanced use only, a string to be inserted into yarn-site.xml for this role only.", + "display_name": "JobHistory Server Advanced Configuration Snippet (Safety Valve) for yarn-site.xml", + "name": "jobhistory_config_safety_valve", + "value": null + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "JobHistory Server Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "The port of the MapReduce JobHistory Server administrative interface. Together with the host name of the JobHistory role forms the address.", + "display_name": "MapReduce JobHistory Server Admin Interface Port", + "name": "mapreduce_jobhistory_admin_address", + "value": "10033" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "The group that the JobHistory Server process should run as.", + "display_name": "System Group", + "name": "history_process_groupname", + "value": "hadoop" + }, + { + "desc": "Job history files older than this time duration will deleted when the history cleaner runs.", + "display_name": "Job History Files Maximum Age", + "name": "mapreduce_jobhistory_max_age_ms", + "value": "604800000" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The minimum log level for JobHistory Server logs", + "display_name": "JobHistory Server Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "The HTTP port of the MapReduce JobHistory Server web application. Together with the host name of the JobHistory role forms the address.", + "display_name": "MapReduce JobHistory Web Application HTTP Port", + "name": "mapreduce_jobhistory_webapp_address", + "value": "19888" + }, + { + "desc": "The root HDFS directory of the staging area for users' MR2 jobs; for example /user. The staging directories are always named after the user.", + "display_name": "MapReduce ApplicationMaster Staging Root Directory", + "name": "yarn_app_mapreduce_am_staging_dir", + "value": "/user" + }, + { + "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", + "display_name": "Web Metric Collection", + "name": "jobhistory_web_metric_collection_enabled", + "value": "true" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "The port of the MapReduce JobHistory Server. Together with the hostname of the JobHistory role, forms the address. ", + "display_name": "MapReduce JobHistory Server Port", + "name": "mapreduce_jobhistory_address", + "value": "10020" + }, + { + "desc": "For advanced use only, a string to be inserted into mapred-site.xml for this role only.", + "display_name": "JobHistory Server Advanced Configuration Snippet (Safety Valve) for mapred-site.xml", + "name": "jobhistory_mapred_safety_valve", + "value": null + }, + { + "desc": "If enabled, the JobHistory Server binds to the wildcard address (\"0.0.0.0\") on all of its ports.", + "display_name": "Bind JobHistory Server to Wildcard Address", + "name": "yarn_jobhistory_bind_wildcard", + "value": "false" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "The maximum number of rolled log files to keep for JobHistory Server logs. Typically used by log4j or logback.", + "display_name": "JobHistory Server Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "The user that the JobHistory Server process should run as.", + "display_name": "System User", + "name": "history_process_username", + "value": "mapred" + }, + { + "desc": "The HTTPS port of the MapReduce JobHistory Server web application. Together with the host name of the JobHistory role forms the address.", + "display_name": "MapReduce JobHistory Web Application HTTPS Port (SSL)", + "name": "mapreduce_jobhistory_webapp_https_address", + "value": "19890" + }, + { + "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", + "display_name": "Garbage Collection Duration Thresholds", + "name": "jobhistory_gc_duration_thresholds", + "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" + }, + { + "desc": "The health test thresholds on the duration of the metrics request to the web server.", + "display_name": "Web Metric Collection Duration", + "name": "jobhistory_web_metric_collection_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/yarn-nodemanager.json b/sahara/plugins/cdh/v5_4_0/resources/yarn-nodemanager.json new file mode 100644 index 00000000..3dd6fe32 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/yarn-nodemanager.json @@ -0,0 +1,476 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "

This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.

Each rule has some or all of the following fields:


Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}

This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.

", + "display_name": "Rules to Extract Events from Log Files", + "name": "log_event_whitelist", + "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "For advanced use only, a string to be inserted into yarn-site.xml for this role only.", + "display_name": "NodeManager Advanced Configuration Snippet (Safety Valve) for yarn-site.xml", + "name": "nodemanager_config_safety_valve", + "value": null + }, + { + "desc": "The HTTP Port of the NodeManager web application.", + "display_name": "NodeManager Web Application HTTP Port", + "name": "nodemanager_webserver_port", + "value": "8042" + }, + { + "desc": "If enabled, adds 'org.apache.hadoop.mapred.ShuffleHandler' to the NodeManager auxiliary services. This is required for MapReduce applications.", + "display_name": "Enable Shuffle Auxiliary Service", + "name": "mapreduce_aux_service", + "value": "true" + }, + { + "desc": "Heartbeat interval to ResourceManager", + "display_name": "Heartbeat Interval", + "name": "yarn_nodemanager_heartbeat_interval_ms", + "value": "1000" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Enables the health test that the NodeManager's process state is consistent with the role configuration", + "display_name": "NodeManager Process Health Test", + "name": "nodemanager_scm_health_enabled", + "value": "true" + }, + { + "desc": "The amount of time to wait for the NodeManager to fully start up and connect to the ResourceManager before enforcing the connectivity check.", + "display_name": "NodeManager Connectivity Tolerance at Startup", + "name": "nodemanager_connectivity_tolerance_seconds", + "value": "180" + }, + { + "desc": "List of directories on the local filesystem where a NodeManager stores intermediate data files.", + "display_name": "NodeManager Local Directories", + "name": "yarn_nodemanager_local_dirs", + "value": null + }, + { + "desc": "Enables the health check that verifies the NodeManager is connected to the ResourceManager", + "display_name": "NodeManager Connectivity Health Check", + "name": "nodemanager_connectivity_health_enabled", + "value": "true" + }, + { + "desc": "Number of threads to handle localization requests.", + "display_name": "Localizer Client Thread Count", + "name": "yarn_nodemanager_localizer_client_thread_count", + "value": "5" + }, + { + "desc": "The local filesystem directory in which the NodeManager stores state when recovery is enabled. Recovery is enabled by default.", + "display_name": "NodeManager Recovery Directory", + "name": "yarn_nodemanager_recovery_dir", + "value": "/var/lib/hadoop-yarn/yarn-nm-recovery" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "Number of threads to use for localization fetching.", + "display_name": "Localizer Fetch Thread Count", + "name": "yarn_nodemanager_localizer_fetch_thread_count", + "value": "4" + }, + { + "desc": "Target size of localizer cache in MB, per local directory.", + "display_name": "Localizer Cache Target Size", + "name": "yarn_nodemanager_localizer_cache_target_size_mb", + "value": "10240" + }, + { + "desc": "The address of the NodeManager IPC.", + "display_name": "NodeManager IPC Address", + "name": "yarn_nodemanager_address", + "value": "8041" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's The local filesystem directory in which the NodeManager stores state when recovery is enabled. Recovery is enabled by default.. Specified as a percentage of the capacity on that filesystem. This setting is not used if a The local filesystem directory in which the NodeManager stores state when recovery is enabled. Recovery is enabled by default. Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "NodeManager Recovery Directory Free Space Monitoring Percentage Thresholds", + "name": "nodemanager_recovery_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for NodeManager", + "name": "node_manager_java_opts", + "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "true" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "List of users explicitly whitelisted to be allowed to run containers. Users with IDs lower than the \"Minimum User Id\" setting may be whitelisted by using this setting.", + "display_name": "Allowed System Users", + "name": "container_executor_allowed_system_users", + "value": "nobody,impala,hive,llama" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of NodeManager in Bytes", + "name": "node_manager_java_heapsize", + "value": "1073741824" + }, + { + "desc": "Directory where NodeManager will place its log files.", + "display_name": "NodeManager Log Directory", + "name": "node_manager_log_dir", + "value": "/var/log/hadoop-yarn" + }, + { + "desc": "The system group that owns the container-executor binary. This does not need to be changed unless the ownership of the binary is explicitly changed.", + "display_name": "Container Executor Group", + "name": "container_executor_group", + "value": "yarn" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "NodeManager Environment Advanced Configuration Snippet (Safety Valve)", + "name": "NODEMANAGER_role_env_safety_valve", + "value": null + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", + "display_name": "Garbage Collection Duration Thresholds", + "name": "nodemanager_gc_duration_thresholds", + "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "nodemanager_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", + "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", + "name": "hadoop_metrics2_safety_valve", + "value": null + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's The local filesystem directory in which the NodeManager stores state when recovery is enabled. Recovery is enabled by default..", + "display_name": "NodeManager Recovery Directory Free Space Monitoring Absolute Thresholds", + "name": "nodemanager_recovery_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Environment variables that should be forwarded from the NodeManager's environment to the container's.", + "display_name": "Containers Environment Variable", + "name": "yarn_nodemanager_admin_env", + "value": "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX" + }, + { + "desc": "The period to review when computing the moving average of garbage collection time.", + "display_name": "Garbage Collection Duration Monitoring Period", + "name": "nodemanager_gc_duration_window", + "value": "5" + }, + { + "desc": "The minimum Linux user ID allowed. Used to prevent other super users.", + "display_name": "Minimum User ID", + "name": "container_executor_min_user_id", + "value": "1000" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "NodeManager Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Number of seconds after an application finishes before the NodeManager's DeletionService will delete the application's localized file and log directory. To diagnose YARN application problems, set this property's value large enough (for example, to 600 = 10 minutes) to permit examination of these directories.", + "display_name": "Localized Dir Deletion Delay", + "name": "yarn_nodemanager_delete_debug_delay_sec", + "value": "0" + }, + { + "desc": "HDFS directory where application logs are stored when an application completes.", + "display_name": "Remote App Log Directory", + "name": "yarn_nodemanager_remote_app_log_dir", + "value": "/tmp/logs" + }, + { + "desc": "Time in seconds to retain user logs. Only applicable if log aggregation is disabled.", + "display_name": "Log Retain Duration", + "name": "yarn_nodemanager_log_retain_seconds", + "value": "10800" + }, + { + "desc": "Number of threads used in cleanup.", + "display_name": "Cleanup Thread Count", + "name": "yarn_nodemanager_delete_thread_count", + "value": "4" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "Number of threads container manager uses.", + "display_name": "Container Manager Thread Count", + "name": "yarn_nodemanager_container_manager_thread_count", + "value": "20" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "Address where the localizer IPC is.", + "display_name": "Localizer Port", + "name": "yarn_nodemanager_localizer_address", + "value": "8040" + }, + { + "desc": "Address where the localizer IPC is.", + "display_name": "Localizer Cache Cleanup Interval", + "name": "yarn_nodemanager_localizer_cache_cleanup_interval_ms", + "value": "600000" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "Absolute path to the script which is periodically run by the node health monitoring service to determine if the node is healthy or not. If the value of this key is empty or the file does not exist in the location configured here, the node health monitoring service is not started.", + "display_name": "Healthchecker Script Path", + "name": "mapred_healthchecker_script_path", + "value": "" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The HTTPS port of the NodeManager web application.", + "display_name": "NodeManager Web Application HTTPS Port (SSL)", + "name": "nodemanager_webserver_https_port", + "value": "8044" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "When computing the overall NodeManager health, consider the host's health.", + "display_name": "NodeManager Host Health Test", + "name": "nodemanager_host_health_enabled", + "value": "true" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's List of directories on the local filesystem where a NodeManager stores intermediate data files..", + "display_name": "NodeManager Local Directories Free Space Monitoring Absolute Thresholds", + "name": "nodemanager_local_data_directories_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "List of directories on the local filesystem where a NodeManager stores container log files.", + "display_name": "NodeManager Container Log Directories", + "name": "yarn_nodemanager_log_dirs", + "value": "/var/log/hadoop-yarn/container" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Path (rooted in the cgroups hierarchy on the machine) where to place YARN-managed cgroups.", + "display_name": "CGroups Hierarchy", + "name": "linux_container_executor_cgroups_hierarchy", + "value": "/hadoop-yarn" + }, + { + "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", + "display_name": "Web Metric Collection", + "name": "nodemanager_web_metric_collection_enabled", + "value": "true" + }, + { + "desc": "For advanced use only, a string to be inserted into mapred-site.xml for this role only.", + "display_name": "NodeManager Advanced Configuration Snippet (Safety Valve) for mapred-site.xml", + "name": "nodemanager_mapred_safety_valve", + "value": null + }, + { + "desc": "The maximum size, in megabytes, per log file for NodeManager logs. Typically used by log4j or logback.", + "display_name": "NodeManager Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "The remote log dir will be created at {yarn.nodemanager.remote-app-log-dir}/${user}/{thisParam}", + "display_name": "Remote App Log Directory Suffix", + "name": "yarn_nodemanager_remote_app_log_dir_suffix", + "value": "logs" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "The maximum number of rolled log files to keep for NodeManager logs. Typically used by log4j or logback.", + "display_name": "NodeManager Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "Amount of physical memory, in MiB, that can be allocated for containers.", + "display_name": "Container Memory", + "name": "yarn_nodemanager_resource_memory_mb", + "value": "8192" + }, + { + "desc": "The health test thresholds on the duration of the metrics request to the web server.", + "display_name": "Web Metric Collection Duration", + "name": "nodemanager_web_metric_collection_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Number of virtual CPU cores that can be allocated for containers. This parameter has no effect prior to CDH 4.4.", + "display_name": "Container Virtual CPU Cores", + "name": "yarn_nodemanager_resource_cpu_vcores", + "value": "8" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "Environment variables that containers may override rather than use NodeManager's default.", + "display_name": "Containers Environment Variables Whitelist ", + "name": "yarn_nodemanager_env_whitelist", + "value": "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "Comma-separated list of arguments which are to be passed to node health script when it is being launched.", + "display_name": "Healthchecker Script Arguments", + "name": "mapred_healthchecker_script_args", + "value": "" + }, + { + "desc": "List of users banned from running containers.", + "display_name": "Banned System Users", + "name": "container_executor_banned_users", + "value": "hdfs,yarn,mapred,bin" + }, + { + "desc": "The minimum log level for NodeManager logs", + "display_name": "NodeManager Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's List of directories on the local filesystem where a NodeManager stores intermediate data files.. Specified as a percentage of the capacity on that filesystem. This setting is not used if a List of directories on the local filesystem where a NodeManager stores intermediate data files. Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "NodeManager Local Directories Free Space Monitoring Percentage Thresholds", + "name": "nodemanager_local_data_directories_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/yarn-resourcemanager.json b/sahara/plugins/cdh/v5_4_0/resources/yarn-resourcemanager.json new file mode 100644 index 00000000..893d4241 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/yarn-resourcemanager.json @@ -0,0 +1,506 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "resourcemanager_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "

This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.

Each rule has some or all of the following fields:


Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}

This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.

", + "display_name": "Rules to Extract Events from Log Files", + "name": "log_event_whitelist", + "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" + }, + { + "desc": "An XML string that will be inserted verbatim into the Fair Scheduler allocations file. For CDH5, overrides the configuration set using the Pools configuration UI. For CDH4, this is the only way to configure the Fair Scheduler for YARN.", + "display_name": "Fair Scheduler XML Advanced Configuration Snippet (Safety Valve)", + "name": "resourcemanager_fair_scheduler_configuration", + "value": null + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "For advanced use only, a string to be inserted into nodes_allow.txt for this role only.", + "display_name": "ResourceManager Advanced Configuration Snippet (Safety Valve) for nodes_allow.txt", + "name": "rm_hosts_allow_safety_valve", + "value": null + }, + { + "desc": "The maximum number of application attempts. This is a global setting for all ApplicationMasters. Each ApplicationMaster can specify its individual maximum through the API, but if the individual maximum is more than the global maximum, the ResourceManager overrides it.", + "display_name": "ApplicationMaster Maximum Attempts", + "name": "yarn_resourcemanager_am_max_retries", + "value": "2" + }, + { + "desc": "For advanced use only, a string to be inserted into nodes_exclude.txt for this role only.", + "display_name": "ResourceManager Advanced Configuration Snippet (Safety Valve) for nodes_exclude.txt", + "name": "rm_hosts_exclude_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds on the duration of the metrics request to the web server.", + "display_name": "Web Metric Collection Duration", + "name": "resourcemanager_web_metric_collection_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The maximum number of completed applications that the ResourceManager keeps.", + "display_name": "Max Completed Applications", + "name": "yarn_resourcemanager_max_completed_applications", + "value": "10000" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "For advanced use only, a string to be inserted into yarn-site.xml for this role only.", + "display_name": "ResourceManager Advanced Configuration Snippet (Safety Valve) for yarn-site.xml", + "name": "resourcemanager_config_safety_valve", + "value": null + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for ResourceManager", + "name": "resource_manager_java_opts", + "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" + }, + { + "desc": "Enables the health test that the ResourceManager's process state is consistent with the role configuration", + "display_name": "ResourceManager Process Health Test", + "name": "resourcemanager_scm_health_enabled", + "value": "true" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "Directory where ResourceManager will place its log files.", + "display_name": "ResourceManager Log Directory", + "name": "resource_manager_log_dir", + "value": "/var/log/hadoop-yarn" + }, + { + "desc": "The smallest amount of physical memory, in MiB, that can be requested for a container. If using the Capacity or FIFO scheduler (or any scheduler, prior to CDH 5), memory requests will be rounded up to the nearest multiple of this number.", + "display_name": "Container Memory Minimum", + "name": "yarn_scheduler_minimum_allocation_mb", + "value": "1024" + }, + { + "desc": "The number of threads used to handle applications manager requests.", + "display_name": "Client Thread Count", + "name": "yarn_resourcemanager_client_thread_count", + "value": "50" + }, + { + "desc": "The periodic interval that the ResourceManager will check whether containers are still alive.", + "display_name": "Container Monitor Interval", + "name": "yarn_resourcemanager_container_liveness_monitor_interval_ms", + "value": "600000" + }, + { + "desc": "The HTTP port of the ResourceManager web application.", + "display_name": "ResourceManager Web Application HTTP Port", + "name": "resourcemanager_webserver_port", + "value": "8088" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "false" + }, + { + "desc": "The expiry interval to wait until a NodeManager is considered dead.", + "display_name": "NodeManager Monitor Expiry", + "name": "yarn_nm_liveness_monitor_expiry_interval_ms", + "value": "600000" + }, + { + "desc": "The largest amount of physical memory, in MiB, that can be requested for a container.", + "display_name": "Container Memory Maximum", + "name": "yarn_scheduler_maximum_allocation_mb", + "value": "65536" + }, + { + "desc": "The largest number of virtual CPU cores that can be requested for a container. This parameter has no effect prior to CDH 4.4.", + "display_name": "Container Virtual CPU Cores Maximum", + "name": "yarn_scheduler_maximum_allocation_vcores", + "value": "32" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", + "display_name": "Web Metric Collection", + "name": "resourcemanager_web_metric_collection_enabled", + "value": "true" + }, + { + "desc": "Number of threads to handle resource tracker calls.", + "display_name": "Resource Tracker Thread Count", + "name": "yarn_resourcemanager_resource_tracker_client_thread_count", + "value": "50" + }, + { + "desc": "If using the Fair Scheduler, virtual core requests will be rounded up to the nearest multiple of this number. This parameter has no effect prior to CDH 5.", + "display_name": "Container Virtual CPU Cores Increment", + "name": "yarn_scheduler_increment_allocation_vcores", + "value": "1" + }, + { + "desc": "The address of the scheduler interface in the ResourceManager.", + "display_name": "Scheduler Address", + "name": "yarn_resourcemanager_scheduler_address", + "value": "8030" + }, + { + "desc": "For advanced use only, a string to be inserted into mapred-site.xml for this role only.", + "display_name": "ResourceManager Advanced Configuration Snippet (Safety Valve) for mapred-site.xml", + "name": "resourcemanager_mapred_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The HTTPS port of the ResourceManager web application.", + "display_name": "ResourceManager Web Application HTTPS Port (SSL)", + "name": "resourcemanager_webserver_https_port", + "value": "8090" + }, + { + "desc": "The period to review when computing the moving average of garbage collection time.", + "display_name": "Garbage Collection Duration Monitoring Period", + "name": "resourcemanager_gc_duration_window", + "value": "5" + }, + { + "desc": "For applications that request containers on particular racks, the number of scheduling opportunities since the last container assignment to wait before accepting a placement on another rack. Expressed as a float between 0 and 1, which, as a fraction of the cluster size, is the number of scheduling opportunities to pass up. If not set, this means don't pass up any scheduling opportunities. Requires Fair Scheduler continuous scheduling to be disabled. If continuous scheduling is enabled, yarn.scheduler.fair.locality-delay-rack-ms should be used instead.", + "display_name": "Fair Scheduler Rack Locality Threshold", + "name": "resourcemanager_fair_scheduler_locality_threshold_rack", + "value": null + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", + "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", + "name": "hadoop_metrics2_safety_valve", + "value": null + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "The address of the applications manager interface in the ResourceManager.", + "display_name": "ResourceManager Address", + "name": "yarn_resourcemanager_address", + "value": "8032" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "ResourceManager Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "Enables multiple Fair Scheduler container assignments in one heartbeat, which improves cluster throughput when there are many small tasks to run.", + "display_name": "Fair Scheduler Assign Multiple Tasks", + "name": "resourcemanager_fair_scheduler_assign_multiple", + "value": "false" + }, + { + "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", + "display_name": "Garbage Collection Duration Thresholds", + "name": "resourcemanager_gc_duration_thresholds", + "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" + }, + { + "desc": "When enabled, any applications that were running on the cluster when the ResourceManager died will be recovered when the ResourceManager next starts. Note: If RM-HA is enabled, then this configuration is always enabled.", + "display_name": "Enable ResourceManager Recovery", + "name": "yarn_resourcemanager_recovery_enabled", + "value": "false" + }, + { + "desc": "The periodic interval that the ResourceManager will check whether ApplicationMasters is still alive.", + "display_name": "ApplicationMaster Monitor Interval", + "name": "yarn_resourcemanager_amliveliness_monitor_interval_ms", + "value": "1000" + }, + { + "desc": "For applications that request containers on particular nodes, the minimum time in milliseconds the Fair Scheduler waits before accepting a placement on another node. Requires Fair Scheduler continuous scheduling to be enabled. If continuous scheduling is disabled, yarn.scheduler.fair.locality.threshold.node should be used instead.", + "display_name": "Fair Scheduler Node Locality Delay", + "name": "yarn_scheduler_fair_locality_delay_node_ms", + "value": "2000" + }, + { + "desc": "Number of threads used to handle the ResourceManager admin interface.", + "display_name": "Admin Client Thread Count", + "name": "yarn_resourcemanager_admin_client_thread_count", + "value": "1" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of ResourceManager in Bytes", + "name": "resource_manager_java_heapsize", + "value": "1073741824" + }, + { + "desc": "The maximum size, in megabytes, per log file for ResourceManager logs. Typically used by log4j or logback.", + "display_name": "ResourceManager Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "When enabled, if a pool's minimum share is not met for some period of time, the Fair Scheduler preempts applications in other pools. Preemption guarantees that production applications are not starved while also allowing the cluster to be used for experimental and research applications. To minimize wasted computation, the Fair Scheduler preempts the most recently launched applications.", + "display_name": "Fair Scheduler Preemption", + "name": "resourcemanager_fair_scheduler_preemption", + "value": "false" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The smallest number of virtual CPU cores that can be requested for a container. If using the Capacity or FIFO scheduler (or any scheduler, prior to CDH 5), virtual core requests will be rounded up to the nearest multiple of this number. This parameter has no effect prior to CDH 4.4.", + "display_name": "Container Virtual CPU Cores Minimum", + "name": "yarn_scheduler_minimum_allocation_vcores", + "value": "1" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "ResourceManager Environment Advanced Configuration Snippet (Safety Valve)", + "name": "RESOURCEMANAGER_role_env_safety_valve", + "value": null + }, + { + "desc": "If using the Fair Scheduler, memory requests will be rounded up to the nearest multiple of this number. This parameter has no effect prior to CDH 5.", + "display_name": "Container Memory Increment", + "name": "yarn_scheduler_increment_allocation_mb", + "value": "512" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "If enabled, the ResourceManager binds to the wildcard address (\"0.0.0.0\") on all of its ports.", + "display_name": "Bind ResourceManager to Wildcard Address", + "name": "yarn_rm_bind_wildcard", + "value": "false" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "For applications that request containers on particular racks, the minimum time in milliseconds the Fair Scheduler waits before accepting a placement on another rack. Requires Fair Scheduler continuous scheduling to be enabled. If continuous scheduling is disabled, yarn.scheduler.fair.locality.threshold.rack should be used instead.", + "display_name": "Fair Scheduler Rack Locality Delay", + "name": "yarn_scheduler_fair_locality_delay_rack_ms", + "value": "4000" + }, + { + "desc": "Enable continuous scheduling in the Fair Scheduler. When enabled, scheduling decisions are decoupled from NodeManager heartbeats, leading to faster resource allocations.", + "display_name": "Enable Fair Scheduler Continuous Scheduling", + "name": "yarn_scheduler_fair_continuous_scheduling_enabled", + "value": "true" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "When computing the overall ResourceManager health, consider the host's health.", + "display_name": "ResourceManager Host Health Test", + "name": "resourcemanager_host_health_enabled", + "value": "true" + }, + { + "desc": "The number of threads used to handle requests through the scheduler interface.", + "display_name": "Scheduler Thread Count", + "name": "yarn_resourcemanager_scheduler_client_thread_count", + "value": "50" + }, + { + "desc": "The maximum number of rolled log files to keep for ResourceManager logs. Typically used by log4j or logback.", + "display_name": "ResourceManager Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "When enabled, the Fair Scheduler will assign shares to individual apps based on their size, rather than providing an equal share to all apps regardless of size.", + "display_name": "Fair Scheduler Size-Based Weight", + "name": "resourcemanager_fair_scheduler_size_based_weight", + "value": "false" + }, + { + "desc": "For applications that request containers on particular nodes, the number of scheduling opportunities since the last container assignment to wait before accepting a placement on another node. Expressed as a float between 0 and 1, which, as a fraction of the cluster size, is the number of scheduling opportunities to pass up. If not set, this means don't pass up any scheduling opportunities. Requires Fair Scheduler continuous scheduling to be disabled. If continuous scheduling is enabled, yarn.scheduler.fair.locality-delay-node-ms should be used instead.", + "display_name": "Fair Scheduler Node Locality Threshold", + "name": "resourcemanager_fair_scheduler_locality_threshold_node", + "value": null + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "When enabled, ResourceManager has proxy user privileges.", + "display_name": "Enable ResourceManger Proxy User Privileges", + "name": "yarn_resourcemanager_proxy_user_privileges_enabled", + "value": "true" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "When set to true, the Fair Scheduler uses the username as the default pool name, in the event that a pool name is not specified. When set to false, all applications are run in a shared pool, called default.", + "display_name": "Fair Scheduler User As Default Queue", + "name": "resourcemanager_fair_scheduler_user_as_default_queue", + "value": "true" + }, + { + "desc": "The address of the resource tracker interface in the ResourceManager.", + "display_name": "Resource Tracker Address", + "name": "yarn_resourcemanager_resource_tracker_address", + "value": "8031" + }, + { + "desc": "The expiry interval to wait until an ApplicationMaster is considered dead.", + "display_name": "ApplicationMaster Monitor Expiry", + "name": "yarn_am_liveness_monitor_expiry_interval_ms", + "value": "600000" + }, + { + "desc": "The class to use as the resource scheduler. FairScheduler is only supported in CDH 4.2.1 and later.", + "display_name": "Scheduler Class", + "name": "yarn_resourcemanager_scheduler_class", + "value": "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler" + }, + { + "desc": "The minimum log level for ResourceManager logs", + "display_name": "ResourceManager Logging Threshold", + "name": "log_threshold", + "value": "INFO" + }, + { + "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", + "display_name": "Health Check Startup Tolerance", + "name": "resourcemanager_startup_tolerance_minutes", + "value": "5" + }, + { + "desc": "The address of the admin interface in the ResourceManager.", + "display_name": "Administration Address", + "name": "yarn_resourcemanager_admin_address", + "value": "8033" + }, + { + "desc": "The periodic interval that the ResourceManager will check whether NodeManagers are still alive.", + "display_name": "NodeManager Monitor Interval", + "name": "yarn_resourcemanager_nm_liveness_monitor_interval_ms", + "value": "1000" + }, + { + "desc": "Enter an XML string that represents the Capacity Scheduler configuration.", + "display_name": "Capacity Scheduler Configuration", + "name": "resourcemanager_capacity_scheduler_configuration", + "value": "\n\n \n yarn.scheduler.capacity.root.queues\n default\n \n \n yarn.scheduler.capacity.root.capacity\n 100\n \n \n yarn.scheduler.capacity.root.default.capacity\n 100\n \n\n" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/yarn-service.json b/sahara/plugins/cdh/v5_4_0/resources/yarn-service.json new file mode 100644 index 00000000..14cc2120 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/yarn-service.json @@ -0,0 +1,290 @@ +[ + { + "desc": "ACL that determines which users and groups can submit and kill applications in any pool, and can issue commands on ResourceManager roles.", + "display_name": "Admin ACL", + "name": "yarn_admin_acl", + "value": "*" + }, + { + "desc": "The health test thresholds of the overall NodeManager health. The check returns \"Concerning\" health if the percentage of \"Healthy\" NodeManagers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" NodeManagers falls below the critical threshold.", + "display_name": "Healthy NodeManager Monitoring Thresholds", + "name": "yarn_nodemanagers_healthy_thresholds", + "value": "{\"critical\":\"90.0\",\"warning\":\"95.0\"}" + }, + { + "desc": "Entries to add to the classpaths of YARN applications.", + "display_name": "YARN Application Classpath", + "name": "yarn_application_classpath", + "value": "$HADOOP_CLIENT_CONF_DIR,$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,$HADOOP_YARN_HOME/*,$HADOOP_YARN_HOME/lib/*" + }, + { + "desc": "JSON representation of all the configurations that the Fair Scheduler can take on across all schedules. Typically edited using the Pools configuration UI.", + "display_name": "Fair Scheduler Allocations", + "name": "yarn_fs_scheduled_allocations", + "value": "{\"defaultMinSharePreemptionTimeout\":null,\"defaultQueueSchedulingPolicy\":null,\"fairSharePreemptionTimeout\":null,\"queueMaxAMShareDefault\":null,\"queueMaxAppsDefault\":null,\"queuePlacementRules\":null,\"queues\":[{\"aclAdministerApps\":null,\"aclSubmitApps\":null,\"minSharePreemptionTimeout\":null,\"name\":\"root\",\"queues\":[{\"aclAdministerApps\":null,\"aclSubmitApps\":null,\"minSharePreemptionTimeout\":null,\"name\":\"default\",\"queues\":[],\"schedulablePropertiesList\":[{\"impalaMaxMemory\":null,\"impalaMaxQueuedQueries\":null,\"impalaMaxRunningQueries\":null,\"maxAMShare\":null,\"maxResources\":null,\"maxRunningApps\":null,\"minResources\":null,\"scheduleName\":\"default\",\"weight\":null}],\"schedulingPolicy\":null}],\"schedulablePropertiesList\":[{\"impalaMaxMemory\":null,\"impalaMaxQueuedQueries\":null,\"impalaMaxRunningQueries\":null,\"maxAMShare\":null,\"maxResources\":null,\"maxRunningApps\":null,\"minResources\":null,\"scheduleName\":\"default\",\"weight\":null}],\"schedulingPolicy\":null}],\"userMaxAppsDefault\":null,\"users\":[]}" + }, + { + "desc": "Controls which applications non-admin users can see in the applications list view", + "display_name": "Non-Admin Users Applications List Visibility Settings", + "name": "user_application_list_settings", + "value": "ALL" + }, + { + "desc": "When computing the overall YARN health, consider JobHistory Server's health", + "display_name": "JobHistory Server Role Health Test", + "name": "yarn_jobhistoryserver_health_enabled", + "value": "true" + }, + { + "desc": "A list specifying the rules to run to determine which Fair Scheduler configuration to use. Typically edited using the Rules configuration UI.", + "display_name": "Fair Scheduler Configuration Rules", + "name": "yarn_fs_schedule_rules", + "value": "[]" + }, + { + "desc": "Password that protects the private key contained in the server keystore used for encrypted shuffle and encrypted web UIs. Applies to all configurations of daemon roles of this service.", + "display_name": "Hadoop SSL Server Keystore Key Password", + "name": "ssl_server_keystore_keypassword", + "value": null + }, + { + "desc": "The user that this service's processes should run as. (Except the Job History Server, which has its own user)", + "display_name": "System User", + "name": "process_username", + "value": "yarn" + }, + { + "desc": "The home directory of the system user on the local filesystem. This setting must reflect the system's configured value - only changing it here will not change the actual home directory.", + "display_name": "System User's Home Directory", + "name": "hdfs_user_home_dir", + "value": "/var/lib/hadoop-yarn" + }, + { + "desc": "Whether YARN creates a cgroup per container, thereby isolating the CPU usage of containers. When set, yarn.nodemanager.linux-container-executor.resources-handler.class is configured to org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler. The host (in Cloudera Manager) must have cgroups enabled. The number of shares allocated to all YARN containers is configured by adjusting the CPU shares value of the Node Manager in the Resource Management configuration group.", + "display_name": "Use CGroups for Resource Management", + "name": "yarn_service_cgroups", + "value": "false" + }, + { + "desc": "Name of the HDFS service that this YARN service instance depends on", + "display_name": "HDFS Service", + "name": "hdfs_service", + "value": null + }, + { + "desc": "Cluster ID used when ResourceManager is Highly Available.", + "display_name": "RM-HA Cluster ID", + "name": "yarn_rm_ha_cluster_id", + "value": "yarnRM" + }, + { + "desc": "For advanced use only, a string to be inserted into yarn-site.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "YARN Service Advanced Configuration Snippet (Safety Valve) for yarn-site.xml", + "name": "yarn_service_config_safety_valve", + "value": null + }, + { + "desc": "Password for the server keystore file used for encrypted shuffle and encrypted web UIs. Applies to configurations of all daemon roles of this service.", + "display_name": "Hadoop SSL Server Keystore File Password", + "name": "ssl_server_keystore_password", + "value": null + }, + { + "desc": "The tolerance window used in YARN service tests that depend on detection of the active ResourceManager.", + "display_name": "Active ResourceManager Detection Window", + "name": "yarn_active_resourcemanager_detecton_window", + "value": "3" + }, + { + "desc": "For advanced use only, a string to be inserted into ssl-server.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "YARN (MR2 Included) Service Advanced Configuration Snippet (Safety Valve) for ssl-server.xml", + "name": "yarn_ssl_server_safety_valve", + "value": null + }, + { + "desc": "Name of the ZooKeeper service that this YARN service instance depends on", + "display_name": "ZooKeeper Service", + "name": "zookeeper_service", + "value": null + }, + { + "desc": "For advanced use only, a string to be inserted into ssl-client.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "YARN (MR2 Included) Service Advanced Configuration Snippet (Safety Valve) for ssl-client.xml", + "name": "yarn_ssl_client_safety_valve", + "value": null + }, + { + "desc": "Enables Kerberos authentication for Hadoop HTTP web consoles for all roles of this service using the SPNEGO protocol. Note: This is effective only if Kerberos is enabled for the HDFS service.", + "display_name": "Enable Kerberos Authentication for HTTP Web-Consoles", + "name": "hadoop_secure_web_ui", + "value": "false" + }, + { + "desc": "When computing the overall YARN service health, whether to consider the active ResourceManager's health.", + "display_name": "Active ResourceManager Role Health Check", + "name": "yarn_resourcemanagers_health_enabled", + "value": "true" + }, + { + "desc": "Kerberos principal used by all roles of this service.", + "display_name": "Kerberos Principal", + "name": "kerberos_princ_name", + "value": "yarn" + }, + { + "desc": "When set to true, pools specified in applications but not explicitly configured, are created at runtime with default settings. When set to false, applications specifying pools not explicitly configured run in a pool named default. This setting applies when an application explicitly specifies a pool and when the application runs in a pool named with the username associated with the application.", + "display_name": "Allow Undeclared Pools", + "name": "yarn_scheduler_fair_allow_undeclared_pools", + "value": "true" + }, + { + "desc": "The amount of time after ResourceManager(s) start that the lack of an active ResourceManager will be tolerated. This is an advanced option that does not often need to be changed.", + "display_name": "ResourceManager Activation Startup Tolerance", + "name": "yarn_resourcemanager_activation_startup_tolerance", + "value": "180" + }, + { + "desc": "Path to the truststore file used when roles of this service act as SSL clients. Overrides the cluster-wide default truststore location set in HDFS. This truststore must be in JKS format. The truststore contains certificates of trusted servers, or of Certificate Authorities trusted to identify servers. The contents of the truststore can be modified without restarting any roles. By default, changes to its contents are picked up within ten seconds. If not set, the default Java truststore is used to verify certificates.", + "display_name": "SSL Client Truststore File Location", + "name": "ssl_client_truststore_location", + "value": null + }, + { + "desc": "How long to keep aggregation logs before deleting them.", + "display_name": "Log Aggregation Retention Period", + "name": "yarn_log_aggregation_retain_seconds", + "value": "604800" + }, + { + "desc": "When computing the overall YARN service health, whether to consider the health of the standby ResourceManager.", + "display_name": "Standby ResourceManager Health Check", + "name": "yarn_standby_resourcemanager_health_enabled", + "value": "true" + }, + { + "desc": "Whether users and groups specified in Admin ACL should be checked for authorization to perform admin operations.", + "display_name": "Enable ResourceManager ACLs", + "name": "yarn_acl_enable", + "value": "true" + }, + { + "desc": "The group that this service's processes should run as. (Except the Job History Server, which has its own group)", + "display_name": "System Group", + "name": "process_groupname", + "value": "hadoop" + }, + { + "desc": "For advanced use only, a string to be inserted into hadoop-policy.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "YARN Service Advanced Configuration Snippet (Safety Valve) for hadoop-policy.xml", + "name": "yarn_hadoop_policy_config_safety_valve", + "value": null + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Service Level Health Alerts", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "Password for the SSL client truststore. Overrides the cluster-wide default truststore password set in HDFS.", + "display_name": "SSL Client Truststore File Password", + "name": "ssl_client_truststore_password", + "value": null + }, + { + "desc": "The frequency in which the log4j event publication appender will retry sending undelivered log events to the Event server, in seconds", + "display_name": "Log Event Retry Frequency", + "name": "log_event_retry_frequency", + "value": "30" + }, + { + "desc": "For advanced use only, a string to be inserted into core-site.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "YARN Service Advanced Configuration Snippet (Safety Valve) for core-site.xml", + "name": "yarn_core_site_safety_valve", + "value": null + }, + { + "desc": "

The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n  \"streamThreshold\": 10, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Service Triggers", + "name": "service_triggers", + "value": "[]" + }, + { + "desc": "Controls which applications an admin user can see in the applications list view", + "display_name": "Admin Users Applications List Visibility Settings", + "name": "admin_application_list_settings", + "value": "ALL" + }, + { + "desc": "Whether YARN uses the Linux Container Executor both in secure (Kerberos) and insecure (not Kerberos) environments. Cgroups enforcement only works when the Linux Container Executor is used.", + "display_name": "Always Use Linux Container Executor", + "name": "yarn_service_lce_always", + "value": "false" + }, + { + "desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.", + "display_name": "Enable Log Event Capture", + "name": "catch_events", + "value": "true" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Comma-delimited list of groups that you want to allow the Llama (AM for Impala) user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", + "display_name": "Llama Proxy User Groups", + "name": "llama_proxy_user_groups_list", + "value": "*" + }, + { + "desc": "Path to the keystore file containing the server certificate and private key used for encrypted shuffle and encrypted web UIs. Applies to configurations of all daemon roles of this service.", + "display_name": "Hadoop SSL Server Keystore File Location", + "name": "ssl_server_keystore_location", + "value": null + }, + { + "desc": "Controls the aggregate metrics generated for YARN applications. The structure is a JSON list of the attributes to aggregate and the entities to aggregate to. For example, if the attributeName is 'maps_completed' and the aggregationTargets is ['USER'] then the Service Monitor will create the metric 'yarn_application_maps_completed_rate' and, every ten minutes, will record the total maps completed for each user across all their YARN applications. By default it will also record the number of applications submitted ('apps_submitted_rate') for both users and pool. For a full list of the supported attributes see the YARN search page. Note that the valid aggregation targets are USER, YARN_POOL, and YARN (the service), and that these aggregate metrics can be viewed on both the reports and charts search pages.", + "display_name": "YARN Application Aggregates", + "name": "yarn_application_aggregates", + "value": "[\n {\n \"attributeName\": \"maps_total\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"reduces_total\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"cpu_milliseconds\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"mb_millis_maps\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"mb_millis_reduces\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"vcores_millis_maps\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"vcores_millis_reduces\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"file_bytes_read\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"file_bytes_written\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"hdfs_bytes_read\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"hdfs_bytes_written\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"cm_cpu_milliseconds\",\n \"aggregationTargets\": [\"USER\"]\n }\n]\n" + }, + { + "desc": "Comma-delimited list of hosts where you want to allow the Llama (AM for Impala) user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", + "display_name": "Llama Proxy User Hosts", + "name": "llama_proxy_user_hosts_list", + "value": "*" + }, + { + "desc": "UNIX user that containers run as when Linux-container-executor is used in nonsecure mode.", + "display_name": "UNIX User for Nonsecure Mode with Linux Container Executor", + "name": "yarn_nodemanager_linux_container_executor_nonsecure_mode_local_user", + "value": "nobody" + }, + { + "desc": "For advanced use only, a string to be inserted into mapred-site.xml. Applies to configurations of all roles in this service except client configuration.", + "display_name": "YARN Service MapReduce Advanced Configuration Snippet (Safety Valve)", + "name": "yarn_service_mapred_safety_valve", + "value": null + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", + "display_name": "YARN (MR2 Included) Service Environment Advanced Configuration Snippet (Safety Valve)", + "name": "yarn_service_env_safety_valve", + "value": null + }, + { + "desc": "Whether to enable log aggregation", + "display_name": "Enable Log Aggregation", + "name": "yarn_log_aggregation_enable", + "value": "true" + }, + { + "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", + "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", + "name": "smon_derived_configs_safety_valve", + "value": null + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/zookeeper-server.json b/sahara/plugins/cdh/v5_4_0/resources/zookeeper-server.json new file mode 100644 index 00000000..e20e80a3 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/zookeeper-server.json @@ -0,0 +1,410 @@ +[ + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", + "name": "log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "

This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.

Each rule has some or all of the following fields:


Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}

This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.

", + "display_name": "Rules to Extract Events from Log Files", + "name": "log_event_whitelist", + "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" + }, + { + "desc": "The port to monitor for inter-server communication", + "display_name": "Quorum Port", + "name": "quorumPort", + "value": "3181" + }, + { + "desc": "Whether or not periodic stacks collection is enabled.", + "display_name": "Stacks Collection Enabled", + "name": "stacks_collection_enabled", + "value": "false" + }, + { + "desc": "The port to monitor for leadership election", + "display_name": "Election Port", + "name": "electionPort", + "value": "4181" + }, + { + "desc": "Specifies the name of the user that has read-only privileges when using password file based authentication for JMX access. JMX authentication must be enabled for this setting to take effect.", + "display_name": "Name of User with Read-Only access to the JMX Agent", + "name": "jmx_passwd_file_readonly_user", + "value": "monitorRole" + }, + { + "desc": "The health check thresholds of the weighted average size of the ZooKeeper Server connection count over a recent period. See ZooKeeper Server Connection Count Monitoring Period.", + "display_name": "ZooKeeper Server Connection Count Thresholds", + "name": "zookeeper_server_connection_count_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", + "name": "heap_dump_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The disk location that ZooKeeper will use to store its transaction logs.", + "display_name": "Transaction Log Directory", + "name": "dataLogDir", + "value": "/var/lib/zookeeper" + }, + { + "desc": "The port used by the ZooKeeper Server's RMI registry. This is required to enable JMX access through RMI which is required for Cloudera Manager ZooKeeper monitoring. This is added as \"-Dcom.sun.management.jmxremote.port\" to the ZooKeeper Server's JVM command line.", + "display_name": "JMX Remote Port", + "name": "server_jmx_agent_port", + "value": "9010" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", + "display_name": "Server Environment Advanced Configuration Snippet (Safety Valve)", + "name": "SERVER_role_env_safety_valve", + "value": null + }, + { + "desc": "The tolerance window that will be used in the detection of a ZooKeeper server's membership in a quorum. Specified in minutes.", + "display_name": "Quorum Membership Detection Window", + "name": "zookeeper_server_quorum_membership_detection_window", + "value": "3" + }, + { + "desc": "Enables the health test that the Server's process state is consistent with the role configuration", + "display_name": "Server Process Health Test", + "name": "zookeeper_server_scm_health_enabled", + "value": "true" + }, + { + "desc": "The address (IPv4, IPv6, or hostname) to monitor for client connections. This is the address that clients attempt to connect to. This setting is optional, because by default, ZooKeeper binds in such a way that any connection to the client port for any address/interface/NIC on the server will be accepted.", + "display_name": "Client Port Address", + "name": "clientPortAddress", + "value": null + }, + { + "desc": "When computing the overall Server health, consider the host's health.", + "display_name": "Server Host Health Test", + "name": "zookeeper_server_host_health_enabled", + "value": "true" + }, + { + "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", + "display_name": "Stacks Collection Method", + "name": "stacks_collection_method", + "value": "jstack" + }, + { + "desc": "Specifies the password of the user that has read-only privileges when using password file based authentication for JMX access. JMX authentication must be enabled for this setting to take effect.", + "display_name": "Password of User with Read-Only Access to the JMX agent", + "name": "jmx_passwd_file_readonly_user_password", + "value": "MONITOR" + }, + { + "desc": "The health check thresholds of the weighted average size of the ZooKeeper Server outstanding requests queue over a recent period. See ZooKeeper Server Outstanding Requests Monitoring Period.", + "display_name": "ZooKeeper Server Outstanding Requests Thresholds", + "name": "zookeeper_server_outstanding_requests_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The period to review when computing the moving average of the outstanding requests queue size. Specified in minutes.", + "display_name": "ZooKeeper Server Outstanding Requests Monitoring Period", + "name": "zookeeper_server_outstanding_requests_window", + "value": "3" + }, + { + "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", + "display_name": "Java Heap Size of ZooKeeper Server in Bytes", + "name": "zookeeper_server_java_heapsize", + "value": "1073741824" + }, + { + "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", + "display_name": "Automatically Restart Process", + "name": "process_auto_restart", + "value": "true" + }, + { + "desc": "

The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n  \"streamThreshold\": 0, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Role Triggers", + "name": "role_triggers", + "value": "[]" + }, + { + "desc": "The minimum session timeout, in milliseconds, that the ZooKeeper Server will allow the client to negotiate", + "display_name": "Minimum Session Timeout", + "name": "minSessionTimeout", + "value": "4000" + }, + { + "desc": "Specifies the name of the user that has read-write privileges when using password file based authentication for JMX access. JMX authentication must be enabled for this setting to take effect.", + "display_name": "Name of User with Read-Write Access to the JMX Agent", + "name": "jmx_passwd_file_readwrite_user", + "value": "controlRole" + }, + { + "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Soft Limit", + "name": "rm_memory_soft_limit", + "value": "-1" + }, + { + "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", + "display_name": "Dump Heap When Out of Memory", + "name": "oom_heap_dump_enabled", + "value": "false" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's The disk location that ZooKeeper will use to store its database snapshots.. Specified as a percentage of the capacity on that filesystem. This setting is not used if a The disk location that ZooKeeper will use to store its database snapshots. Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Data Directory Free Space Monitoring Percentage Thresholds", + "name": "zookeeper_server_data_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", + "display_name": "Cgroup Memory Hard Limit", + "name": "rm_memory_hard_limit", + "value": "-1" + }, + { + "desc": "The disk location that ZooKeeper will use to store its database snapshots.", + "display_name": "Data Directory", + "name": "dataDir", + "value": "/var/lib/zookeeper" + }, + { + "desc": "The health test thresholds on the swap memory usage of the process.", + "display_name": "Process Swap Memory Thresholds", + "name": "process_swap_memory_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"any\"}" + }, + { + "desc": "The period to review when computing unexpected exits.", + "display_name": "Unexpected Exits Monitoring Period", + "name": "unexpected_exits_window", + "value": "5" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's The disk location that ZooKeeper will use to store its database snapshots..", + "display_name": "Data Directory Free Space Monitoring Absolute Thresholds", + "name": "zookeeper_server_data_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup I/O Weight", + "name": "rm_io_weight", + "value": "500" + }, + { + "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", + "display_name": "Stacks Collection Data Retention", + "name": "stacks_collection_data_retention", + "value": "104857600" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", + "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", + "name": "heap_dump_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "Enables authentication when interacting with the JMX agent on the ZooKeeper server.", + "display_name": "Enable Authenticated Communication with the JMX Agent", + "name": "enable_jmx_authentication", + "value": "false" + }, + { + "desc": "The period to review when computing the moving average of the connection count. Specified in minutes.", + "display_name": "ZooKeeper Server Connection Count Monitoring Period", + "name": "zookeeper_server_connection_count_window", + "value": "3" + }, + { + "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", + "display_name": "Server Logging Advanced Configuration Snippet (Safety Valve)", + "name": "log4j_safety_valve", + "value": null + }, + { + "desc": "The maximum session timeout, in milliseconds, that the ZooKeeper Server will allow the client to negotiate", + "display_name": "Maximum Session Timeout", + "name": "maxSessionTimeout", + "value": "40000" + }, + { + "desc": "For advanced use only, a string to be inserted into zoo.cfg for this role only.", + "display_name": "Server Advanced Configuration Snippet (Safety Valve) for zoo.cfg", + "name": "zookeeper_config_safety_valve", + "value": null + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", + "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", + "name": "log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "The maximum number of concurrent connections (at the socket level) that a single client, identified by the IP address, may make to a single member of the ZooKeeper ensemble. This setting is used to prevent certain classes of DoS attacks, including file descriptor exhaustion. To remove the limit on concurrent connections, set this value to 0.", + "display_name": "Maximum Client Connections", + "name": "maxClientCnxns", + "value": "60" + }, + { + "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", + "display_name": "Maximum Process File Descriptors", + "name": "rlimit_fds", + "value": null + }, + { + "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", + "display_name": "Java Configuration Options for Zookeeper Server", + "name": "zk_server_java_opts", + "value": "" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Health Alerts for this Role", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The port used by the ZooKeeper Server's RMI server to handle JMX RMI requests. This is added as \"-Dcom.sun.management.jmxremote.rmi.port=\" to the ZooKeeper Server's JVM command line. This has an effect only in Oracle JDK 7u4 and higher. If the setting is left blank, the JMX Remote Port value is used. If set to 0 or -1, this setting is ignored. When this setting is not in effect, the JVM uses a random port for the RMI server.", + "display_name": "JMX RMI Server Port", + "name": "server_jmx_rmi_port", + "value": null + }, + { + "desc": "The maximum size, in megabytes, per log file for Server logs. Typically used by log4j or logback.", + "display_name": "Server Max Log Size", + "name": "max_log_size", + "value": "200" + }, + { + "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", + "display_name": "Kill When Out of Memory", + "name": "oom_sigkill_enabled", + "value": "true" + }, + { + "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", + "display_name": "File Descriptor Monitoring Thresholds", + "name": "zookeeper_server_fd_thresholds", + "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" + }, + { + "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", + "display_name": "Unexpected Exits Thresholds", + "name": "unexpected_exits_thresholds", + "value": "{\"critical\":\"any\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's The disk location that ZooKeeper will use to store its transaction logs..", + "display_name": "Transaction Log Directory Free Space Monitoring Absolute Thresholds", + "name": "zookeeper_server_data_log_directory_free_space_absolute_thresholds", + "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "The port to monitor for client connections. This is the port that clients attempt to connect to.", + "display_name": "Client Port", + "name": "clientPort", + "value": "2181" + }, + { + "desc": "The period to review when computing the moving average of garbage collection time.", + "display_name": "Garbage Collection Duration Monitoring Period", + "name": "zookeeper_server_gc_duration_window", + "value": "5" + }, + { + "desc": "Specifies the password of the user that has read-write privileges when using password file based authentication for JMX access. JMX authentication must be enabled for this setting to take effect.", + "display_name": "Password of user with read-write access to the JMX agent", + "name": "jmx_passwd_file_readwrite_user_password", + "value": "CONTROL" + }, + { + "desc": "Directory where ZooKeeper will place its log files.", + "display_name": "ZooKeeper Log Directory", + "name": "zk_server_log_dir", + "value": "/var/log/zookeeper" + }, + { + "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", + "display_name": "Stacks Collection Directory", + "name": "stacks_collection_directory", + "value": null + }, + { + "desc": "The maximum number of rolled log files to keep for Server logs. Typically used by log4j or logback.", + "display_name": "Server Maximum Log File Backups", + "name": "max_log_backup_index", + "value": "10" + }, + { + "desc": "Enables the JMX agent on the ZooKeeper server. Turning this off on any of the ZooKeeper servers that are part of a service will prevent Cloudera Manager from being able to monitor that server and may affect the monitoring provided on the entire service.", + "display_name": "Enable JMX Agent", + "name": "enable_jmx_agent", + "value": "true" + }, + { + "desc": "The percentage thresholds of the ratio of the maximum request latency to the maximum client-negotiable session timeout since the server was started.", + "display_name": "Maximum Latency Monitoring Thresholds", + "name": "zookeeper_server_max_latency_thresholds", + "value": "{\"critical\":\"100.0\",\"warning\":\"75.0\"}" + }, + { + "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", + "display_name": "Cgroup CPU Shares", + "name": "rm_cpu_shares", + "value": "1024" + }, + { + "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", + "display_name": "Heap Dump Directory", + "name": "oom_heap_dump_dir", + "value": "/tmp" + }, + { + "desc": "The frequency with which stacks are collected.", + "display_name": "Stacks Collection Frequency", + "name": "stacks_collection_frequency", + "value": "5.0" + }, + { + "desc": "Enables the quorum membership check for this ZooKeeper Server.", + "display_name": "Enable the Quorum Membership Check", + "name": "zookeeper_server_quorum_membership_enabled", + "value": "true" + }, + { + "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's The disk location that ZooKeeper will use to store its transaction logs.. Specified as a percentage of the capacity on that filesystem. This setting is not used if a The disk location that ZooKeeper will use to store its transaction logs. Free Space Monitoring Absolute Thresholds setting is configured.", + "display_name": "Transaction Log Directory Free Space Monitoring Percentage Thresholds", + "name": "zookeeper_server_data_log_directory_free_space_percentage_thresholds", + "value": "{\"critical\":\"never\",\"warning\":\"never\"}" + }, + { + "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", + "display_name": "Garbage Collection Duration Thresholds", + "name": "zookeeper_server_gc_duration_thresholds", + "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" + }, + { + "desc": "Unique identifier for each ZooKeeper server, typically starts at 1", + "display_name": "ZooKeeper Server ID", + "name": "serverId", + "value": null + }, + { + "desc": "The minimum log level for Server logs", + "display_name": "Server Logging Threshold", + "name": "log_threshold", + "value": "INFO" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/zookeeper-service.json b/sahara/plugins/cdh/v5_4_0/resources/zookeeper-service.json new file mode 100644 index 00000000..e5cefad6 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/resources/zookeeper-service.json @@ -0,0 +1,158 @@ +[ + { + "desc": "Configures the path of the root znode under which all canary updates are performed", + "display_name": "ZooKeeper Canary Root Znode Path", + "name": "zookeeper_canary_root_path", + "value": "/cloudera_manager_zookeeper_canary" + }, + { + "desc": "The user that this service's processes should run as.", + "display_name": "System User", + "name": "process_username", + "value": "zookeeper" + }, + { + "desc": "The frequency in which the log4j event publication appender will retry sending undelivered log events to the Event server, in seconds", + "display_name": "Log Event Retry Frequency", + "name": "log_event_retry_frequency", + "value": "30" + }, + { + "desc": "Amount of time, in ticks, to allow followers to sync with ZooKeeper. If followers fall too far behind a leader, they are dropped.", + "display_name": "Synchronization Limit", + "name": "syncLimit", + "value": "5" + }, + { + "desc": "When enabled, ZooKeeper auto purge feature retains this many most recent snapshots and the corresponding transaction logs in the dataDir and dataLogDir respectively and deletes the rest. Defaults to 5. Minimum value is 3.", + "display_name": "Auto Purge Snapshots Retain Count", + "name": "autopurgeSnapRetainCount", + "value": "5" + }, + { + "desc": "The time interval in hours for which the purge task has to be triggered. Set to a positive integer (1 and above) to enable the auto purging. Defaults to 24.", + "display_name": "Auto Purge Time Interval", + "name": "autopurgeInterval", + "value": "24" + }, + { + "desc": "Enable Kerberos authentication for ZooKeeper.", + "display_name": "Enable Kerberos Authentication", + "name": "enableSecurity", + "value": "false" + }, + { + "desc": "The number of snapshot files and corresponding transaction logs to keep when running the Cleanup command.", + "display_name": "Cleanup Retain Count", + "name": "cleanupRetainCount", + "value": "5" + }, + { + "desc": "Enables the health check that a client can connect to ZooKeeper and perform basic operations", + "display_name": "ZooKeeper Canary Health Check", + "name": "zookeeper_canary_health_enabled", + "value": "true" + }, + { + "desc": "Kerberos principal used by all roles of this service.", + "display_name": "Kerberos Principal", + "name": "kerberos_princ_name", + "value": "zookeeper" + }, + { + "desc": "Configures the timeout used by the canary sessions with ZooKeeper servers", + "display_name": "ZooKeeper Canary Session Timeout", + "name": "zookeeper_canary_session_timeout", + "value": "30000" + }, + { + "desc": "The maximum size of the data that can be stored in a znode in bytes.", + "display_name": "Jute Max Buffer", + "name": "jute_maxbuffer", + "value": "4194304" + }, + { + "desc": "The group that this service's processes should run as.", + "display_name": "System Group", + "name": "process_groupname", + "value": "zookeeper" + }, + { + "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", + "display_name": "Enable Configuration Change Alerts", + "name": "enable_config_alerts", + "value": "false" + }, + { + "desc": "Amount of time, in ticks, to allow followers to connect and sync to a leader. Increase this value as needed, if the amount of data managed by ZooKeeper is large.", + "display_name": "Initialization Limit", + "name": "initLimit", + "value": "10" + }, + { + "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", + "display_name": "ZooKeeper Service Environment Advanced Configuration Snippet (Safety Valve)", + "name": "zookeeper_env_safety_valve", + "value": null + }, + { + "desc": "Configures the timeout used by the canary for connection establishment with ZooKeeper servers", + "display_name": "ZooKeeper Canary Connection Timeout", + "name": "zookeeper_canary_connection_timeout", + "value": "10000" + }, + { + "desc": "Configures the timeout used by the canary for ZooKeeper operations", + "display_name": "ZooKeeper Canary Operation Timeout", + "name": "zookeeper_canary_operation_timeout", + "value": "30000" + }, + { + "desc": "

The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.

Each trigger has all of the following fields:

  • triggerName (mandatory) - The name of the trigger. This value must be unique for the specific service.
  • triggerExpression (mandatory) - A tsquery expression representing the trigger.
  • streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire.
  • enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.
  • expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.

For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:

[{\"triggerName\": \"sample-trigger\",\n  \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n  \"streamThreshold\": 10, \"enabled\": \"true\"}]

See the trigger rules documentation for more details on how to write triggers using tsquery.

The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.

", + "display_name": "Service Triggers", + "name": "service_triggers", + "value": "[]" + }, + { + "desc": "The length of time, in milliseconds, of a single tick, which is the basic time unit used by ZooKeeper. A tick is used to regulate heartbeats and timeouts.", + "display_name": "Tick Time", + "name": "tickTime", + "value": "2000" + }, + { + "desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.", + "display_name": "Enable Log Event Capture", + "name": "catch_events", + "value": "true" + }, + { + "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", + "display_name": "Enable Service Level Health Alerts", + "name": "enable_alerts", + "value": "true" + }, + { + "desc": "The health test thresholds of the overall Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Servers falls below the critical threshold.", + "display_name": "Healthy Server Monitoring Thresholds", + "name": "zookeeper_servers_healthy_thresholds", + "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" + }, + { + "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", + "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", + "name": "smon_derived_configs_safety_valve", + "value": null + }, + { + "desc": "Whether the leader accepts client connections.", + "display_name": "Leader Serves", + "name": "leaderServes", + "value": "yes" + }, + { + "desc": "Automatically create data directories at startup, if they do not exist. Enabling this configuration should be used with care as it will suppress any errors in setup of data directories.", + "display_name": "Enable auto-creation of data directories", + "name": "zookeeper_datadir_autocreate", + "value": "false" + } +] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/validation.py b/sahara/plugins/cdh/v5_4_0/validation.py new file mode 100644 index 00000000..93db93c2 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/validation.py @@ -0,0 +1,268 @@ +# Copyright (c) 2015 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara.i18n import _ +from sahara.plugins.cdh.v5_4_0 import plugin_utils as pu +from sahara.plugins import exceptions as ex +from sahara.plugins import utils as u +from sahara.utils import general as gu + +PU = pu.PluginUtilsV540() + + +def validate_cluster_creating(cluster): + mng_count = _get_inst_count(cluster, 'CLOUDERA_MANAGER') + if mng_count != 1: + raise ex.InvalidComponentCountException('CLOUDERA_MANAGER', + 1, mng_count) + + nn_count = _get_inst_count(cluster, 'HDFS_NAMENODE') + if nn_count != 1: + raise ex.InvalidComponentCountException('HDFS_NAMENODE', 1, nn_count) + + snn_count = _get_inst_count(cluster, 'HDFS_SECONDARYNAMENODE') + if snn_count != 1: + raise ex.InvalidComponentCountException('HDFS_SECONDARYNAMENODE', 1, + snn_count) + + dn_count = _get_inst_count(cluster, 'HDFS_DATANODE') + replicas = PU.get_config_value('HDFS', 'dfs_replication', cluster) + if dn_count < replicas: + raise ex.InvalidComponentCountException( + 'HDFS_DATANODE', replicas, dn_count, + _('Number of datanodes must be not less than dfs_replication.')) + + rm_count = _get_inst_count(cluster, 'YARN_RESOURCEMANAGER') + if rm_count > 1: + raise ex.InvalidComponentCountException('YARN_RESOURCEMANAGER', + _('0 or 1'), rm_count) + + hs_count = _get_inst_count(cluster, 'YARN_JOBHISTORY') + if hs_count > 1: + raise ex.InvalidComponentCountException('YARN_JOBHISTORY', + _('0 or 1'), hs_count) + + if rm_count > 0 and hs_count < 1: + raise ex.RequiredServiceMissingException( + 'YARN_JOBHISTORY', required_by='YARN_RESOURCEMANAGER') + + nm_count = _get_inst_count(cluster, 'YARN_NODEMANAGER') + if rm_count == 0: + if nm_count > 0: + raise ex.RequiredServiceMissingException( + 'YARN_RESOURCEMANAGER', required_by='YARN_NODEMANAGER') + + oo_count = _get_inst_count(cluster, 'OOZIE_SERVER') + if oo_count > 1: + raise ex.InvalidComponentCountException('OOZIE_SERVER', _('0 or 1'), + oo_count) + + if oo_count == 1: + if dn_count < 1: + raise ex.RequiredServiceMissingException( + 'HDFS_DATANODE', required_by='OOZIE_SERVER') + + if nm_count < 1: + raise ex.RequiredServiceMissingException( + 'YARN_NODEMANAGER', required_by='OOZIE_SERVER') + + if hs_count != 1: + raise ex.RequiredServiceMissingException( + 'YARN_JOBHISTORY', required_by='OOZIE_SERVER') + + hms_count = _get_inst_count(cluster, 'HIVE_METASTORE') + hvs_count = _get_inst_count(cluster, 'HIVE_SERVER2') + whc_count = _get_inst_count(cluster, 'HIVE_WEBHCAT') + + if hms_count and rm_count < 1: + raise ex.RequiredServiceMissingException( + 'YARN_RESOURCEMANAGER', required_by='HIVE_METASTORE') + + if hms_count and not hvs_count: + raise ex.RequiredServiceMissingException( + 'HIVE_SERVER2', required_by='HIVE_METASTORE') + + if hvs_count and not hms_count: + raise ex.RequiredServiceMissingException( + 'HIVE_METASTORE', required_by='HIVE_SERVER2') + + if whc_count and not hms_count: + raise ex.RequiredServiceMissingException( + 'HIVE_METASTORE', required_by='HIVE_WEBHCAT') + + hue_count = _get_inst_count(cluster, 'HUE_SERVER') + if hue_count > 1: + raise ex.InvalidComponentCountException('HUE_SERVER', _('0 or 1'), + hue_count) + + shs_count = _get_inst_count(cluster, 'SPARK_YARN_HISTORY_SERVER') + if shs_count > 1: + raise ex.InvalidComponentCountException('SPARK_YARN_HISTORY_SERVER', + _('0 or 1'), shs_count) + if shs_count and not rm_count: + raise ex.RequiredServiceMissingException( + 'YARN_RESOURCEMANAGER', required_by='SPARK_YARN_HISTORY_SERVER') + + if oo_count < 1 and hue_count: + raise ex.RequiredServiceMissingException( + 'OOZIE_SERVER', required_by='HUE_SERVER') + + if hms_count < 1 and hue_count: + raise ex.RequiredServiceMissingException( + 'HIVE_METASTORE', required_by='HUE_SERVER') + + hbm_count = _get_inst_count(cluster, 'HBASE_MASTER') + hbr_count = _get_inst_count(cluster, 'HBASE_REGIONSERVER') + zk_count = _get_inst_count(cluster, 'ZOOKEEPER_SERVER') + + if hbm_count >= 1: + if zk_count < 1: + raise ex.RequiredServiceMissingException('ZOOKEEPER', + required_by='HBASE') + if hbr_count < 1: + raise ex.InvalidComponentCountException( + 'HBASE_REGIONSERVER', _('at least 1'), hbr_count) + elif hbr_count >= 1: + raise ex.InvalidComponentCountException('HBASE_MASTER', + _('at least 1'), hbm_count) + + a_count = _get_inst_count(cluster, 'FLUME_AGENT') + if a_count >= 1: + if dn_count < 1: + raise ex.RequiredServiceMissingException( + 'HDFS_DATANODE', required_by='FLUME_AGENT') + + snt_count = _get_inst_count(cluster, 'SENTRY_SERVER') + if snt_count > 1: + raise ex.InvalidComponentCountException('SENTRY_SERVER', _('0 or 1'), + snt_count) + if snt_count == 1: + if dn_count < 1: + raise ex.RequiredServiceMissingException( + 'HDFS_DATANODE', required_by='SENTRY_SERVER') + if zk_count < 1: + raise ex.RequiredServiceMissingException( + 'ZOOKEEPER', required_by='SENTRY_SERVER') + + slr_count = _get_inst_count(cluster, 'SOLR_SERVER') + if slr_count >= 1: + if dn_count < 1: + raise ex.RequiredServiceMissingException( + 'HDFS_DATANODE', required_by='SOLR_SERVER') + if zk_count < 1: + raise ex.RequiredServiceMissingException( + 'ZOOKEEPER', required_by='SOLR_SERVER') + + s2s_count = _get_inst_count(cluster, 'SQOOP_SERVER') + if s2s_count > 1: + raise ex.InvalidComponentCountException('SQOOP_SERVER', _('0 or 1'), + s2s_count) + if s2s_count == 1: + if dn_count < 1: + raise ex.RequiredServiceMissingException( + 'HDFS_DATANODE', required_by='SQOOP_SERVER') + if nm_count < 1: + raise ex.RequiredServiceMissingException( + 'YARN_NODEMANAGER', required_by='SQOOP_SERVER') + if hs_count != 1: + raise ex.RequiredServiceMissingException( + 'YARN_JOBHISTORY', required_by='SQOOP_SERVER') + + lhbi_count = _get_inst_count(cluster, 'HBASE_INDEXER') + if lhbi_count >= 1: + if dn_count < 1: + raise ex.RequiredServiceMissingException( + 'HDFS_DATANODE', required_by='HBASE_INDEXER') + if zk_count < 1: + raise ex.RequiredServiceMissingException( + 'ZOOKEEPER', required_by='HBASE_INDEXER') + if slr_count < 1: + raise ex.RequiredServiceMissingException( + 'SOLR_SERVER', required_by='HBASE_INDEXER') + if hbm_count < 1: + raise ex.RequiredServiceMissingException( + 'HBASE_MASTER', required_by='HBASE_INDEXER') + + ics_count = _get_inst_count(cluster, 'IMPALA_CATALOGSERVER') + iss_count = _get_inst_count(cluster, 'IMPALA_STATESTORE') + id_count = _get_inst_count(cluster, 'IMPALAD') + if ics_count > 1: + raise ex.InvalidComponentCountException('IMPALA_CATALOGSERVER', + _('0 or 1'), ics_count) + if iss_count > 1: + raise ex.InvalidComponentCountException('IMPALA_STATESTORE', + _('0 or 1'), iss_count) + if ics_count == 1: + if iss_count != 1: + raise ex.RequiredServiceMissingException( + 'IMPALA_STATESTORE', required_by='IMPALA') + if id_count < 1: + raise ex.RequiredServiceMissingException( + 'IMPALAD', required_by='IMPALA') + if dn_count < 1: + raise ex.RequiredServiceMissingException( + 'HDFS_DATANODE', required_by='IMPALA') + if hms_count < 1: + raise ex.RequiredServiceMissingException( + 'HIVE_METASTORE', required_by='IMPALA') + + +def validate_additional_ng_scaling(cluster, additional): + rm = PU.get_resourcemanager(cluster) + scalable_processes = _get_scalable_processes() + + for ng_id in additional: + ng = gu.get_by_id(cluster.node_groups, ng_id) + if not set(ng.node_processes).issubset(scalable_processes): + msg = _("CDH plugin cannot scale nodegroup with processes: " + "%(processes)s") + raise ex.NodeGroupCannotBeScaled( + ng.name, msg % {'processes': ' '.join(ng.node_processes)}) + + if not rm and 'YARN_NODEMANAGER' in ng.node_processes: + msg = _("CDH plugin cannot scale node group with processes " + "which have no master-processes run in cluster") + raise ex.NodeGroupCannotBeScaled(ng.name, msg) + + +def validate_existing_ng_scaling(cluster, existing): + scalable_processes = _get_scalable_processes() + dn_to_delete = 0 + for ng in cluster.node_groups: + if ng.id in existing: + if (ng.count > existing[ng.id] and + 'HDFS_DATANODE' in ng.node_processes): + dn_to_delete += ng.count - existing[ng.id] + + if not set(ng.node_processes).issubset(scalable_processes): + msg = _("CDH plugin cannot scale nodegroup with processes: " + "%(processes)s") + raise ex.NodeGroupCannotBeScaled( + ng.name, msg % {'processes': ' '.join(ng.node_processes)}) + + dn_count = _get_inst_count(cluster, 'HDFS_DATANODE') - dn_to_delete + replicas = PU.get_config_value('HDFS', 'dfs_replication', cluster) + if dn_count < replicas: + raise ex.ClusterCannotBeScaled( + cluster, + _('Number of datanodes must be not less than dfs_replication.')) + + +def _get_scalable_processes(): + return ['HDFS_DATANODE', 'YARN_NODEMANAGER'] + + +def _get_inst_count(cluster, process): + return sum([ng.count for ng in u.get_node_groups(cluster, process)]) diff --git a/sahara/plugins/cdh/v5_4_0/versionhandler.py b/sahara/plugins/cdh/v5_4_0/versionhandler.py new file mode 100644 index 00000000..fd993b20 --- /dev/null +++ b/sahara/plugins/cdh/v5_4_0/versionhandler.py @@ -0,0 +1,120 @@ +# Copyright (c) 2015 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara import conductor +from sahara import context +from sahara.plugins.cdh import abstractversionhandler as avm +from sahara.plugins.cdh import db_helper +from sahara.plugins.cdh.v5_4_0 import cloudera_utils as cu +from sahara.plugins.cdh.v5_4_0 import config_helper as c_helper +from sahara.plugins.cdh.v5_4_0 import deploy as dp +from sahara.plugins.cdh.v5_4_0 import edp_engine +from sahara.plugins.cdh.v5_4_0 import validation as vl + + +conductor = conductor.API +CU = cu.ClouderaUtilsV540() + + +class VersionHandler(avm.AbstractVersionHandler): + + def get_plugin_configs(self): + return c_helper.get_plugin_configs() + + def get_node_processes(self): + return { + "CLOUDERA": ['CLOUDERA_MANAGER'], + "HDFS": [], + "NAMENODE": ['HDFS_NAMENODE'], + "DATANODE": ['HDFS_DATANODE'], + "SECONDARYNAMENODE": ['HDFS_SECONDARYNAMENODE'], + "YARN": [], + "RESOURCEMANAGER": ['YARN_RESOURCEMANAGER'], + "NODEMANAGER": ['YARN_NODEMANAGER'], + "JOBHISTORY": ['YARN_JOBHISTORY'], + "OOZIE": ['OOZIE_SERVER'], + "HIVE": [], + "HIVESERVER": ['HIVE_SERVER2'], + "HIVEMETASTORE": ['HIVE_METASTORE'], + "WEBHCAT": ['HIVE_WEBHCAT'], + "HUE": ['HUE_SERVER'], + "SPARK_ON_YARN": ['SPARK_YARN_HISTORY_SERVER'], + "ZOOKEEPER": ['ZOOKEEPER_SERVER'], + "HBASE": [], + "MASTER": ['HBASE_MASTER'], + "REGIONSERVER": ['HBASE_REGIONSERVER'], + "FLUME": ['FLUME_AGENT'], + "IMPALA": [], + "CATALOGSERVER": ['IMPALA_CATALOGSERVER'], + "STATESTORE": ['IMPALA_STATESTORE'], + "IMPALAD": ['IMPALAD'], + "KS_INDEXER": ['KEY_VALUE_STORE_INDEXER'], + "SOLR": ['SOLR_SERVER'], + "SQOOP": ['SQOOP_SERVER'], + "SENTRY": ['SENTRY_SERVER'] + } + + def validate(self, cluster): + vl.validate_cluster_creating(cluster) + + def configure_cluster(self, cluster): + dp.configure_cluster(cluster) + + def start_cluster(self, cluster): + dp.start_cluster(cluster) + + self._set_cluster_info(cluster) + + def decommission_nodes(self, cluster, instances): + dp.decommission_cluster(cluster, instances) + + def validate_scaling(self, cluster, existing, additional): + vl.validate_existing_ng_scaling(cluster, existing) + vl.validate_additional_ng_scaling(cluster, additional) + + def scale_cluster(self, cluster, instances): + dp.scale_cluster(cluster, instances) + + def _set_cluster_info(self, cluster): + mng = CU.pu.get_manager(cluster) + info = { + 'Cloudera Manager': { + 'Web UI': 'http://%s:7180' % mng.management_ip, + 'Username': 'admin', + 'Password': db_helper.get_cm_password(cluster) + } + } + hue = CU.pu.get_hue(cluster) + if hue: + info['Hue Dashboard'] = { + 'Web UI': 'http://%s:8888' % hue.management_ip + } + + ctx = context.ctx() + conductor.cluster_update(ctx, cluster, {'info': info}) + + def get_edp_engine(self, cluster, job_type): + if job_type in edp_engine.EdpOozieEngine.get_supported_job_types(): + return edp_engine.EdpOozieEngine(cluster) + return None + + def get_edp_job_types(self): + return edp_engine.EdpOozieEngine.get_supported_job_types() + + def get_edp_config_hints(self, job_type): + return edp_engine.EdpOozieEngine.get_possible_job_config(job_type) + + def get_open_ports(self, node_group): + return dp.get_open_ports(node_group) diff --git a/sahara/plugins/default_templates/cdh/version_5_4_0/cluster.json b/sahara/plugins/default_templates/cdh/version_5_4_0/cluster.json new file mode 100644 index 00000000..cad31bf2 --- /dev/null +++ b/sahara/plugins/default_templates/cdh/version_5_4_0/cluster.json @@ -0,0 +1,24 @@ +{ + "plugin_name": "cdh", + "hadoop_version": "5.4.0", + "node_groups": [ + { + "name": "datanode", + "count": 3, + "node_group_template_id": "{cdh-540-default-datanode}" + }, + { + "name": "secondary-namenode", + "count": 1, + "node_group_template_id": "{cdh-540-default-secondary-namenode}" + }, + { + "name": "namenode", + "count": 1, + "node_group_template_id": "{cdh-540-default-namenode}" + } + ], + "name": "cdh-540-default-cluster", + "neutron_management_network": "{neutron_management_network}", + "cluster_configs": {} +} diff --git a/sahara/plugins/default_templates/cdh/version_5_4_0/datanode.json b/sahara/plugins/default_templates/cdh/version_5_4_0/datanode.json new file mode 100644 index 00000000..0652fe3d --- /dev/null +++ b/sahara/plugins/default_templates/cdh/version_5_4_0/datanode.json @@ -0,0 +1,11 @@ +{ + "plugin_name": "cdh", + "hadoop_version": "5.4.0", + "node_processes": [ + "HDFS_DATANODE", + "YARN_NODEMANAGER" + ], + "name": "cdh-540-default-datanode", + "floating_ip_pool": "{floating_ip_pool}", + "flavor_id": "{flavor_id}" +} diff --git a/sahara/plugins/default_templates/cdh/version_5_4_0/namenode.json b/sahara/plugins/default_templates/cdh/version_5_4_0/namenode.json new file mode 100644 index 00000000..541bcc62 --- /dev/null +++ b/sahara/plugins/default_templates/cdh/version_5_4_0/namenode.json @@ -0,0 +1,14 @@ +{ + "plugin_name": "cdh", + "hadoop_version": "5.4.0", + "node_processes": [ + "HDFS_NAMENODE", + "YARN_RESOURCEMANAGER", + "HIVE_SERVER2", + "HIVE_METASTORE", + "CLOUDERA_MANAGER" + ], + "name": "cdh-540-default-namenode", + "floating_ip_pool": "{floating_ip_pool}", + "flavor_id": "{flavor_id}" +} diff --git a/sahara/plugins/default_templates/cdh/version_5_4_0/secondary-namenode.json b/sahara/plugins/default_templates/cdh/version_5_4_0/secondary-namenode.json new file mode 100644 index 00000000..e7a7b0dc --- /dev/null +++ b/sahara/plugins/default_templates/cdh/version_5_4_0/secondary-namenode.json @@ -0,0 +1,12 @@ +{ + "plugin_name": "cdh", + "hadoop_version": "5.4.0", + "node_processes": [ + "HDFS_SECONDARYNAMENODE", + "OOZIE_SERVER", + "YARN_JOBHISTORY" + ], + "name": "cdh-540-default-secondary-namenode", + "floating_ip_pool": "{floating_ip_pool}", + "flavor_id": "{flavor_id}" +} diff --git a/sahara/tests/integration/configs/config.py b/sahara/tests/integration/configs/config.py index 9d09b3aa..5e53f6b2 100644 --- a/sahara/tests/integration/configs/config.py +++ b/sahara/tests/integration/configs/config.py @@ -296,6 +296,7 @@ VANILLA_TWO_CONFIG_OPTS = [ cfg.BoolOpt('SKIP_SCALING_TEST', default=False) ] + CDH_CONFIG_GROUP = cfg.OptGroup(name='CDH') CDH_CONFIG_OPTS = [ cfg.StrOpt('PLUGIN_NAME', @@ -390,7 +391,6 @@ CDH_CONFIG_OPTS = [ cfg.BoolOpt('SKIP_CHECK_SERVICES_TEST', default=True) ] - HDP_CONFIG_GROUP = cfg.OptGroup(name='HDP') HDP_CONFIG_OPTS = [ cfg.StrOpt('PLUGIN_NAME', diff --git a/sahara/tests/unit/plugins/cdh/test_cloudera_utils_540.py b/sahara/tests/unit/plugins/cdh/test_cloudera_utils_540.py new file mode 100644 index 00000000..e41f78bc --- /dev/null +++ b/sahara/tests/unit/plugins/cdh/test_cloudera_utils_540.py @@ -0,0 +1,66 @@ +# Copyright (c) 2014 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +from sahara.plugins.cdh.v5_4_0 import cloudera_utils as cu +from sahara.tests.unit import base +from sahara.tests.unit.plugins.cdh import utils as ctu + + +CU = cu.ClouderaUtilsV540() + + +class ClouderaUtilsTestCase(base.SaharaTestCase): + @mock.patch('sahara.plugins.cdh.v5_4_0.cloudera_utils.ClouderaUtilsV540.' + 'get_cloudera_cluster') + def test_get_service(self, mock_get_cl_cluster): + self.assertRaises(ValueError, CU.get_service_by_role, 'NAMENODE') + + cluster = ctu.get_fake_cluster() + inst = cluster.node_groups[0].instances[0] + mock_get_cl_cluster.return_value = None + + self.assertRaises(ValueError, CU.get_service_by_role, 'spam', + cluster) + self.assertRaises(ValueError, CU.get_service_by_role, 'spam', + instance=inst) + + mock_get_cl_cluster.reset_mock() + + mock_get_service = mock.MagicMock() + mock_get_service.get_service.return_value = mock.Mock() + mock_get_cl_cluster.return_value = mock_get_service + + CU.get_service_by_role('NAMENODE', cluster) + args = ((CU.HDFS_SERVICE_NAME,),) + self.assertEqual(args, mock_get_service.get_service.call_args) + + mock_get_service.reset_mock() + CU.get_service_by_role('JOBHISTORY', instance=inst) + args = ((CU.YARN_SERVICE_NAME,),) + self.assertEqual(args, mock_get_service.get_service.call_args) + + mock_get_service.reset_mock() + CU.get_service_by_role('OOZIE_SERVER', cluster) + args = ((CU.OOZIE_SERVICE_NAME,),) + self.assertEqual(args, mock_get_service.get_service.call_args) + + def test_get_role_name(self): + inst_mock = mock.Mock() + inst_mock.hostname.return_value = 'spam-host' + + self.assertEqual('eggs_spam_host', + CU.pu.get_role_name(inst_mock, 'eggs')) diff --git a/sahara/tests/unit/plugins/cdh/test_config_helper_540.py b/sahara/tests/unit/plugins/cdh/test_config_helper_540.py new file mode 100644 index 00000000..d286a0e3 --- /dev/null +++ b/sahara/tests/unit/plugins/cdh/test_config_helper_540.py @@ -0,0 +1,37 @@ +# Copyright (c) 2014 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara.plugins.cdh.v5_4_0 import config_helper as c_h +from sahara.tests.unit import base +from sahara.tests.unit.plugins.cdh import utils as ctu + + +class ConfigHelperTestCase(base.SaharaTestCase): + def test_is_swift_enabled(self): + cluster = ctu.get_fake_cluster(cluster_configs={}) + self.assertTrue(c_h.is_swift_enabled(cluster)) + + cluster = ctu.get_fake_cluster( + cluster_configs={'general': {c_h.ENABLE_SWIFT.name: False}}) + self.assertFalse(c_h.is_swift_enabled(cluster)) + + def test_get_swift_lib_url(self): + cluster = ctu.get_fake_cluster(cluster_configs={}) + self.assertEqual(c_h.DEFAULT_SWIFT_LIB_URL, + c_h.get_swift_lib_url(cluster)) + + cluster = ctu.get_fake_cluster( + cluster_configs={'general': {c_h.SWIFT_LIB_URL.name: 'spam'}}) + self.assertEqual('spam', c_h.get_swift_lib_url(cluster)) diff --git a/sahara/tests/unit/plugins/cdh/test_utils_540.py b/sahara/tests/unit/plugins/cdh/test_utils_540.py new file mode 100644 index 00000000..633bfc02 --- /dev/null +++ b/sahara/tests/unit/plugins/cdh/test_utils_540.py @@ -0,0 +1,65 @@ +# Copyright (c) 2014 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sahara.plugins.cdh.v5_4_0 import plugin_utils as pu +from sahara.tests.unit import base +from sahara.tests.unit.plugins.cdh import utils as tu + + +PU = pu.PluginUtilsV540() + + +class UtilsTestCase(base.SaharaTestCase): + def test_get_manager(self): + cluster = tu.get_fake_cluster() + inst = PU.get_manager(cluster) + self.assertEqual('id1', inst.instance_id) + + def test_get_namenode(self): + cluster = tu.get_fake_cluster() + inst = PU.get_namenode(cluster) + self.assertEqual('id2', inst.instance_id) + + def test_get_secondarynamenode(self): + cluster = tu.get_fake_cluster() + inst = PU.get_secondarynamenode(cluster) + self.assertEqual('id2', inst.instance_id) + + def test_get_resourcemanager(self): + cluster = tu.get_fake_cluster() + inst = PU.get_resourcemanager(cluster) + self.assertEqual('id2', inst.instance_id) + + def test_get_datanodes(self): + cluster = tu.get_fake_cluster() + dns = PU.get_datanodes(cluster) + ids = [dn.instance_id for dn in dns] + self.assertEqual(sorted(['id00', 'id01', 'id02']), sorted(ids)) + + def test_get_nodemanagers(self): + cluster = tu.get_fake_cluster() + nms = PU.get_nodemanagers(cluster) + ids = [nm.instance_id for nm in nms] + self.assertEqual(sorted(['id00', 'id01', 'id02']), sorted(ids)) + + def test_get_historyserver(self): + cluster = tu.get_fake_cluster() + inst = PU.get_historyserver(cluster) + self.assertEqual('id2', inst.instance_id) + + def test_get_oozie(self): + cluster = tu.get_fake_cluster() + inst = PU.get_oozie(cluster) + self.assertEqual('id2', inst.instance_id) diff --git a/sahara/tests/unit/plugins/cdh/v5_4_0/__init__.py b/sahara/tests/unit/plugins/cdh/v5_4_0/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sahara/tests/unit/plugins/cdh/v5_4_0/test_edp_engine.py b/sahara/tests/unit/plugins/cdh/v5_4_0/test_edp_engine.py new file mode 100644 index 00000000..4e0a0820 --- /dev/null +++ b/sahara/tests/unit/plugins/cdh/v5_4_0/test_edp_engine.py @@ -0,0 +1,94 @@ +# Copyright (c) 2015 Intel Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +from sahara.plugins.cdh.v5_4_0 import edp_engine +from sahara.tests.unit import base as sahara_base +from sahara.utils import edp + + +class CDH54ConfigHintsTest(sahara_base.SaharaTestCase): + @mock.patch( + 'sahara.plugins.cdh.confighints_helper.get_possible_hive_config_from', + return_value={}) + def test_get_possible_job_config_hive(self, + get_possible_hive_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_HIVE) + get_possible_hive_config_from.assert_called_once_with( + 'plugins/cdh/v5_4_0/resources/hive-site.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch('sahara.plugins.cdh.v5_4_0.edp_engine.EdpOozieEngine') + def test_get_possible_job_config_java(self, BaseCDHEdpOozieEngine): + expected_config = {'job_config': {}} + BaseCDHEdpOozieEngine.get_possible_job_config.return_value = ( + expected_config) + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_JAVA) + BaseCDHEdpOozieEngine.get_possible_job_config.assert_called_once_with( + edp.JOB_TYPE_JAVA) + self.assertEqual(expected_config, actual_config) + + @mock.patch( + 'sahara.plugins.cdh.confighints_helper.' + 'get_possible_mapreduce_config_from', + return_value={}) + def test_get_possible_job_config_mapreduce( + self, get_possible_mapreduce_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_MAPREDUCE) + get_possible_mapreduce_config_from.assert_called_once_with( + 'plugins/cdh/v5_4_0/resources/mapred-site.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch( + 'sahara.plugins.cdh.confighints_helper.' + 'get_possible_mapreduce_config_from', + return_value={}) + def test_get_possible_job_config_mapreduce_streaming( + self, get_possible_mapreduce_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_MAPREDUCE_STREAMING) + get_possible_mapreduce_config_from.assert_called_once_with( + 'plugins/cdh/v5_4_0/resources/mapred-site.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch( + 'sahara.plugins.cdh.confighints_helper.get_possible_pig_config_from', + return_value={}) + def test_get_possible_job_config_pig(self, + get_possible_pig_config_from): + expected_config = {'job_config': {}} + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_PIG) + get_possible_pig_config_from.assert_called_once_with( + 'plugins/cdh/v5_4_0/resources/mapred-site.xml') + self.assertEqual(expected_config, actual_config) + + @mock.patch('sahara.plugins.cdh.v5_4_0.edp_engine.EdpOozieEngine') + def test_get_possible_job_config_shell(self, BaseCDHEdpOozieEngine): + expected_config = {'job_config': {}} + BaseCDHEdpOozieEngine.get_possible_job_config.return_value = ( + expected_config) + actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( + edp.JOB_TYPE_SHELL) + BaseCDHEdpOozieEngine.get_possible_job_config.assert_called_once_with( + edp.JOB_TYPE_SHELL) + self.assertEqual(expected_config, actual_config)