openstack/deb-sahara
Code Issues Proposed changes

Separate the codes of CDH5 and CDH5.3.0

Browse Source 
We use v5 and v5.3.0 to put different python files for CDH5 and
CDH5.3.0. CDH5 is CDH5.0.0, we use the name "CDH5" instead of
"CDH5.0.0" for backward support. Currently since CDH5.0.0 does
not support cm_api>6, we cannot use first_run API in CDH5.0.0, so
we only implemented parts of the services that we implemented in
CDH5.3.0.

implements bp: cdh-version-management
Change-Id: I3b3058f25912ddf6206d64db88ac40138a45a53f
This commit is contained in:
Ken Chen 2015-01-16 23:41:22 +08:00
parent c293092a88
commit 2a56aa003d
121 changed files with 12707 additions and 1224 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
MANIFEST.in
View File

@ -9,10 +9,14 @@ include sahara/db/migration/alembic_migrations/versions/README
recursive-include sahara/locale *
include sahara/plugins/cdh/resources/cdh_config.py
include sahara/plugins/cdh/resources/*.sh
include sahara/plugins/cdh/resources/*.json
include sahara/plugins/cdh/resources/*.sql
include sahara/plugins/cdh/v5/resources/cdh_config.py
include sahara/plugins/cdh/v5/resources/*.sh
include sahara/plugins/cdh/v5/resources/*.json
include sahara/plugins/cdh/v5/resources/*.sql
include sahara/plugins/cdh/v5_3_0/resources/cdh_config.py
include sahara/plugins/cdh/v5_3_0/resources/*.sh
include sahara/plugins/cdh/v5_3_0/resources/*.json
include sahara/plugins/cdh/v5_3_0/resources/*.sql
include sahara/plugins/vanilla/hadoop2/resources/*.sh
include sahara/plugins/vanilla/hadoop2/resources/*.sql
include sahara/plugins/vanilla/hadoop2/resources/*.template

65
sahara/plugins/cdh/abstractversionhandler.py Normal file
View File

@ -0,0 +1,65 @@
# Copyright (c) 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
import six
@six.add_metaclass(abc.ABCMeta)
class AbstractVersionHandler(object):
@abc.abstractmethod
def get_node_processes(self):
return
@abc.abstractmethod
def get_plugin_configs(self):
return
@abc.abstractmethod
def configure_cluster(self, cluster):
return
@abc.abstractmethod
def start_cluster(self, cluster):
return
@abc.abstractmethod
def validate(self, cluster):
return
@abc.abstractmethod
def scale_cluster(self, cluster, instances):
return
@abc.abstractmethod
def decommission_nodes(self, cluster, instances):
return
@abc.abstractmethod
def validate_scaling(self, cluster, existing, additional):
return
@abc.abstractmethod
def get_edp_engine(self, cluster, job_type):
return
@abc.abstractmethod
def get_open_ports(self, node_group):
return
def on_terminate_cluster(self, cluster):
pass

340
sahara/plugins/cdh/cloudera_utils.py
View File

@ -23,32 +23,16 @@ except ImportError:
api_client = None
services = None
from oslo_log import log as logging
from oslo_utils import timeutils
from sahara import context
from sahara.i18n import _
from sahara.plugins.cdh import utils as pu
from sahara.i18n import _LE
from sahara.plugins import exceptions as ex
CM_DEFAULT_USERNAME = 'admin'
CM_DEFAULT_PASSWD = 'admin'
CM_API_VERSION = 7
HDFS_SERVICE_NAME = 'hdfs01'
YARN_SERVICE_NAME = 'yarn01'
OOZIE_SERVICE_NAME = 'oozie01'
HIVE_SERVICE_NAME = 'hive01'
HUE_SERVICE_NAME = 'hue01'
SPARK_SERVICE_NAME = 'spark_on_yarn01'
ZOOKEEPER_SERVICE_NAME = 'zookeeper01'
HBASE_SERVICE_NAME = 'hbase01'
FLUME_SERVICE_NAME = 'flume01'
SENTRY_SERVICE_NAME = 'sentry01'
SOLR_SERVICE_NAME = 'solr01'
SQOOP_SERVICE_NAME = 'sqoop01'
KS_INDEXER_SERVICE_NAME = 'ks_indexer01'
IMPALA_SERVICE_NAME = 'impala01'
def have_cm_api_libs():
return api_client and services
LOG = logging.getLogger(__name__)
def cloudera_cmd(f):
@ -63,183 +47,195 @@ def cloudera_cmd(f):
raise ex.HadoopProvisionError(c.resultMessage)
else:
raise ex.HadoopProvisionError(result.resultMessage)
return wrapper
def get_api_client(cluster):
manager_ip = pu.get_manager(cluster).management_ip
return api_client.ApiResource(manager_ip, username=CM_DEFAULT_USERNAME,
password=CM_DEFAULT_PASSWD,
version=CM_API_VERSION)
class ClouderaUtils(object):
CM_DEFAULT_USERNAME = 'admin'
CM_DEFAULT_PASSWD = 'admin'
CM_API_VERSION = 6
HDFS_SERVICE_NAME = 'hdfs01'
YARN_SERVICE_NAME = 'yarn01'
OOZIE_SERVICE_NAME = 'oozie01'
HIVE_SERVICE_NAME = 'hive01'
HUE_SERVICE_NAME = 'hue01'
SPARK_SERVICE_NAME = 'spark_on_yarn01'
ZOOKEEPER_SERVICE_NAME = 'zookeeper01'
HBASE_SERVICE_NAME = 'hbase01'
def get_cloudera_cluster(cluster):
api = get_api_client(cluster)
return api.get_cluster(cluster.name)
def __init__(self):
# pu will be defined in derived class.
self.pu = None
def have_cm_api_libs(self):
return api_client and services
@cloudera_cmd
def start_instances(cluster):
cm_cluster = get_cloudera_cluster(cluster)
yield cm_cluster.start()
def validate_cm_api_libs(self):
if not self.have_cm_api_libs():
LOG.error(_LE("For provisioning cluster with CDH plugin install"
" 'cm_api' package version 6.0.2 or later."))
raise ex.HadoopProvisionError(_("'cm_api' is not installed."))
def get_api_client(self, cluster):
manager_ip = self.pu.get_manager(cluster).management_ip
return api_client.ApiResource(manager_ip,
username=self.CM_DEFAULT_USERNAME,
password=self.CM_DEFAULT_PASSWD,
version=self.CM_API_VERSION)
def delete_instances(cluster, instances):
api = get_api_client(cluster)
cm_cluster = get_cloudera_cluster(cluster)
hosts = api.get_all_hosts(view='full')
hostsnames_to_deleted = [i.fqdn() for i in instances]
for host in hosts:
if host.hostname in hostsnames_to_deleted:
cm_cluster.remove_host(host.hostId)
api.delete_host(host.hostId)
def get_cloudera_cluster(self, cluster):
api = self.get_api_client(cluster)
return api.get_cluster(cluster.name)
@cloudera_cmd
def start_instances(self, cluster):
cm_cluster = self.get_cloudera_cluster(cluster)
yield cm_cluster.start()
def get_service(process, cluster=None, instance=None):
cm_cluster = None
if cluster:
cm_cluster = get_cloudera_cluster(cluster)
elif instance:
cm_cluster = get_cloudera_cluster(instance.node_group.cluster)
else:
raise ValueError(_("'cluster' or 'instance' argument missed"))
def delete_instances(self, cluster, instances):
api = self.get_api_client(cluster)
cm_cluster = self.get_cloudera_cluster(cluster)
hosts = api.get_all_hosts(view='full')
hostsnames_to_deleted = [i.fqdn() for i in instances]
for host in hosts:
if host.hostname in hostsnames_to_deleted:
cm_cluster.remove_host(host.hostId)
api.delete_host(host.hostId)
if process in ['NAMENODE', 'DATANODE', 'SECONDARYNAMENODE']:
return cm_cluster.get_service(HDFS_SERVICE_NAME)
elif process in ['RESOURCEMANAGER', 'NODEMANAGER', 'JOBHISTORY']:
return cm_cluster.get_service(YARN_SERVICE_NAME)
elif process in ['OOZIE_SERVER']:
return cm_cluster.get_service(OOZIE_SERVICE_NAME)
elif process in ['HIVESERVER2', 'HIVEMETASTORE', 'WEBHCAT']:
return cm_cluster.get_service(HIVE_SERVICE_NAME)
elif process in ['HUE_SERVER']:
return cm_cluster.get_service(HUE_SERVICE_NAME)
elif process in ['SPARK_YARN_HISTORY_SERVER']:
return cm_cluster.get_service(SPARK_SERVICE_NAME)
elif process in ['SERVER']:
return cm_cluster.get_service(ZOOKEEPER_SERVICE_NAME)
elif process in ['MASTER', 'REGIONSERVER']:
return cm_cluster.get_service(HBASE_SERVICE_NAME)
elif process in ['AGENT']:
return cm_cluster.get_service(FLUME_SERVICE_NAME)
elif process in ['SQOOP_SERVER']:
return cm_cluster.get_service(SQOOP_SERVICE_NAME)
elif process in ['SENTRY_SERVER']:
return cm_cluster.get_service(SENTRY_SERVICE_NAME)
elif process in ['SOLR_SERVER']:
return cm_cluster.get_service(SOLR_SERVICE_NAME)
elif process in ['HBASE_INDEXER']:
return cm_cluster.get_service(KS_INDEXER_SERVICE_NAME)
elif process in ['CATALOGSERVER', 'STATESTORE', 'IMPALAD', 'LLAMA']:
return cm_cluster.get_service(IMPALA_SERVICE_NAME)
else:
raise ValueError(
_("Process %(process)s is not supported by CDH plugin") %
{'process': process})
def decommission_nodes(self, cluster, process, role_names):
service = self.get_service_by_role(process, cluster)
service.decommission(*role_names).wait()
for role_name in role_names:
service.delete_role(role_name)
@cloudera_cmd
def refresh_nodes(self, cluster, process, service_name):
cm_cluster = self.get_cloudera_cluster(cluster)
service = cm_cluster.get_service(service_name)
nds = [n.name for n in service.get_roles_by_type(process)]
for nd in nds:
for st in service.refresh(nd):
yield st
def decommission_nodes(cluster, process, role_names):
service = get_service(process, cluster)
service.decommission(*role_names).wait()
for role_name in role_names:
service.delete_role(role_name)
@cloudera_cmd
def deploy_configs(self, cluster):
cm_cluster = self.get_cloudera_cluster(cluster)
yield cm_cluster.deploy_client_config()
@cloudera_cmd
def update_configs(self, instance):
for process in instance.node_group.node_processes:
process = self.pu.convert_role_showname(process)
service = self.get_service_by_role(process, instance=instance)
yield service.deploy_client_config(self.pu.get_role_name(instance,
process))
@cloudera_cmd
def refresh_nodes(cluster, process, service_name):
cm_cluster = get_cloudera_cluster(cluster)
service = cm_cluster.get_service(service_name)
@cloudera_cmd
def restart_mgmt_service(self, cluster):
api = self.get_api_client(cluster)
cm = api.get_cloudera_manager()
mgmt_service = cm.get_service()
yield mgmt_service.restart()
nds = [n.name for n in service.get_roles_by_type(process)]
for nd in nds:
for st in service.refresh(nd):
yield st
@cloudera_cmd
def start_service(self, service):
yield service.start()
@cloudera_cmd
def start_roles(self, service, *role_names):
for role in service.start_roles(*role_names):
yield role
@cloudera_cmd
def deploy_configs(cluster):
cm_cluster = get_cloudera_cluster(cluster)
yield cm_cluster.deploy_client_config()
def create_mgmt_service(self, cluster):
api = self.get_api_client(cluster)
cm = api.get_cloudera_manager()
setup_info = services.ApiServiceSetupInfo()
manager = self.pu.get_manager(cluster)
hostname = manager.fqdn()
processes = ['SERVICEMONITOR', 'HOSTMONITOR',
'EVENTSERVER', 'ALERTPUBLISHER']
for proc in processes:
setup_info.add_role_info(self.pu.get_role_name(manager, proc),
proc, hostname)
@cloudera_cmd
def update_configs(instance):
for process in instance.node_group.node_processes:
service = get_service(process, instance=instance)
yield service.deploy_client_config(get_role_name(instance, process))
cm.create_mgmt_service(setup_info)
cm.hosts_start_roles([hostname])
def get_service_by_role(self, process, cluster=None, instance=None):
cm_cluster = None
if cluster:
cm_cluster = self.get_cloudera_cluster(cluster)
elif instance:
cm_cluster = self.get_cloudera_cluster(instance.node_group.cluster)
else:
raise ValueError(_("'cluster' or 'instance' argument missed"))
@cloudera_cmd
def first_run(cluster):
cm_cluster = get_cloudera_cluster(cluster)
yield cm_cluster.first_run()
if process in ['NAMENODE', 'DATANODE', 'SECONDARYNAMENODE']:
return cm_cluster.get_service(self.HDFS_SERVICE_NAME)
elif process in ['RESOURCEMANAGER', 'NODEMANAGER', 'JOBHISTORY']:
return cm_cluster.get_service(self.YARN_SERVICE_NAME)
elif process in ['OOZIE_SERVER']:
return cm_cluster.get_service(self.OOZIE_SERVICE_NAME)
elif process in ['HIVESERVER2', 'HIVEMETASTORE', 'WEBHCAT']:
return cm_cluster.get_service(self.HIVE_SERVICE_NAME)
elif process in ['HUE_SERVER']:
return cm_cluster.get_service(self.HUE_SERVICE_NAME)
elif process in ['SPARK_YARN_HISTORY_SERVER']:
return cm_cluster.get_service(self.SPARK_SERVICE_NAME)
elif process in ['SERVER']:
return cm_cluster.get_service(self.ZOOKEEPER_SERVICE_NAME)
elif process in ['MASTER', 'REGIONSERVER']:
return cm_cluster.get_service(self.HBASE_SERVICE_NAME)
else:
raise ValueError(
_("Process %(process)s is not supported by CDH plugin") %
{'process': process})
def await_agents(self, instances):
api = self.get_api_client(instances[0].node_group.cluster)
timeout = 300
LOG.debug("Waiting %(timeout)s seconds for agent connected to manager"
% {'timeout': timeout})
s_time = timeutils.utcnow()
while timeutils.delta_seconds(s_time, timeutils.utcnow()) < timeout:
hostnames = [i.fqdn() for i in instances]
hostnames_to_manager = [h.hostname for h in
api.get_all_hosts('full')]
is_ok = True
for hostname in hostnames:
if hostname not in hostnames_to_manager:
is_ok = False
break
def get_role_name(instance, service):
# NOTE: role name must match regexp "[_A-Za-z][-_A-Za-z0-9]{0,63}"
shortcuts = {
'AGENT': 'A',
'ALERTPUBLISHER': 'AP',
'CATALOGSERVER': 'ICS',
'DATANODE': 'DN',
'EVENTSERVER': 'ES',
'HBASE_INDEXER': 'LHBI',
'HIVEMETASTORE': 'HVM',
'HIVESERVER2': 'HVS',
'HOSTMONITOR': 'HM',
'IMPALAD': 'ID',
'JOBHISTORY': 'JS',
'MASTER': 'M',
'NAMENODE': 'NN',
'NODEMANAGER': 'NM',
'OOZIE_SERVER': 'OS',
'REGIONSERVER': 'RS',
'RESOURCEMANAGER': 'RM',
'SECONDARYNAMENODE': 'SNN',
'SENTRY_SERVER': 'SNT',
'SERVER': 'S',
'SERVICEMONITOR': 'SM',
'SOLR_SERVER': 'SLR',
'SPARK_YARN_HISTORY_SERVER': 'SHS',
'SQOOP_SERVER': 'S2S',
'STATESTORE': 'ISS',
'WEBHCAT': 'WHC'
}
return '%s_%s' % (shortcuts.get(service, service),
instance.hostname().replace('-', '_'))
if not is_ok:
context.sleep(5)
else:
break
else:
raise ex.HadoopProvisionError(_("Cloudera agents failed to connect"
" to Cloudera Manager"))
def configure_instances(self, instances, cluster=None):
for inst in instances:
self.configure_instance(inst, cluster)
def create_mgmt_service(cluster):
api = get_api_client(cluster)
cm = api.get_cloudera_manager()
def configure_instance(self, instance, cluster=None):
for process in instance.node_group.node_processes:
self._add_role(instance, process, cluster)
setup_info = services.ApiServiceSetupInfo()
manager = pu.get_manager(cluster)
hostname = manager.fqdn()
processes = ['SERVICEMONITOR', 'HOSTMONITOR',
'EVENTSERVER', 'ALERTPUBLISHER']
for proc in processes:
setup_info.add_role_info(get_role_name(manager, proc), proc, hostname)
def _add_role(self, instance, process, cluster):
if process in ['CLOUDERA_MANAGER']:
return
cm.create_mgmt_service(setup_info)
cm.hosts_start_roles([hostname])
process = self.pu.convert_role_showname(process)
service = self.get_service_by_role(process, instance=instance)
role = service.create_role(self.pu.get_role_name(instance, process),
process, instance.fqdn())
role.update_config(self._get_configs(process, cluster,
node_group=instance.node_group))
@cloudera_cmd
def restart_mgmt_service(cluster):
api = get_api_client(cluster)
cm = api.get_cloudera_manager()
mgmt_service = cm.get_service()
yield mgmt_service.restart()
@cloudera_cmd
def start_service(service):
yield service.start()
@cloudera_cmd
def start_roles(service, *role_names):
for role in service.start_roles(*role_names):
yield role
def _get_configs(self, service, cluster=None, node_group=None):
# Defined in derived class.
return

12
sahara/plugins/cdh/commands.py
View File

@ -87,6 +87,8 @@ def install_packages(remote, packages, timeout=1800):
def update_repository(remote):
if is_ubuntu_os(remote):
_root(remote, 'apt-get update')
if is_centos_os(remote):
_root(remote, 'yum clean all')
def push_remote_file(remote, src, dst):
@ -99,6 +101,11 @@ def add_ubuntu_repository(r, repo_list_url, repo_name):
'/etc/apt/sources.list.d/%s.list' % repo_name)
def write_ubuntu_repository(r, repo_content, repo_name):
r.write_file_to('/etc/apt/sources.list.d/%s.list' % repo_name,
repo_content, run_as_root=True)
def add_apt_key(remote, key_url):
cmd = 'wget -qO - %s | apt-key add -' % key_url
_root(remote, cmd)
@ -108,5 +115,10 @@ def add_centos_repository(r, repo_list_url, repo_name):
push_remote_file(r, repo_list_url, '/etc/yum.repos.d/%s.repo' % repo_name)
def write_centos_repository(r, repo_content, repo_name):
r.write_file_to('/etc/yum.repos.d/%s.repo' % repo_name,
repo_content, root_as_root=True)
def start_mysql_server(remote):
_root(remote, 'service mysql start')

674
sahara/plugins/cdh/deploy.py
View File

@ -1,674 +0,0 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import telnetlib
from oslo_log import log as logging
from oslo_utils import timeutils
import six
from sahara import context
from sahara.i18n import _
from sahara.i18n import _LI
from sahara.plugins.cdh import cloudera_utils as cu
from sahara.plugins.cdh import commands as cmd
from sahara.plugins.cdh import config_helper as c_helper
from sahara.plugins.cdh import db_helper
from sahara.plugins.cdh import utils as pu
from sahara.plugins.cdh import validation as v
from sahara.plugins import exceptions as ex
from sahara.plugins import utils as gu
from sahara.swift import swift_helper
from sahara.utils import edp as edp_u
from sahara.utils import xmlutils
CM_API_PORT = 7180
HDFS_SERVICE_TYPE = 'HDFS'
YARN_SERVICE_TYPE = 'YARN'
OOZIE_SERVICE_TYPE = 'OOZIE'
HIVE_SERVICE_TYPE = 'HIVE'
HUE_SERVICE_TYPE = 'HUE'
SPARK_SERVICE_TYPE = 'SPARK_ON_YARN'
ZOOKEEPER_SERVICE_TYPE = 'ZOOKEEPER'
HBASE_SERVICE_TYPE = 'HBASE'
FLUME_SERVICE_TYPE = 'FLUME'
SENTRY_SERVICE_TYPE = 'SENTRY'
SOLR_SERVICE_TYPE = 'SOLR'
SQOOP_SERVICE_TYPE = 'SQOOP'
KS_INDEXER_SERVICE_TYPE = 'KS_INDEXER'
IMPALA_SERVICE_TYPE = 'IMPALA'
PATH_TO_CORE_SITE_XML = '/etc/hadoop/conf/core-site.xml'
HADOOP_LIB_DIR = '/usr/lib/hadoop-mapreduce'
PACKAGES = [
'cloudera-manager-agent',
'cloudera-manager-daemons',
'cloudera-manager-server',
'cloudera-manager-server-db-2',
'flume-ng',
'hadoop-hdfs-datanode',
'hadoop-hdfs-namenode',
'hadoop-hdfs-secondarynamenode',
'hadoop-mapreduce',
'hadoop-mapreduce-historyserver',
'hadoop-yarn-nodemanager',
'hadoop-yarn-resourcemanager',
'hbase',
'hbase-solr',
'hive-hcatalog',
'hive-metastore',
'hive-server2',
'hive-webhcat-server',
'hue',
'impala',
'impala-server',
'impala-state-store',
'impala-catalog',
'ntp',
'oozie',
'oracle-j2sdk1.7',
'sentry',
'solr-server',
'spark-history-server',
'sqoop2',
'unzip',
'zookeeper'
]
LOG = logging.getLogger(__name__)
def _merge_dicts(a, b):
res = {}
def update(cfg):
for service, configs in six.iteritems(cfg):
if not res.get(service):
res[service] = {}
res[service].update(configs)
update(a)
update(b)
return res
def _get_configs(service, cluster=None, node_group=None):
def get_hadoop_dirs(mount_points, suffix):
return ','.join([x + suffix for x in mount_points])
all_confs = {}
if cluster:
zk_count = v._get_inst_count(cluster, 'ZOOKEEPER_SERVER')
hbm_count = v._get_inst_count(cluster, 'HBASE_MASTER')
snt_count = v._get_inst_count(cluster, 'SENTRY_SERVER')
ks_count = v._get_inst_count(cluster, 'KEY_VALUE_STORE_INDEXER')
imp_count = v._get_inst_count(cluster, 'IMPALA_CATALOGSERVER')
core_site_safety_valve = ''
if c_helper.is_swift_enabled(cluster):
configs = swift_helper.get_swift_configs()
confs = dict((c['name'], c['value']) for c in configs)
core_site_safety_valve = xmlutils.create_elements_xml(confs)
all_confs = {
'HDFS': {
'zookeeper_service':
cu.ZOOKEEPER_SERVICE_NAME if zk_count else '',
'dfs_block_local_path_access_user':
'impala' if imp_count else '',
'core_site_safety_valve': core_site_safety_valve
},
'HIVE': {
'mapreduce_yarn_service': cu.YARN_SERVICE_NAME,
'zookeeper_service':
cu.ZOOKEEPER_SERVICE_NAME if zk_count else ''
},
'OOZIE': {
'mapreduce_yarn_service': cu.YARN_SERVICE_NAME,
'zookeeper_service':
cu.ZOOKEEPER_SERVICE_NAME if zk_count else ''
},
'YARN': {
'hdfs_service': cu.HDFS_SERVICE_NAME,
'zookeeper_service':
cu.ZOOKEEPER_SERVICE_NAME if zk_count else ''
},
'HUE': {
'hive_service': cu.HIVE_SERVICE_NAME,
'oozie_service': cu.OOZIE_SERVICE_NAME,
'sentry_service': cu.SENTRY_SERVICE_NAME if snt_count else '',
'zookeeper_service':
cu.ZOOKEEPER_SERVICE_NAME if zk_count else ''
},
'SPARK_ON_YARN': {
'yarn_service': cu.YARN_SERVICE_NAME
},
'HBASE': {
'hdfs_service': cu.HDFS_SERVICE_NAME,
'zookeeper_service': cu.ZOOKEEPER_SERVICE_NAME,
'hbase_enable_indexing': 'true' if ks_count else 'false',
'hbase_enable_replication': 'true' if ks_count else 'false'
},
'FLUME': {
'hdfs_service': cu.HDFS_SERVICE_NAME,
'hbase_service': cu.HBASE_SERVICE_NAME if hbm_count else ''
},
'SENTRY': {
'hdfs_service': cu.HDFS_SERVICE_NAME
},
'SOLR': {
'hdfs_service': cu.HDFS_SERVICE_NAME,
'zookeeper_service': cu.ZOOKEEPER_SERVICE_NAME
},
'SQOOP': {
'mapreduce_yarn_service': cu.YARN_SERVICE_NAME
},
'KS_INDEXER': {
'hbase_service': cu.HBASE_SERVICE_NAME,
'solr_service': cu.SOLR_SERVICE_NAME
},
'IMPALA': {
'hdfs_service': cu.HDFS_SERVICE_NAME,
'hbase_service': cu.HBASE_SERVICE_NAME if hbm_count else '',
'hive_service': cu.HIVE_SERVICE_NAME
}
}
hive_confs = {
'HIVE': {
'hive_metastore_database_type': 'postgresql',
'hive_metastore_database_host':
pu.get_manager(cluster).internal_ip,
'hive_metastore_database_port': '7432',
'hive_metastore_database_password':
db_helper.get_hive_db_password(cluster)
}
}
hue_confs = {
'HUE': {
'hue_webhdfs': cu.get_role_name(pu.get_namenode(cluster),
'NAMENODE')
}
}
sentry_confs = {
'SENTRY': {
'sentry_server_database_type': 'postgresql',
'sentry_server_database_host':
pu.get_manager(cluster).internal_ip,
'sentry_server_database_port': '7432',
'sentry_server_database_password':
db_helper.get_sentry_db_password(cluster)
}
}
all_confs = _merge_dicts(all_confs, hue_confs)
all_confs = _merge_dicts(all_confs, hive_confs)
all_confs = _merge_dicts(all_confs, sentry_confs)
all_confs = _merge_dicts(all_confs, cluster.cluster_configs)
if node_group:
paths = node_group.storage_paths()
ng_default_confs = {
'NAMENODE': {
'dfs_name_dir_list': get_hadoop_dirs(paths, '/fs/nn')
},
'SECONDARYNAMENODE': {
'fs_checkpoint_dir_list': get_hadoop_dirs(paths, '/fs/snn')
},
'DATANODE': {
'dfs_data_dir_list': get_hadoop_dirs(paths, '/fs/dn'),
'dfs_datanode_data_dir_perm': 755,
'dfs_datanode_handler_count': 30
},
'NODEMANAGER': {
'yarn_nodemanager_local_dirs': get_hadoop_dirs(paths,
'/yarn/local')
},
'SERVER': {
'maxSessionTimeout': 60000
}
}
ng_user_confs = pu.convert_process_configs(node_group.node_configs)
all_confs = _merge_dicts(all_confs, ng_user_confs)
all_confs = _merge_dicts(all_confs, ng_default_confs)
return all_confs.get(service, {})
def configure_cluster(cluster):
instances = gu.get_instances(cluster)
if not cmd.is_pre_installed_cdh(pu.get_manager(cluster).remote()):
_configure_os(instances)
_install_packages(instances, PACKAGES)
_start_cloudera_agents(instances)
_start_cloudera_manager(cluster)
_await_agents(instances)
_configure_manager(cluster)
_create_services(cluster)
_configure_services(cluster)
_configure_instances(instances)
cu.deploy_configs(cluster)
def scale_cluster(cluster, instances):
if not instances:
return
if not cmd.is_pre_installed_cdh(instances[0].remote()):
_configure_os(instances)
_install_packages(instances, PACKAGES)
_start_cloudera_agents(instances)
_await_agents(instances)
for instance in instances:
_configure_instance(instance)
cu.update_configs(instance)
if 'HDFS_DATANODE' in instance.node_group.node_processes:
cu.refresh_nodes(cluster, 'DATANODE', cu.HDFS_SERVICE_NAME)
_configure_swift_to_inst(instance)
if 'HDFS_DATANODE' in instance.node_group.node_processes:
hdfs = cu.get_service('DATANODE', instance=instance)
cu.start_roles(hdfs, cu.get_role_name(instance, 'DATANODE'))
if 'YARN_NODEMANAGER' in instance.node_group.node_processes:
yarn = cu.get_service('NODEMANAGER', instance=instance)
cu.start_roles(yarn, cu.get_role_name(instance, 'NODEMANAGER'))
def decommission_cluster(cluster, instances):
dns = []
nms = []
for i in instances:
if 'HDFS_DATANODE' in i.node_group.node_processes:
dns.append(cu.get_role_name(i, 'DATANODE'))
if 'YARN_NODEMANAGER' in i.node_group.node_processes:
nms.append(cu.get_role_name(i, 'NODEMANAGER'))
if dns:
cu.decommission_nodes(cluster, 'DATANODE', dns)
if nms:
cu.decommission_nodes(cluster, 'NODEMANAGER', nms)
cu.delete_instances(cluster, instances)
cu.refresh_nodes(cluster, 'DATANODE', cu.HDFS_SERVICE_NAME)
cu.refresh_nodes(cluster, 'NODEMANAGER', cu.YARN_SERVICE_NAME)
def _configure_os(instances):
with context.ThreadGroup() as tg:
for inst in instances:
tg.spawn('cdh-repo-conf-%s' % inst.instance_name,
_configure_repo_from_inst, inst)
def _configure_repo_from_inst(instance):
LOG.debug("Configure repos from instance '%(instance)s'" % {
'instance': instance.instance_name})
cluster = instance.node_group.cluster
cdh5_repo = c_helper.get_cdh5_repo_url(cluster)
cdh5_key = c_helper.get_cdh5_key_url(cluster)
cm5_repo = c_helper.get_cm5_repo_url(cluster)
cm5_key = c_helper.get_cm5_key_url(cluster)
with instance.remote() as r:
if cmd.is_ubuntu_os(r):
cdh5_repo = cdh5_repo or c_helper.DEFAULT_CDH5_UBUNTU_REPO_LIST_URL
cdh5_key = cdh5_key or c_helper.DEFAULT_CDH5_UBUNTU_REPO_KEY_URL
cm5_repo = cm5_repo or c_helper.DEFAULT_CM5_UBUNTU_REPO_LIST_URL
cm5_key = cm5_key or c_helper.DEFAULT_CM5_UBUNTU_REPO_KEY_URL
cmd.add_ubuntu_repository(r, cdh5_repo, 'cdh')
cmd.add_apt_key(r, cdh5_key)
cmd.add_ubuntu_repository(r, cm5_repo, 'cm')
cmd.add_apt_key(r, cm5_key)
cmd.update_repository(r)
if cmd.is_centos_os(r):
cdh5_repo = cdh5_repo or c_helper.DEFAULT_CDH5_CENTOS_REPO_LIST_URL
cm5_repo = cm5_repo or c_helper.DEFAULT_CM5_CENTOS_REPO_LIST_URL
cmd.add_centos_repository(r, cdh5_repo, 'cdh')
cmd.add_centos_repository(r, cm5_repo, 'cm')
def _install_packages(instances, packages):
with context.ThreadGroup() as tg:
for i in instances:
tg.spawn('cdh-inst-pkgs-%s' % i.instance_name,
_install_pkgs, i, packages)
def _install_pkgs(instance, packages):
with instance.remote() as r:
cmd.install_packages(r, packages)
def _start_cloudera_agents(instances):
with context.ThreadGroup() as tg:
for i in instances:
tg.spawn('cdh-agent-start-%s' % i.instance_name,
_start_cloudera_agent, i)
def _await_agents(instances):
api = cu.get_api_client(instances[0].node_group.cluster)
timeout = 300
LOG.debug("Waiting %(timeout)s seconds for agent connected to manager" % {
'timeout': timeout})
s_time = timeutils.utcnow()
while timeutils.delta_seconds(s_time, timeutils.utcnow()) < timeout:
hostnames = [i.fqdn() for i in instances]
hostnames_to_manager = [h.hostname for h in api.get_all_hosts('full')]
is_ok = True
for hostname in hostnames:
if hostname not in hostnames_to_manager:
is_ok = False
break
if not is_ok:
context.sleep(5)
else:
break
else:
raise ex.HadoopProvisionError(_("Cloudera agents failed to connect to"
" Cloudera Manager"))
def _start_cloudera_agent(instance):
mng_hostname = pu.get_manager(instance.node_group.cluster).hostname()
with instance.remote() as r:
cmd.start_ntp(r)
cmd.configure_agent(r, mng_hostname)
cmd.start_agent(r)
def _start_cloudera_manager(cluster):
manager = pu.get_manager(cluster)
with manager.remote() as r:
cmd.start_cloudera_db(r)
cmd.start_manager(r)
timeout = 300
LOG.debug("Waiting %(timeout)s seconds for Manager to start : " % {
'timeout': timeout})
s_time = timeutils.utcnow()
while timeutils.delta_seconds(s_time, timeutils.utcnow()) < timeout:
try:
conn = telnetlib.Telnet(manager.management_ip, CM_API_PORT)
conn.close()
break
except IOError:
context.sleep(2)
else:
message = _("Cloudera Manager failed to start in %(timeout)s minutes "
"on node '%(node)s' of cluster '%(cluster)s'") % {
'timeout': timeout / 60,
'node': manager.management_ip,
'cluster': cluster.name}
raise ex.HadoopProvisionError(message)
LOG.info(_LI("Cloudera Manager has been started"))
def _create_services(cluster):
api = cu.get_api_client(cluster)
fullversion = ('5.0.0' if cluster.hadoop_version == '5'
else cluster.hadoop_version)
cm_cluster = api.create_cluster(cluster.name,
fullVersion=fullversion)
if len(pu.get_zookeepers(cluster)) > 0:
cm_cluster.create_service(cu.ZOOKEEPER_SERVICE_NAME,
ZOOKEEPER_SERVICE_TYPE)
cm_cluster.create_service(cu.HDFS_SERVICE_NAME, HDFS_SERVICE_TYPE)
cm_cluster.create_service(cu.YARN_SERVICE_NAME, YARN_SERVICE_TYPE)
cm_cluster.create_service(cu.OOZIE_SERVICE_NAME, OOZIE_SERVICE_TYPE)
if pu.get_hive_metastore(cluster):
cm_cluster.create_service(cu.HIVE_SERVICE_NAME, HIVE_SERVICE_TYPE)
if pu.get_hue(cluster):
cm_cluster.create_service(cu.HUE_SERVICE_NAME, HUE_SERVICE_TYPE)
if pu.get_spark_historyserver(cluster):
cm_cluster.create_service(cu.SPARK_SERVICE_NAME, SPARK_SERVICE_TYPE)
if pu.get_hbase_master(cluster):
cm_cluster.create_service(cu.HBASE_SERVICE_NAME, HBASE_SERVICE_TYPE)
if len(pu.get_flumes(cluster)) > 0:
cm_cluster.create_service(cu.FLUME_SERVICE_NAME, FLUME_SERVICE_TYPE)
if pu.get_sentry(cluster):
cm_cluster.create_service(cu.SENTRY_SERVICE_NAME, SENTRY_SERVICE_TYPE)
if len(pu.get_solrs(cluster)) > 0:
cm_cluster.create_service(cu.SOLR_SERVICE_NAME,
SOLR_SERVICE_TYPE)
if pu.get_sqoop(cluster):
cm_cluster.create_service(cu.SQOOP_SERVICE_NAME, SQOOP_SERVICE_TYPE)
if len(pu.get_hbase_indexers(cluster)) > 0:
cm_cluster.create_service(cu.KS_INDEXER_SERVICE_NAME,
KS_INDEXER_SERVICE_TYPE)
if pu.get_catalogserver(cluster):
cm_cluster.create_service(cu.IMPALA_SERVICE_NAME, IMPALA_SERVICE_TYPE)
def _configure_services(cluster):
cm_cluster = cu.get_cloudera_cluster(cluster)
if len(pu.get_zookeepers(cluster)) > 0:
zookeeper = cm_cluster.get_service(cu.ZOOKEEPER_SERVICE_NAME)
zookeeper.update_config(_get_configs(ZOOKEEPER_SERVICE_TYPE,
cluster=cluster))
hdfs = cm_cluster.get_service(cu.HDFS_SERVICE_NAME)
hdfs.update_config(_get_configs(HDFS_SERVICE_TYPE, cluster=cluster))
yarn = cm_cluster.get_service(cu.YARN_SERVICE_NAME)
yarn.update_config(_get_configs(YARN_SERVICE_TYPE, cluster=cluster))
oozie = cm_cluster.get_service(cu.OOZIE_SERVICE_NAME)
oozie.update_config(_get_configs(OOZIE_SERVICE_TYPE, cluster=cluster))
if pu.get_hive_metastore(cluster):
hive = cm_cluster.get_service(cu.HIVE_SERVICE_NAME)
hive.update_config(_get_configs(HIVE_SERVICE_TYPE, cluster=cluster))
if pu.get_hue(cluster):
hue = cm_cluster.get_service(cu.HUE_SERVICE_NAME)
hue.update_config(_get_configs(HUE_SERVICE_TYPE, cluster=cluster))
if pu.get_spark_historyserver(cluster):
spark = cm_cluster.get_service(cu.SPARK_SERVICE_NAME)
spark.update_config(_get_configs(SPARK_SERVICE_TYPE, cluster=cluster))
if pu.get_hbase_master(cluster):
hbase = cm_cluster.get_service(cu.HBASE_SERVICE_NAME)
hbase.update_config(_get_configs(HBASE_SERVICE_TYPE, cluster=cluster))
if len(pu.get_flumes(cluster)) > 0:
flume = cm_cluster.get_service(cu.FLUME_SERVICE_NAME)
flume.update_config(_get_configs(FLUME_SERVICE_TYPE, cluster=cluster))
if pu.get_sentry(cluster):
sentry = cm_cluster.get_service(cu.SENTRY_SERVICE_NAME)
sentry.update_config(_get_configs(SENTRY_SERVICE_TYPE,
cluster=cluster))
if len(pu.get_solrs(cluster)) > 0:
solr = cm_cluster.get_service(cu.SOLR_SERVICE_NAME)
solr.update_config(_get_configs(SOLR_SERVICE_TYPE, cluster=cluster))
if pu.get_sqoop(cluster):
sqoop = cm_cluster.get_service(cu.SQOOP_SERVICE_NAME)
sqoop.update_config(_get_configs(SQOOP_SERVICE_TYPE, cluster=cluster))
if len(pu.get_hbase_indexers(cluster)) > 0:
ks_indexer = cm_cluster.get_service(cu.KS_INDEXER_SERVICE_NAME)
ks_indexer.update_config(_get_configs(KS_INDEXER_SERVICE_TYPE,
cluster=cluster))
if pu.get_catalogserver(cluster):
impala = cm_cluster.get_service(cu.IMPALA_SERVICE_NAME)
impala.update_config(_get_configs(IMPALA_SERVICE_TYPE,
cluster=cluster))
def _configure_instances(instances):
for inst in instances:
_configure_instance(inst)
def _configure_instance(instance):
for process in instance.node_group.node_processes:
_add_role(instance, process)
def _add_role(instance, process):
if process in ['CLOUDERA_MANAGER']:
return
process = pu.convert_role_showname(process)
service = cu.get_service(process, instance=instance)
role = service.create_role(cu.get_role_name(instance, process),
process, instance.fqdn())
role.update_config(_get_configs(process, node_group=instance.node_group))
def _configure_manager(cluster):
cu.create_mgmt_service(cluster)
def _configure_swift(instances):
with context.ThreadGroup() as tg:
for i in instances:
tg.spawn('cdh-swift-conf-%s' % i.instance_name,
_configure_swift_to_inst, i)
def _configure_swift_to_inst(instance):
cluster = instance.node_group.cluster
with instance.remote() as r:
r.execute_command('sudo curl %s -o %s/hadoop-openstack.jar' % (
c_helper.get_swift_lib_url(cluster), HADOOP_LIB_DIR))
def _put_hive_hdfs_xml(cluster):
servers = pu.get_hive_servers(cluster)
with servers[0].remote() as r:
conf_path = edp_u.get_hive_shared_conf_path('hdfs')
r.execute_command(
'sudo su - -c "hadoop fs -mkdir -p %s" hdfs'
% os.path.dirname(conf_path))
r.execute_command(
'sudo su - -c "hadoop fs -put /etc/hive/conf/hive-site.xml '
'%s" hdfs' % conf_path)
def _configure_hive(cluster):
manager = pu.get_manager(cluster)
with manager.remote() as r:
db_helper.create_hive_database(cluster, r)
def _configure_sentry(cluster):
manager = pu.get_manager(cluster)
with manager.remote() as r:
db_helper.create_sentry_database(cluster, r)
def _install_extjs(cluster):
extjs_remote_location = c_helper.get_extjs_lib_url(cluster)
extjs_vm_location_dir = '/var/lib/oozie'
extjs_vm_location_path = extjs_vm_location_dir + '/extjs.zip'
with pu.get_oozie(cluster).remote() as r:
if r.execute_command('ls %s/ext-2.2' % extjs_vm_location_dir,
raise_when_error=False)[0] != 0:
r.execute_command('curl -L -o \'%s\' %s' % (
extjs_vm_location_path, extjs_remote_location),
run_as_root=True)
r.execute_command('unzip %s -d %s' % (
extjs_vm_location_path, extjs_vm_location_dir),
run_as_root=True)
def start_cluster(cluster):
if pu.get_oozie(cluster):
_install_extjs(cluster)
if pu.get_hive_metastore(cluster):
_configure_hive(cluster)
if pu.get_sentry(cluster):
_configure_sentry(cluster)
cu.first_run(cluster)
if c_helper.is_swift_enabled(cluster):
instances = gu.get_instances(cluster)
_configure_swift(instances)
if pu.get_hive_metastore(cluster):
_put_hive_hdfs_xml(cluster)
if pu.get_flumes(cluster):
cm_cluster = cu.get_cloudera_cluster(cluster)
flume = cm_cluster.get_service(cu.FLUME_SERVICE_NAME)
cu.start_service(flume)
cu.restart_mgmt_service(cluster)
def get_open_ports(node_group):
ports = [9000] # for CM agent
ports_map = {
'CLOUDERA_MANAGER': [7180, 7182, 7183, 7432, 7184, 8084, 8086, 10101,
9997, 9996, 8087, 9998, 9999, 8085, 9995, 9994],
'HDFS_NAMENODE': [8020, 8022, 50070, 50470],
'HDFS_SECONDARYNAMENODE': [50090, 50495],
'HDFS_DATANODE': [50010, 1004, 50075, 1006, 50020],
'YARN_RESOURCEMANAGER': [8030, 8031, 8032, 8033, 8088],
'YARN_NODEMANAGER': [8040, 8041, 8042],
'YARN_JOBHISTORY': [10020, 19888],
'HIVE_METASTORE': [9083],
'HIVE_SERVER2': [10000],
'HUE_SERVER': [8888],
'OOZIE_SERVER': [11000, 11001],
'SPARK_YARN_HISTORY_SERVER': [18088],
'ZOOKEEPER_SERVER': [2181, 3181, 4181, 9010],
'HBASE_MASTER': [60000],
'HBASE_REGIONSERVER': [60020],
'FLUME_AGENT': [41414],
'SENTRY_SERVER': [8038],
'SOLR_SERVER': [8983, 8984],
'SQOOP_SERVER': [8005, 12000],
'KEY_VALUE_STORE_INDEXER': [],
'IMPALA_CATALOGSERVER': [25020, 26000],
'IMPALA_STATESTORE': [25010, 24000],
'IMPALAD': [21050, 21000, 23000, 25000, 28000, 22000]
}
for process in node_group.node_processes:
if process in ports_map:
ports.extend(ports_map[process])
return ports

101
sahara/plugins/cdh/plugin.py
View File

@ -13,20 +13,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from sahara import conductor
from sahara import context
from sahara.i18n import _
from sahara.plugins.cdh import config_helper as c_helper
from sahara.plugins.cdh import deploy as dp
from sahara.plugins.cdh import edp_engine
from sahara.plugins.cdh import utils as cu
from sahara.plugins.cdh import validation as vl
from sahara.plugins.cdh import versionfactory as vhf
from sahara.plugins import provisioning as p
conductor = conductor.API
class CDHPluginProvider(p.ProvisioningPluginBase):
def __init__(self):
self.version_factory = vhf.VersionFactory.get_instance()
def get_title(self):
return "Cloudera Plugin"
@ -36,90 +30,47 @@ class CDHPluginProvider(p.ProvisioningPluginBase):
'launch the Cloudera distribution of Apache Hadoop '
'(CDH) with Cloudera Manager management console.')
def _get_version_handler(self, hadoop_version):
return self.version_factory.get_version_handler(hadoop_version)
def get_versions(self):
return ['5', '5.2.0', '5.3.0']
return self.version_factory.get_versions()
def get_node_processes(self, hadoop_version):
processes = {
"CLOUDERA": ['CLOUDERA_MANAGER'],
"HDFS": [],
"NAMENODE": ['HDFS_NAMENODE'],
"DATANODE": ['HDFS_DATANODE'],
"SECONDARYNAMENODE": ['HDFS_SECONDARYNAMENODE'],
"YARN": [],
"RESOURCEMANAGER": ['YARN_RESOURCEMANAGER'],
"NODEMANAGER": ['YARN_NODEMANAGER'],
"JOBHISTORY": ['YARN_JOBHISTORY'],
"OOZIE": ['OOZIE_SERVER'],
"HIVE": [],
"HIVESERVER": ['HIVE_SERVER2'],
"HIVEMETASTORE": ['HIVE_METASTORE'],
"WEBHCAT": ['HIVE_WEBHCAT'],
"HUE": ['HUE_SERVER'],
"SPARK_ON_YARN": ['SPARK_YARN_HISTORY_SERVER'],
"ZOOKEEPER": ['ZOOKEEPER_SERVER'],
"HBASE": [],
"MASTER": ['HBASE_MASTER'],
"REGIONSERVER": ['HBASE_REGIONSERVER'],
"FLUME": ['FLUME_AGENT'],
"IMPALA": [],
"CATALOGSERVER": ['IMPALA_CATALOGSERVER'],
"STATESTORE": ['IMPALA_STATESTORE'],
"IMPALAD": ['IMPALAD'],
"KS_INDEXER": ['KEY_VALUE_STORE_INDEXER'],
"SOLR": ['SOLR_SERVER'],
"SQOOP": ['SQOOP_SERVER']
}
if hadoop_version >= '5.2.0':
processes["SENTRY"] = ['SENTRY_SERVER']
return processes
return self._get_version_handler(hadoop_version).get_node_processes()
def get_configs(self, hadoop_version):
return c_helper.get_plugin_configs()
return self._get_version_handler(hadoop_version).get_plugin_configs()
def configure_cluster(self, cluster):
dp.configure_cluster(cluster)
return self._get_version_handler(
cluster.hadoop_version).configure_cluster(cluster)
def start_cluster(self, cluster):
dp.start_cluster(cluster)
self._set_cluster_info(cluster)
return self._get_version_handler(
cluster.hadoop_version).start_cluster(cluster)
def validate(self, cluster):
vl.validate_cluster_creating(cluster)
return self._get_version_handler(
cluster.hadoop_version).validate(cluster)
def scale_cluster(self, cluster, instances):
dp.scale_cluster(cluster, instances)
return self._get_version_handler(
cluster.hadoop_version).scale_cluster(cluster, instances)
def decommission_nodes(self, cluster, instances):
dp.decommission_cluster(cluster, instances)
return self._get_version_handler(
cluster.hadoop_version).decommission_nodes(cluster, instances)
def validate_scaling(self, cluster, existing, additional):
vl.validate_existing_ng_scaling(cluster, existing)
vl.validate_additional_ng_scaling(cluster, additional)
def _set_cluster_info(self, cluster):
mng = cu.get_manager(cluster)
info = {
'Cloudera Manager': {
'Web UI': 'http://%s:7180' % mng.management_ip,
'Username': 'admin',
'Password': 'admin'
}
}
hue = cu.get_hue(cluster)
if hue:
info['Hue Dashboard'] = {
'Web UI': 'http://%s:8888' % hue.management_ip
}
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {'info': info})
return self._get_version_handler(
cluster.hadoop_version).validate_scaling(cluster, existing,
additional)
def get_edp_engine(self, cluster, job_type):
if job_type in edp_engine.EdpOozieEngine.get_supported_job_types():
return edp_engine.EdpOozieEngine(cluster)
return None
return self._get_version_handler(
cluster.hadoop_version).get_edp_engine(cluster, job_type)
def get_open_ports(self, node_group):
return dp.get_open_ports(node_group)
return self._get_version_handler(
node_group.cluster.hadoop_version).get_open_ports(node_group)

312
sahara/plugins/cdh/plugin_utils.py Normal file
View File

@ -0,0 +1,312 @@
# Copyright (c) 2014 Intel Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file only contains utils not related to cm_api, while in
# cloudera_utils the functios are cm_api involved.
import os
import telnetlib
from oslo_log import log as logging
from oslo_utils import timeutils
from sahara.conductor import resource as res
from sahara import context
from sahara.i18n import _
from sahara.i18n import _LI
from sahara.plugins.cdh import commands as cmd
from sahara.plugins import exceptions as ex
from sahara.plugins import utils as u
from sahara.utils import edp as edp_u
PATH_TO_CORE_SITE_XML = '/etc/hadoop/conf/core-site.xml'
HADOOP_LIB_DIR = '/usr/lib/hadoop-mapreduce'
CM_API_PORT = 7180
LOG = logging.getLogger(__name__)
class AbstractPluginUtils(object):
# c_helper and db_helper will be defined in derived classes.
c_helper = None
db_helper = None
def get_role_name(self, instance, service):
# NOTE: role name must match regexp "[_A-Za-z][-_A-Za-z0-9]{0,63}"
shortcuts = {
'ALERTPUBLISHER': 'AP',
'DATANODE': 'DN',
'EVENTSERVER': 'ES',
'HIVEMETASTORE': 'HVM',
'HIVESERVER2': 'HVS',
'HOSTMONITOR': 'HM',
'JOBHISTORY': 'JS',
'MASTER': 'M',
'NAMENODE': 'NN',
'NODEMANAGER': 'NM',
'OOZIE_SERVER': 'OS',
'REGIONSERVER': 'RS',
'RESOURCEMANAGER': 'RM',
'SECONDARYNAMENODE': 'SNN',
'SERVER': 'S',
'SERVICEMONITOR': 'SM',
'SPARK_YARN_HISTORY_SERVER': 'SHS',
'WEBHCAT': 'WHC'
}
return '%s_%s' % (shortcuts.get(service, service),
instance.hostname().replace('-', '_'))
def get_manager(self, cluster):
return u.get_instance(cluster, 'CLOUDERA_MANAGER')
def get_namenode(self, cluster):
return u.get_instance(cluster, "HDFS_NAMENODE")
def get_datanodes(self, cluster):
return u.get_instances(cluster, 'HDFS_DATANODE')
def get_secondarynamenode(self, cluster):
return u.get_instance(cluster, 'HDFS_SECONDARYNAMENODE')
def get_historyserver(self, cluster):
return u.get_instance(cluster, 'YARN_JOBHISTORY')
def get_resourcemanager(self, cluster):
return u.get_instance(cluster, 'YARN_RESOURCEMANAGER')
def get_nodemanagers(self, cluster):
return u.get_instances(cluster, 'YARN_NODEMANAGER')
def get_oozie(self, cluster):
return u.get_instance(cluster, 'OOZIE_SERVER')
def get_hive_metastore(self, cluster):
return u.get_instance(cluster, 'HIVE_METASTORE')
def get_hive_servers(self, cluster):
return u.get_instances(cluster, 'HIVE_SERVER2')
def get_hue(self, cluster):
return u.get_instance(cluster, 'HUE_SERVER')
def get_spark_historyserver(self, cluster):
return u.get_instance(cluster, 'SPARK_YARN_HISTORY_SERVER')
def get_zookeepers(self, cluster):
return u.get_instances(cluster, 'ZOOKEEPER_SERVER')
def get_hbase_master(self, cluster):
return u.get_instance(cluster, 'HBASE_MASTER')
def convert_process_configs(self, configs):
p_dict = {
"CLOUDERA": ['MANAGER'],
"NAMENODE": ['NAMENODE'],
"DATANODE": ['DATANODE'],
"SECONDARYNAMENODE": ['SECONDARYNAMENODE'],
"RESOURCEMANAGER": ['RESOURCEMANAGER'],
"NODEMANAGER": ['NODEMANAGER'],
"JOBHISTORY": ['JOBHISTORY'],
"OOZIE": ['OOZIE_SERVER'],
"HIVESERVER": ['HIVESERVER2'],
"HIVEMETASTORE": ['HIVEMETASTORE'],
"WEBHCAT": ['WEBHCAT'],
"HUE": ['HUE_SERVER'],
"SPARK_ON_YARN": ['SPARK_YARN_HISTORY_SERVER'],
"ZOOKEEPER": ['SERVER'],
"MASTER": ['MASTER'],
"REGIONSERVER": ['REGIONSERVER']
}
if isinstance(configs, res.Resource):
configs = configs.to_dict()
for k in configs.keys():
if k in p_dict.keys():
item = configs[k]
del configs[k]
newkey = p_dict[k][0]
configs[newkey] = item
return res.Resource(configs)
def convert_role_showname(self, showname):
name_dict = {
'CLOUDERA_MANAGER': 'MANAGER',
'HDFS_NAMENODE': 'NAMENODE',
'HDFS_DATANODE': 'DATANODE',
'HDFS_SECONDARYNAMENODE': 'SECONDARYNAMENODE',
'YARN_RESOURCEMANAGER': 'RESOURCEMANAGER',
'YARN_NODEMANAGER': 'NODEMANAGER',
'YARN_JOBHISTORY': 'JOBHISTORY',
'OOZIE_SERVER': 'OOZIE_SERVER',
'HIVE_SERVER2': 'HIVESERVER2',
'HIVE_METASTORE': 'HIVEMETASTORE',
'HIVE_WEBHCAT': 'WEBHCAT',
'HUE_SERVER': 'HUE_SERVER',
'SPARK_YARN_HISTORY_SERVER': 'SPARK_YARN_HISTORY_SERVER',
'ZOOKEEPER_SERVER': 'SERVER',
'HBASE_MASTER': 'MASTER',
'HBASE_REGIONSERVER': 'REGIONSERVER',
'FLUME_AGENT': 'AGENT',
'IMPALA_CATALOGSERVER': 'CATALOGSERVER',
'IMPALA_STATESTORE': 'STATESTORE',
'IMPALAD': 'IMPALAD',
'KEY_VALUE_STORE_INDEXER': 'HBASE_INDEXER',
'SENTRY_SERVER': 'SENTRY_SERVER',
'SOL_SERVER': 'SOLR_SERVER',
'SQOOP_SERVER': 'SQOOP_SERVER',
}
return name_dict.get(showname, showname)
def install_packages(self, instances, packages):
with context.ThreadGroup() as tg:
for i in instances:
tg.spawn('cdh-inst-pkgs-%s' % i.instance_name,
self._install_pkgs, i, packages)
def _install_pkgs(self, instance, packages):
with instance.remote() as r:
cmd.install_packages(r, packages)
def start_cloudera_agents(self, instances):
with context.ThreadGroup() as tg:
for i in instances:
tg.spawn('cdh-agent-start-%s' % i.instance_name,
self.start_cloudera_agent, i)
def start_cloudera_agent(self, instance):
mng_hostname = self.get_manager(instance.node_group.cluster).hostname()
with instance.remote() as r:
cmd.start_ntp(r)
cmd.configure_agent(r, mng_hostname)
cmd.start_agent(r)
def configure_swift(self, cluster):
if self.c_helper.is_swift_enabled(cluster):
instances = u.get_instances(cluster)
with context.ThreadGroup() as tg:
for i in instances:
tg.spawn('cdh-swift-conf-%s' % i.instance_name,
self.configure_swift_to_inst, i)
def configure_swift_to_inst(self, instance):
cluster = instance.node_group.cluster
with instance.remote() as r:
r.execute_command('sudo curl %s -o %s/hadoop-openstack.jar' % (
self.c_helper.get_swift_lib_url(cluster), HADOOP_LIB_DIR))
def put_hive_hdfs_xml(self, cluster):
servers = self.get_hive_servers(cluster)
with servers[0].remote() as r:
conf_path = edp_u.get_hive_shared_conf_path('hdfs')
r.execute_command(
'sudo su - -c "hadoop fs -mkdir -p %s" hdfs'
% os.path.dirname(conf_path))
r.execute_command(
'sudo su - -c "hadoop fs -put /etc/hive/conf/hive-site.xml '
'%s" hdfs' % conf_path)
def configure_hive(self, cluster):
manager = self.get_manager(cluster)
with manager.remote() as r:
self.db_helper.create_hive_database(cluster, r)
def create_hive_hive_directory(self, cluster):
# Hive requires /tmp/hive-hive directory
namenode = self.get_namenode(cluster)
with namenode.remote() as r:
r.execute_command(
'sudo su - -c "hadoop fs -mkdir -p /tmp/hive-hive" hdfs')
r.execute_command(
'sudo su - -c "hadoop fs -chown hive /tmp/hive-hive" hdfs')
def install_extjs(self, cluster):
extjs_remote_location = self.c_helper.get_extjs_lib_url(cluster)
extjs_vm_location_dir = '/var/lib/oozie'
extjs_vm_location_path = extjs_vm_location_dir + '/extjs.zip'
with self.get_oozie(cluster).remote() as r:
if r.execute_command('ls %s/ext-2.2' % extjs_vm_location_dir,
raise_when_error=False)[0] != 0:
r.execute_command('curl -L -o \'%s\' %s' % (
extjs_vm_location_path, extjs_remote_location),
run_as_root=True)
r.execute_command('unzip %s -d %s' % (
extjs_vm_location_path, extjs_vm_location_dir),
run_as_root=True)
def start_cloudera_manager(self, cluster):
manager = self.get_manager(cluster)
with manager.remote() as r:
cmd.start_cloudera_db(r)
cmd.start_manager(r)
timeout = 300
LOG.debug("Waiting %(timeout)s seconds for Manager to start : " % {
'timeout': timeout})
s_time = timeutils.utcnow()
while timeutils.delta_seconds(s_time, timeutils.utcnow()) < timeout:
try:
conn = telnetlib.Telnet(manager.management_ip, CM_API_PORT)
conn.close()
break
except IOError:
context.sleep(2)
else:
message = _("Cloudera Manager failed to start in %(timeout)s "
"minutes on node '%(node)s' of cluster "
"'%(cluster)s'") % {
'timeout': timeout / 60,
'node': manager.management_ip,
'cluster': cluster.name}
raise ex.HadoopProvisionError(message)
LOG.info(_LI("Cloudera Manager has been started"))
def configure_os(self, instances):
with context.ThreadGroup() as tg:
for inst in instances:
tg.spawn('cdh-repo-conf-%s' % inst.instance_name,
self._configure_repo_from_inst, inst)
def _configure_repo_from_inst(self, instance):
LOG.debug("Configure repos from instance '%(instance)s'" % {
'instance': instance.instance_name})
cluster = instance.node_group.cluster
cdh5_key = self.c_helper.get_cdh5_key_url(cluster)
cm5_key = self.c_helper.get_cm5_key_url(cluster)
with instance.remote() as r:
if cmd.is_ubuntu_os(r):
cdh5_key = (cdh5_key or
self.c_helper.DEFAULT_CDH5_UBUNTU_REPO_KEY_URL)
cm5_key = (cm5_key or
self.c_helper.DEFAULT_CM5_UBUNTU_REPO_KEY_URL)
cdh5_repo_content = self.c_helper.CDH5_UBUNTU_REPO
cm5_repo_content = self.c_helper.CM5_UBUNTU_REPO
cmd.write_ubuntu_repository(r, cdh5_repo_content, 'cdh')
cmd.add_apt_key(r, cdh5_key)
cmd.write_ubuntu_repository(r, cm5_repo_content, 'cm')
cmd.add_apt_key(r, cm5_key)
cmd.update_repository(r)
if cmd.is_centos_os(r):
cdh5_repo_content = self.c_helper.CDH5_CENTOS_REPO
cm5_repo_content = self.c_helper.CM5_CENTOS_REPO
cmd.write_centos_repository(r, cdh5_repo_content, 'cdh')
cmd.write_centos_repository(r, cm5_repo_content, 'cm')
cmd.update_repository(r)

173
sahara/plugins/cdh/utils.py
View File

@ -1,173 +0,0 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from sahara.conductor import resource as res
from sahara.plugins import utils as u
def get_manager(cluster):
return u.get_instance(cluster, 'CLOUDERA_MANAGER')
def get_namenode(cluster):
return u.get_instance(cluster, 'HDFS_NAMENODE')
def get_secondarynamenode(cluster):
return u.get_instance(cluster, 'HDFS_SECONDARYNAMENODE')
def get_datanodes(cluster):
return u.get_instances(cluster, 'HDFS_DATANODE')
def get_resourcemanager(cluster):
return u.get_instance(cluster, 'YARN_RESOURCEMANAGER')
def get_nodemanagers(cluster):
return u.get_instances(cluster, 'YARN_NODEMANAGER')
def get_historyserver(cluster):
return u.get_instance(cluster, 'YARN_JOBHISTORY')
def get_oozie(cluster):
return u.get_instance(cluster, 'OOZIE_SERVER')
def get_hive_metastore(cluster):
return u.get_instance(cluster, 'HIVE_METASTORE')
def get_hive_servers(cluster):
return u.get_instances(cluster, 'HIVE_SERVER2')
def get_hue(cluster):
return u.get_instance(cluster, 'HUE_SERVER')
def get_spark_historyserver(cluster):
return u.get_instance(cluster, 'SPARK_YARN_HISTORY_SERVER')
def get_zookeepers(cluster):
return u.get_instances(cluster, 'ZOOKEEPER_SERVER')
def get_hbase_master(cluster):
return u.get_instance(cluster, 'HBASE_MASTER')
def get_flumes(cluster):
return u.get_instances(cluster, 'FLUME_AGENT')
def get_sentry(cluster):
return u.get_instance(cluster, 'SENTRY_SERVER')
def get_solrs(cluster):
return u.get_instances(cluster, 'SOLR_SERVER')
def get_sqoop(cluster):
return u.get_instance(cluster, 'SQOOP_SERVER')
def get_hbase_indexers(cluster):
return u.get_instances(cluster, 'KEY_VALUE_STORE_INDEXER')
def get_catalogserver(cluster):
return u.get_instance(cluster, 'IMPALA_CATALOGSERVER')
def get_statestore(cluster):
return u.get_instance(cluster, 'IMPALA_STATESTORE')
def get_impalads(cluster):
return u.get_instances(cluster, 'IMPALAD')
def convert_process_configs(configs):
p_dict = {
"CLOUDERA": ['MANAGER'],
"NAMENODE": ['NAMENODE'],
"DATANODE": ['DATANODE'],
"SECONDARYNAMENODE": ['SECONDARYNAMENODE'],
"RESOURCEMANAGER": ['RESOURCEMANAGER'],
"NODEMANAGER": ['NODEMANAGER'],
"JOBHISTORY": ['JOBHISTORY'],
"OOZIE": ['OOZIE_SERVER'],
"HIVESERVER": ['HIVESERVER2'],
"HIVEMETASTORE": ['HIVEMETASTORE'],
"WEBHCAT": ['WEBHCAT'],
"HUE": ['HUE_SERVER'],
"SPARK_ON_YARN": ['SPARK_YARN_HISTORY_SERVER'],
"ZOOKEEPER": ['SERVER'],
"MASTER": ['MASTER'],
"REGIONSERVER": ['REGIONSERVER'],
"FLUME": ['AGENT'],
"CATALOGSERVER": ['CATALOGSERVER'],
"STATESTORE": ['STATESTORE'],
"IMPALAD": ['IMPALAD'],
"KS_INDEXER": ['HBASE_INDEXER'],
"SENTRY": ['SENTRY_SERVER'],
"SOLR": ['SOLR_SERVER'],
"SQOOP": ['SQOOP_SERVER']
}
if isinstance(configs, res.Resource):
configs = configs.to_dict()
for k in configs.keys():
if k in p_dict.keys():
item = configs[k]
del configs[k]
newkey = p_dict[k][0]
configs[newkey] = item
return res.Resource(configs)
def convert_role_showname(showname):
name_dict = {
'CLOUDERA_MANAGER': 'MANAGER',
'HDFS_NAMENODE': 'NAMENODE',
'HDFS_DATANODE': 'DATANODE',
'HDFS_SECONDARYNAMENODE': 'SECONDARYNAMENODE',
'YARN_RESOURCEMANAGER': 'RESOURCEMANAGER',
'YARN_NODEMANAGER': 'NODEMANAGER',
'YARN_JOBHISTORY': 'JOBHISTORY',
'OOZIE_SERVER': 'OOZIE_SERVER',
'HIVE_SERVER2': 'HIVESERVER2',
'HIVE_METASTORE': 'HIVEMETASTORE',
'HIVE_WEBHCAT': 'WEBHCAT',
'HUE_SERVER': 'HUE_SERVER',
'SPARK_YARN_HISTORY_SERVER': 'SPARK_YARN_HISTORY_SERVER',
'ZOOKEEPER_SERVER': 'SERVER',
'HBASE_MASTER': 'MASTER',
'HBASE_REGIONSERVER': 'REGIONSERVER',
'FLUME_AGENT': 'AGENT',
'IMPALA_CATALOGSERVER': 'CATALOGSERVER',
'IMPALA_STATESTORE': 'STATESTORE',
'IMPALAD': 'IMPALAD',
'KEY_VALUE_STORE_INDEXER': 'HBASE_INDEXER',
'SENTRY_SERVER': 'SENTRY_SERVER',
'SOLR_SERVER': 'SOLR_SERVER',
'SQOOP_SERVER': 'SQOOP_SERVER',
}
return name_dict.get(showname, None)

0
sahara/plugins/cdh/v5/__init__.py Normal file
View File

264
sahara/plugins/cdh/v5/cloudera_utils.py Normal file
View File

@ -0,0 +1,264 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# cm_api client is not present in OS requirements
try:
from cm_api import api_client
from cm_api.endpoints import services
except ImportError:
api_client = None
services = None
import six
from sahara.plugins.cdh import cloudera_utils as cu
from sahara.plugins.cdh.v5 import plugin_utils as pu
from sahara.plugins.cdh.v5 import validation as v
from sahara.swift import swift_helper
from sahara.utils import xmlutils
CM_API_PORT = 7180
HDFS_SERVICE_TYPE = 'HDFS'
YARN_SERVICE_TYPE = 'YARN'
OOZIE_SERVICE_TYPE = 'OOZIE'
HIVE_SERVICE_TYPE = 'HIVE'
HUE_SERVICE_TYPE = 'HUE'
SPARK_SERVICE_TYPE = 'SPARK_ON_YARN'
ZOOKEEPER_SERVICE_TYPE = 'ZOOKEEPER'
HBASE_SERVICE_TYPE = 'HBASE'
def _merge_dicts(a, b):
res = {}
def update(cfg):
for service, configs in six.iteritems(cfg):
if not res.get(service):
res[service] = {}
res[service].update(configs)
update(a)
update(b)
return res
class ClouderaUtilsV5(cu.ClouderaUtils):
def __init__(self):
cu.ClouderaUtils.__init__(self)
self.pu = pu.PluginUtilsV5()
@cu.cloudera_cmd
def format_namenode(self, hdfs_service):
for nn in hdfs_service.get_roles_by_type('NAMENODE'):
yield hdfs_service.format_hdfs(nn.name)[0]
@cu.cloudera_cmd
def create_hdfs_tmp(self, hdfs_service):
yield hdfs_service.create_hdfs_tmp()
@cu.cloudera_cmd
def create_yarn_job_history_dir(self, yarn_service):
yield yarn_service.create_yarn_job_history_dir()
@cu.cloudera_cmd
def create_oozie_db(self, oozie_service):
yield oozie_service.create_oozie_db()
@cu.cloudera_cmd
def install_oozie_sharelib(self, oozie_service):
yield oozie_service.install_oozie_sharelib()
@cu.cloudera_cmd
def create_hive_metastore_db(self, hive_service):
yield hive_service.create_hive_metastore_tables()
@cu.cloudera_cmd
def create_hive_dirs(self, hive_service):
yield hive_service.create_hive_userdir()
yield hive_service.create_hive_warehouse()
@cu.cloudera_cmd
def create_hbase_root(self, hbase_service):
yield hbase_service.create_hbase_root()
def create_services(self, cluster):
api = self.get_api_client(cluster)
fullversion = ('5.0.0' if cluster.hadoop_version == '5'
else cluster.hadoop_version)
cm_cluster = api.create_cluster(cluster.name,
fullVersion=fullversion)
if len(self.pu.get_zookeepers(cluster)) > 0:
cm_cluster.create_service(self.ZOOKEEPER_SERVICE_NAME,
ZOOKEEPER_SERVICE_TYPE)
cm_cluster.create_service(self.HDFS_SERVICE_NAME, HDFS_SERVICE_TYPE)
cm_cluster.create_service(self.YARN_SERVICE_NAME, YARN_SERVICE_TYPE)
cm_cluster.create_service(self.OOZIE_SERVICE_NAME, OOZIE_SERVICE_TYPE)
if self.pu.get_hive_metastore(cluster):
cm_cluster.create_service(self.HIVE_SERVICE_NAME,
HIVE_SERVICE_TYPE)
if self.pu.get_hue(cluster):
cm_cluster.create_service(self.HUE_SERVICE_NAME, HUE_SERVICE_TYPE)
if self.pu.get_spark_historyserver(cluster):
cm_cluster.create_service(self.SPARK_SERVICE_NAME,
SPARK_SERVICE_TYPE)
if self.pu.get_hbase_master(cluster):
cm_cluster.create_service(self.HBASE_SERVICE_NAME,
HBASE_SERVICE_TYPE)
def configure_services(self, cluster):
cm_cluster = self.get_cloudera_cluster(cluster)
if len(self.pu.get_zookeepers(cluster)) > 0:
zookeeper = cm_cluster.get_service(self.ZOOKEEPER_SERVICE_NAME)
zookeeper.update_config(self._get_configs(ZOOKEEPER_SERVICE_TYPE,
cluster=cluster))
hdfs = cm_cluster.get_service(self.HDFS_SERVICE_NAME)
hdfs.update_config(self._get_configs(HDFS_SERVICE_TYPE,
cluster=cluster))
yarn = cm_cluster.get_service(self.YARN_SERVICE_NAME)
yarn.update_config(self._get_configs(YARN_SERVICE_TYPE,
cluster=cluster))
oozie = cm_cluster.get_service(self.OOZIE_SERVICE_NAME)
oozie.update_config(self._get_configs(OOZIE_SERVICE_TYPE,
cluster=cluster))
if self.pu.get_hive_metastore(cluster):
hive = cm_cluster.get_service(self.HIVE_SERVICE_NAME)
hive.update_config(self._get_configs(HIVE_SERVICE_TYPE,
cluster=cluster))
if self.pu.get_hue(cluster):
hue = cm_cluster.get_service(self.HUE_SERVICE_NAME)
hue.update_config(self._get_configs(HUE_SERVICE_TYPE,
cluster=cluster))
if self.pu.get_spark_historyserver(cluster):
spark = cm_cluster.get_service(self.SPARK_SERVICE_NAME)
spark.update_config(self._get_configs(SPARK_SERVICE_TYPE,
cluster=cluster))
if self.pu.get_hbase_master(cluster):
hbase = cm_cluster.get_service(self.HBASE_SERVICE_NAME)
hbase.update_config(self._get_configs(HBASE_SERVICE_TYPE,
cluster=cluster))
def _get_configs(self, service, cluster=None, node_group=None):
def get_hadoop_dirs(mount_points, suffix):
return ','.join([x + suffix for x in mount_points])
all_confs = {}
if cluster:
zk_count = v._get_inst_count(cluster, 'ZOOKEEPER_SERVER')
core_site_safety_valve = ''
if self.pu.c_helper.is_swift_enabled(cluster):
configs = swift_helper.get_swift_configs()
confs = dict((c['name'], c['value']) for c in configs)
core_site_safety_valve = xmlutils.create_elements_xml(confs)
all_confs = {
'HDFS': {
'zookeeper_service':
self.ZOOKEEPER_SERVICE_NAME if zk_count else '',
'core_site_safety_valve': core_site_safety_valve
},
'HIVE': {
'mapreduce_yarn_service': self.YARN_SERVICE_NAME,
'zookeeper_service':
self.ZOOKEEPER_SERVICE_NAME if zk_count else ''
},
'OOZIE': {
'mapreduce_yarn_service': self.YARN_SERVICE_NAME,
'zookeeper_service':
self.ZOOKEEPER_SERVICE_NAME if zk_count else ''
},
'YARN': {
'hdfs_service': self.HDFS_SERVICE_NAME,
'zookeeper_service':
self.ZOOKEEPER_SERVICE_NAME if zk_count else ''
},
'HUE': {
'hive_service': self.HIVE_SERVICE_NAME,
'oozie_service': self.OOZIE_SERVICE_NAME,
'zookeeper_service':
self.ZOOKEEPER_SERVICE_NAME if zk_count else ''
},
'SPARK_ON_YARN': {
'yarn_service': self.YARN_SERVICE_NAME
},
'HBASE': {
'hdfs_service': self.HDFS_SERVICE_NAME,
'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME
}
}
hive_confs = {
'HIVE': {
'hive_metastore_database_type': 'postgresql',
'hive_metastore_database_host':
self.pu.get_manager(cluster).internal_ip,
'hive_metastore_database_port': '7432',
'hive_metastore_database_password':
self.pu.db_helper.get_hive_db_password(cluster)
}
}
hue_confs = {
'HUE': {
'hue_webhdfs':
self.pu.get_role_name(self.pu.get_namenode(cluster),
'NAMENODE')
}
}
all_confs = _merge_dicts(all_confs, hue_confs)
all_confs = _merge_dicts(all_confs, hive_confs)
all_confs = _merge_dicts(all_confs, cluster.cluster_configs)
if node_group:
paths = node_group.storage_paths()
ng_default_confs = {
'NAMENODE': {
'dfs_name_dir_list': get_hadoop_dirs(paths, '/fs/nn')
},
'SECONDARYNAMENODE': {
'fs_checkpoint_dir_list':
get_hadoop_dirs(paths, '/fs/snn')
},
'DATANODE': {
'dfs_data_dir_list': get_hadoop_dirs(paths, '/fs/dn'),
'dfs_datanode_data_dir_perm': 755,
'dfs_datanode_handler_count': 30
},
'NODEMANAGER': {
'yarn_nodemanager_local_dirs':
get_hadoop_dirs(paths, '/yarn/local')
},
'SERVER': {
'maxSessionTimeout': 60000
}
}
ng_user_confs = self.pu.convert_process_configs(
node_group.node_configs)
all_confs = _merge_dicts(all_confs, ng_user_confs)
all_confs = _merge_dicts(all_confs, ng_default_confs)
return all_confs.get(service, {})

216
sahara/plugins/cdh/v5/config_helper.py Normal file
View File

@ -0,0 +1,216 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from sahara.plugins import provisioning as p
from sahara.utils import files as f
CDH5_UBUNTU_REPO = ('deb [arch=amd64] http://archive.cloudera.com/cdh5'
'/ubuntu/precise/amd64/cdh precise-cdh5.0.0 contrib'
'\ndeb-src http://archive.cloudera.com/cdh5/ubuntu'
'/precise/amd64/cdh precise-cdh5.0.0 contrib')
DEFAULT_CDH5_UBUNTU_REPO_KEY_URL = ('http://archive.cloudera.com/cdh5/ubuntu'
'/precise/amd64/cdh/archive.key')
CM5_UBUNTU_REPO = ('deb [arch=amd64] http://archive.cloudera.com/cm5'
'/ubuntu/precise/amd64/cm precise-cm5.0.0 contrib'
'\ndeb-src http://archive.cloudera.com/cm5/ubuntu'
'/precise/amd64/cm precise-cm5.0.0 contrib')
DEFAULT_CM5_UBUNTU_REPO_KEY_URL = ('http://archive.cloudera.com/cm5/ubuntu'
'/precise/amd64/cm/archive.key')
CDH5_CENTOS_REPO = ('[cloudera-cdh5]'
'\nname=Cloudera\'s Distribution for Hadoop, Version 5'
'\nbaseurl=http://archive.cloudera.com/cdh5/redhat/6'
'/x86_64/cdh/5.0.0/'
'\ngpgkey = http://archive.cloudera.com/cdh5/redhat/6'
'/x86_64/cdh/RPM-GPG-KEY-cloudera'
'\ngpgcheck = 1')
CM5_CENTOS_REPO = ('[cloudera-manager]'
'\nname=Cloudera Manager'
'\nbaseurl=http://archive.cloudera.com/cm5/redhat/6'
'/x86_64/cm/5.0.0/'
'\ngpgkey = http://archive.cloudera.com/cm5/redhat/6'
'/x86_64/cm/RPM-GPG-KEY-cloudera'
'\ngpgcheck = 1')
DEFAULT_SWIFT_LIB_URL = ('https://repository.cloudera.com/artifactory/repo/org'
'/apache/hadoop/hadoop-openstack/2.3.0-cdh5.0.0'
'/hadoop-openstack-2.3.0-cdh5.0.0.jar')
DEFAULT_EXTJS_LIB_URL = 'http://extjs.com/deploy/ext-2.2.zip'
CDH5_REPO_URL = p.Config(
'CDH5 repo list URL', 'general', 'cluster', priority=1,
default_value="")
CDH5_REPO_KEY_URL = p.Config(
'CDH5 repo key URL (for debian-based only)', 'general', 'cluster',
priority=1, default_value="")
CM5_REPO_URL = p.Config(
'CM5 repo list URL', 'general', 'cluster', priority=1,
default_value="")
CM5_REPO_KEY_URL = p.Config(
'CM5 repo key URL (for debian-based only)', 'general', 'cluster',
priority=1, default_value="")
ENABLE_SWIFT = p.Config('Enable Swift', 'general', 'cluster',
config_type='bool', priority=1,
default_value=True)
SWIFT_LIB_URL = p.Config(
'Hadoop OpenStack library URL', 'general', 'cluster', priority=1,
default_value=DEFAULT_SWIFT_LIB_URL,
description=("Library that adds Swift support to CDH. The file will be "
"downloaded from VM."))
EXTJS_LIB_URL = p.Config(
"ExtJS library URL", 'general', 'cluster', priority=1,
default_value=DEFAULT_EXTJS_LIB_URL,
description=("Ext 2.2 library is required for Oozie Web Console. "
"The file will be downloaded from VM with oozie."))
def _get_cluster_plugin_configs():
return [CDH5_REPO_URL, CDH5_REPO_KEY_URL, CM5_REPO_URL, CM5_REPO_KEY_URL,
ENABLE_SWIFT, SWIFT_LIB_URL, EXTJS_LIB_URL]
# ng wide configs
def _load_json(path_to_file):
data = f.get_file_text(path_to_file)
return json.loads(data)
path_to_config = 'plugins/cdh/v5/resources/'
hdfs_confs = _load_json(path_to_config + 'hdfs-service.json')
namenode_confs = _load_json(path_to_config + 'hdfs-namenode.json')
datanode_confs = _load_json(path_to_config + 'hdfs-datanode.json')
secnamenode_confs = _load_json(path_to_config + 'hdfs-secondarynamenode.json')
yarn_confs = _load_json(path_to_config + 'yarn-service.json')
resourcemanager_confs = _load_json(
path_to_config + 'yarn-resourcemanager.json')
nodemanager_confs = _load_json(path_to_config + 'yarn-nodemanager.json')
jobhistory_confs = _load_json(path_to_config + 'yarn-jobhistory.json')
oozie_service_confs = _load_json(path_to_config + 'oozie-service.json')
oozie_role_confs = _load_json(path_to_config + 'oozie-oozie.json')
hive_service_confs = _load_json(path_to_config + 'hive-service.json')
hive_metastore_confs = _load_json(path_to_config + 'hive-metastore.json')
hive_hiveserver_confs = _load_json(path_to_config + 'hive-hiveserver2.json')
hive_webhcat_confs = _load_json(path_to_config + 'hive-webhcat.json')
hue_service_confs = _load_json(path_to_config + 'hue-service.json')
hue_role_confs = _load_json(path_to_config + 'hue-hue.json')
spark_service_confs = _load_json(path_to_config + 'spark-service.json')
spark_role_confs = _load_json(path_to_config + 'spark-history.json')
zookeeper_server_confs = _load_json(path_to_config + 'zookeeper-server.json')
zookeeper_service_confs = _load_json(path_to_config + 'zookeeper-service.json')
hbase_confs = _load_json(path_to_config + 'hbase-service.json')
master_confs = _load_json(path_to_config + 'hbase-master.json')
regionserver_confs = _load_json(path_to_config + 'hbase-regionserver.json')
priority_one_confs = _load_json(path_to_config + 'priority-one-confs.json')
def _prepare_value(value):
if not value:
return ""
return value.replace('\n', ' ')
def _init_configs(confs, app_target, scope):
cfgs = []
for cfg in confs:
priority = 1 if cfg['name'] in priority_one_confs else 2
c = p.Config(cfg['name'], app_target, scope, priority=priority,
default_value=_prepare_value(cfg['value']),
description=cfg['desc'], is_optional=True)
cfgs.append(c)
return cfgs
def _get_ng_plugin_configs():
cfg = []
cfg += _init_configs(hdfs_confs, 'HDFS', 'cluster')
cfg += _init_configs(namenode_confs, 'NAMENODE', 'node')
cfg += _init_configs(datanode_confs, 'DATANODE', 'node')
cfg += _init_configs(secnamenode_confs, 'SECONDARYNAMENODE', 'node')
cfg += _init_configs(yarn_confs, 'YARN', 'cluster')
cfg += _init_configs(resourcemanager_confs, 'RESOURCEMANAGER', 'node')
cfg += _init_configs(nodemanager_confs, 'NODEMANAGER', 'node')
cfg += _init_configs(jobhistory_confs, 'JOBHISTORY', 'node')
cfg += _init_configs(oozie_service_confs, 'OOZIE', 'cluster')
cfg += _init_configs(oozie_role_confs, 'OOZIE', 'node')
cfg += _init_configs(hive_service_confs, 'HIVE', 'cluster')
cfg += _init_configs(hive_metastore_confs, 'HIVEMETASTORE', 'node')
cfg += _init_configs(hive_hiveserver_confs, 'HIVESERVER', 'node')
cfg += _init_configs(hive_webhcat_confs, 'WEBHCAT', 'node')
cfg += _init_configs(hue_service_confs, 'HUE', 'cluster')
cfg += _init_configs(hue_role_confs, 'HUE', 'node')
cfg += _init_configs(spark_service_confs, 'SPARK_ON_YARN', 'cluster')
cfg += _init_configs(spark_role_confs, 'SPARK_ON_YARN', 'node')
cfg += _init_configs(zookeeper_service_confs, 'ZOOKEEPER', 'cluster')
cfg += _init_configs(zookeeper_server_confs, 'ZOOKEEPER', 'node')
cfg += _init_configs(hbase_confs, 'HBASE', 'cluster')
cfg += _init_configs(master_confs, 'MASTER', 'node')
cfg += _init_configs(regionserver_confs, 'REGIONSERVER', 'node')
return cfg
def get_plugin_configs():
cluster_wide = _get_cluster_plugin_configs()
ng_wide = _get_ng_plugin_configs()
return cluster_wide + ng_wide
def _get_config_value(cluster, key):
return cluster.cluster_configs.get(
'general', {}).get(key.name, key.default_value)
def get_cdh5_repo_url(cluster):
return _get_config_value(cluster, CDH5_REPO_URL)
def get_cdh5_key_url(cluster):
return _get_config_value(cluster, CDH5_REPO_KEY_URL)
def get_cm5_repo_url(cluster):
return _get_config_value(cluster, CM5_REPO_URL)
def get_cm5_key_url(cluster):
return _get_config_value(cluster, CM5_REPO_KEY_URL)
def is_swift_enabled(cluster):
return _get_config_value(cluster, ENABLE_SWIFT)
def get_swift_lib_url(cluster):
return _get_config_value(cluster, SWIFT_LIB_URL)
def get_extjs_lib_url(cluster):
return _get_config_value(cluster, EXTJS_LIB_URL)

53
sahara/plugins/cdh/v5/db_helper.py Normal file
View File

@ -0,0 +1,53 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import uuid
import six
from sahara import conductor
from sahara import context
from sahara.utils import files
conductor = conductor.API
def get_hive_db_password(cluster):
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, cluster.id)
passwd = cluster.extra.get('hive_db_password') if cluster.extra else None
if passwd:
return passwd
passwd = six.text_type(uuid.uuid4())
extra = cluster.extra.to_dict() if cluster.extra else {}
extra['hive_db_password'] = passwd
cluster = conductor.cluster_update(ctx, cluster, {'extra': extra})
return passwd
def create_hive_database(cluster, remote):
db_password = get_hive_db_password(cluster)
create_db_script = files.get_file_text(
'plugins/cdh/v5/resources/create_hive_db.sql')
create_db_script = create_db_script % db_password
script_name = 'create_hive_db.sql'
remote.write_file_to(script_name, create_db_script)
psql_cmd = ('PGPASSWORD=$(sudo head -1 /var/lib/cloudera-scm-server-db'
'/data/generated_password.txt) psql -U cloudera-scm '
'-h localhost -p 7432 -d scm -f %s') % script_name
remote.execute_command(psql_cmd)
remote.execute_command('rm %s' % script_name)

191
sahara/plugins/cdh/v5/deploy.py Normal file
View File

@ -0,0 +1,191 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from sahara.plugins.cdh import commands as cmd
from sahara.plugins.cdh.v5 import cloudera_utils as cu
from sahara.plugins import utils as gu
PACKAGES = [
'cloudera-manager-agent',
'cloudera-manager-daemons',
'cloudera-manager-server',
'cloudera-manager-server-db-2',
'hadoop-hdfs-datanode',
'hadoop-hdfs-namenode',
'hadoop-hdfs-secondarynamenode',
'hadoop-mapreduce',
'hadoop-mapreduce-historyserver',
'hadoop-yarn-nodemanager',
'hadoop-yarn-resourcemanager',
'hbase',
'hive-hcatalog',
'hive-metastore',
'hive-server2',
'hive-webhcat-server',
'hue',
'ntp',
'oozie',
'oracle-j2sdk1.7',
'spark-core',
'unzip',
'zookeeper'
]
CU = cu.ClouderaUtilsV5()
def configure_cluster(cluster):
instances = gu.get_instances(cluster)
if not cmd.is_pre_installed_cdh(CU.pu.get_manager(cluster).remote()):
CU.pu.configure_os(instances)
CU.pu.install_packages(instances, PACKAGES)
CU.pu.start_cloudera_agents(instances)
CU.pu.start_cloudera_manager(cluster)
CU.await_agents(instances)
CU.create_mgmt_service(cluster)
CU.create_services(cluster)
CU.configure_services(cluster)
CU.configure_instances(instances, cluster)
CU.deploy_configs(cluster)
CU.pu.configure_swift(cluster)
def scale_cluster(cluster, instances):
if not instances:
return
if not cmd.is_pre_installed_cdh(instances[0].remote()):
CU.pu.configure_os(instances)
CU.pu.install_packages(instances, PACKAGES)
CU.pu.start_cloudera_agents(instances)
CU.await_agents(instances)
for instance in instances:
CU.configure_instance(instance)
CU.update_configs(instance)
if 'HDFS_DATANODE' in instance.node_group.node_processes:
CU.refresh_nodes(cluster, 'DATANODE', CU.HDFS_SERVICE_NAME)
CU.pu.configure_swift_to_inst(instance)
if 'HDFS_DATANODE' in instance.node_group.node_processes:
hdfs = CU.get_service_by_role('DATANODE', instance=instance)
CU.start_roles(hdfs, CU.pu.get_role_name(instance, 'DATANODE'))
if 'YARN_NODEMANAGER' in instance.node_group.node_processes:
yarn = CU.get_service_by_role('NODEMANAGER', instance=instance)
CU.start_roles(yarn, CU.pu.get_role_name(instance, 'NODEMANAGER'))
def decommission_cluster(cluster, instances):
dns = []
nms = []
for i in instances:
if 'HDFS_DATANODE' in i.node_group.node_processes:
dns.append(CU.pu.get_role_name(i, 'DATANODE'))
if 'YARN_NODEMANAGER' in i.node_group.node_processes:
nms.append(CU.pu.get_role_name(i, 'NODEMANAGER'))
if dns:
CU.decommission_nodes(cluster, 'DATANODE', dns)
if nms:
CU.decommission_nodes(cluster, 'NODEMANAGER', nms)
CU.delete_instances(cluster, instances)
CU.refresh_nodes(cluster, 'DATANODE', CU.HDFS_SERVICE_NAME)
CU.refresh_nodes(cluster, 'NODEMANAGER', CU.YARN_SERVICE_NAME)
def start_cluster(cluster):
cm_cluster = CU.get_cloudera_cluster(cluster)
if len(CU.pu.get_zookeepers(cluster)) > 0:
zookeeper = cm_cluster.get_service(CU.ZOOKEEPER_SERVICE_NAME)
CU.start_service(zookeeper)
hdfs = cm_cluster.get_service(CU.HDFS_SERVICE_NAME)
CU.format_namenode(hdfs)
CU.start_service(hdfs)
CU.create_hdfs_tmp(hdfs)
yarn = cm_cluster.get_service(CU.YARN_SERVICE_NAME)
CU.create_yarn_job_history_dir(yarn)
CU.start_service(yarn)
if CU.pu.get_hive_metastore(cluster):
hive = cm_cluster.get_service(CU.HIVE_SERVICE_NAME)
CU.pu.put_hive_hdfs_xml(cluster)
CU.pu.configure_hive(cluster)
CU.pu.create_hive_hive_directory(cluster)
CU.create_hive_metastore_db(hive)
CU.create_hive_dirs(hive)
CU.start_service(hive)
oozie_inst = CU.pu.get_oozie(cluster)
if oozie_inst:
CU.pu.install_extjs(cluster)
oozie = cm_cluster.get_service(CU.OOZIE_SERVICE_NAME)
CU.create_oozie_db(oozie)
CU.install_oozie_sharelib(oozie)
CU.start_service(oozie)
if CU.pu.get_hue(cluster):
hue = cm_cluster.get_service(CU.HUE_SERVICE_NAME)
CU.start_service(hue)
if CU.pu.get_spark_historyserver(cluster):
CU.pu.configure_spark(cluster)
spark = cm_cluster.get_service(CU.SPARK_SERVICE_NAME)
CU.start_service(spark)
if CU.pu.get_hbase_master(cluster):
hbase = cm_cluster.get_service(CU.HBASE_SERVICE_NAME)
CU.create_hbase_root(hbase)
CU.start_service(hbase)
def get_open_ports(node_group):
ports = [9000] # for CM agent
ports_map = {
'CLOUDERA_MANAGER': [7180, 7182, 7183, 7432, 7184, 8084, 8086, 10101,
9997, 9996, 8087, 9998, 9999, 8085, 9995, 9994],
'HDFS_NAMENODE': [8020, 8022, 50070, 50470],
'HDFS_SECONDARYNAMENODE': [50090, 50495],
'HDFS_DATANODE': [50010, 1004, 50075, 1006, 50020],
'YARN_RESOURCEMANAGER': [8030, 8031, 8032, 8033, 8088],
'YARN_NODEMANAGER': [8040, 8041, 8042],
'YARN_JOBHISTORY': [10020, 19888],
'HIVE_METASTORE': [9083],
'HIVE_SERVER2': [10000],
'HUE_SERVER': [8888],
'OOZIE_SERVER': [11000, 11001],
'SPARK_YARN_HISTORY_SERVER': [18088],
'ZOOKEEPER_SERVER': [2181, 3181, 4181, 9010],
'HBASE_MASTER': [60000],
'HBASE_REGIONSERVER': [60020]
}
for process in node_group.node_processes:
if process in ports_map:
ports.extend(ports_map[process])
return ports

12
sahara/plugins/cdh/edp_engine.py → sahara/plugins/cdh/v5/edp_engine.py
View File

@ -13,12 +13,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from sahara.plugins.cdh import utils as cu
from sahara.plugins.cdh.v5 import cloudera_utils as cu
from sahara.plugins import exceptions as ex
from sahara.plugins import utils as u
from sahara.service.edp import hdfs_helper
from sahara.service.edp.oozie import engine as edp_engine
CU = cu.ClouderaUtilsV5()
class EdpOozieEngine(edp_engine.OozieJobEngine):
@ -29,19 +31,19 @@ class EdpOozieEngine(edp_engine.OozieJobEngine):
hdfs_helper.create_dir_hadoop2(remote, dir_name, self.get_hdfs_user())
def get_oozie_server_uri(self, cluster):
oozie_ip = cu.get_oozie(cluster).management_ip
oozie_ip = CU.pu.get_oozie(cluster).management_ip
return 'http://%s:11000/oozie' % oozie_ip
def get_name_node_uri(self, cluster):
namenode_ip = cu.get_namenode(cluster).fqdn()
namenode_ip = CU.pu.get_namenode(cluster).fqdn()
return 'hdfs://%s:8020' % namenode_ip
def get_resource_manager_uri(self, cluster):
resourcemanager_ip = cu.get_resourcemanager(cluster).fqdn()
resourcemanager_ip = CU.pu.get_resourcemanager(cluster).fqdn()
return '%s:8032' % resourcemanager_ip
def get_oozie_server(self, cluster):
return cu.get_oozie(cluster)
return CU.pu.get_oozie(cluster)
def validate_job_execution(self, cluster, job, data):
oo_count = u.get_instances_count(cluster, 'OOZIE_SERVER')

55
sahara/plugins/cdh/v5/plugin_utils.py Normal file
View File

@ -0,0 +1,55 @@
# Copyright (c) 2014 Intel Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from sahara.plugins.cdh import plugin_utils as pu
from sahara.plugins.cdh.v5 import config_helper as c_helper
from sahara.plugins.cdh.v5 import db_helper
class PluginUtilsV5(pu.AbstractPluginUtils):
def __init__(self):
self.c_helper = c_helper
self.db_helper = db_helper
def configure_spark(self, cluster):
spark = self.get_spark_historyserver(cluster)
with spark.remote() as r:
r.execute_command(
'sudo su - -c "hdfs dfs -mkdir -p '
'/user/spark/applicationHistory" hdfs')
r.execute_command(
'sudo su - -c "hdfs dfs -mkdir -p '
'/user/spark/share/lib" hdfs')
r.execute_command(
'sudo su - -c "hdfs dfs -put /usr/lib/spark/assembly/lib/'
'spark-assembly-hadoop* '
'/user/spark/share/lib/spark-assembly.jar" hdfs')
r.execute_command(
'sudo su - -c "hdfs dfs -chown -R '
'spark:spark /user/spark" hdfs')
r.execute_command(
'sudo su - -c "hdfs dfs -chmod 0751 /user/spark" hdfs')
r.execute_command(
'sudo su - -c "hdfs dfs -chmod 1777 /user/spark/'
'applicationHistory" hdfs')
def create_hive_hive_directory(self, cluster):
# Hive requires /tmp/hive-hive directory
namenode = self.get_namenode(cluster)
with namenode.remote() as r:
r.execute_command(
'sudo su - -c "hadoop fs -mkdir -p /tmp/hive-hive" hdfs')
r.execute_command(
'sudo su - -c "hadoop fs -chown hive /tmp/hive-hive" hdfs')

110
sahara/plugins/cdh/v5/resources/cdh_config.py Normal file
View File

@ -0,0 +1,110 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from cm_api import api_client
# -- cm config --
cm_address = 'localhost'
cm_port = 7180
cm_username = 'admin'
cm_password = 'admin'
hdfs_service_name = 'hdfs01'
yarn_service_name = 'yarn01'
oozie_service_name = 'oozie01'
hive_service_name = 'hive01'
hue_service_name = 'hue01'
spark_service_name = 'spark_on_yarn01'
zookeeper_service_name = 'zookeeper01'
hbase_service_name = 'hbase01'
def get_cm_api():
return api_client.ApiResource(cm_address, server_port=cm_port,
username=cm_username, password=cm_password)
def get_cluster(api):
return api.get_all_clusters()[0]
def process_service(service, service_name):
for role_cfgs in service.get_all_role_config_groups():
role_cm_cfg = role_cfgs.get_config(view='full')
role_cfg = parse_config(role_cm_cfg)
role_name = role_cfgs.displayName.split(' ')[0].lower()
write_cfg(role_cfg, '%s-%s.json' % (service_name, role_name))
service_cm_cfg = service.get_config(view='full')[0]
service_cfg = parse_config(service_cm_cfg)
write_cfg(service_cfg, '%s-service.json' % service_name)
def parse_config(config):
cfg = []
for name, value in config.iteritems():
p = {
'name': value.name,
'value': value.default,
'display_name': value.displayName,
'desc': value.description
}
cfg.append(p)
return cfg
def write_cfg(cfg, file_name):
to_write = json.dumps(cfg, sort_keys=True, indent=4,
separators=(',', ': '))
with open(file_name, 'w') as f:
f.write(to_write)
def main():
client = get_cm_api()
cluster = get_cluster(client)
hdfs = cluster.get_service(hdfs_service_name)
process_service(hdfs, 'hdfs')
yarn = cluster.get_service(yarn_service_name)
process_service(yarn, 'yarn')
oozie = cluster.get_service(oozie_service_name)
process_service(oozie, 'oozie')
hive = cluster.get_service(hive_service_name)
process_service(hive, 'hive')
hue = cluster.get_service(hue_service_name)
process_service(hue, 'hue')
spark = cluster.get_service(spark_service_name)
process_service(spark, 'spark')
zookeeper = cluster.get_service(zookeeper_service_name)
process_service(zookeeper, 'zookeeper')
hbase = cluster.get_service(hbase_service_name)
process_service(hbase, 'hbase')
if __name__ == '__main__':
main()

0
sahara/plugins/cdh/resources/cdh_config.sh → sahara/plugins/cdh/v5/resources/cdh_config.sh
View File

0
sahara/plugins/cdh/resources/create_hive_db.sql → sahara/plugins/cdh/v5/resources/create_hive_db.sql
View File

0
sahara/plugins/cdh/resources/hbase-master.json → sahara/plugins/cdh/v5/resources/hbase-master.json
View File

0
sahara/plugins/cdh/resources/hbase-regionserver.json → sahara/plugins/cdh/v5/resources/hbase-regionserver.json
View File

0
sahara/plugins/cdh/resources/hbase-service.json → sahara/plugins/cdh/v5/resources/hbase-service.json
View File

0
sahara/plugins/cdh/resources/hdfs-balancer.json → sahara/plugins/cdh/v5/resources/hdfs-balancer.json
View File

0
sahara/plugins/cdh/resources/hdfs-datanode.json → sahara/plugins/cdh/v5/resources/hdfs-datanode.json
View File

0
sahara/plugins/cdh/resources/hdfs-failover.json → sahara/plugins/cdh/v5/resources/hdfs-failover.json
View File

0
sahara/plugins/cdh/resources/hdfs-gateway.json → sahara/plugins/cdh/v5/resources/hdfs-gateway.json
View File

0
sahara/plugins/cdh/resources/hdfs-httpfs.json → sahara/plugins/cdh/v5/resources/hdfs-httpfs.json
View File

0
sahara/plugins/cdh/resources/hdfs-journalnode.json → sahara/plugins/cdh/v5/resources/hdfs-journalnode.json
View File

0
sahara/plugins/cdh/resources/hdfs-namenode.json → sahara/plugins/cdh/v5/resources/hdfs-namenode.json
View File

0
sahara/plugins/cdh/resources/hdfs-nfs.json → sahara/plugins/cdh/v5/resources/hdfs-nfs.json
View File

0
sahara/plugins/cdh/resources/hdfs-secondarynamenode.json → sahara/plugins/cdh/v5/resources/hdfs-secondarynamenode.json
View File

0
sahara/plugins/cdh/resources/hdfs-service.json → sahara/plugins/cdh/v5/resources/hdfs-service.json
View File

0
sahara/plugins/cdh/resources/hive-hiveserver2.json → sahara/plugins/cdh/v5/resources/hive-hiveserver2.json
View File

0
sahara/plugins/cdh/resources/hive-metastore.json → sahara/plugins/cdh/v5/resources/hive-metastore.json
View File

0
sahara/plugins/cdh/resources/hive-service.json → sahara/plugins/cdh/v5/resources/hive-service.json
View File

0
sahara/plugins/cdh/resources/hive-webhcat.json → sahara/plugins/cdh/v5/resources/hive-webhcat.json
View File

0
sahara/plugins/cdh/resources/hue-hue.json → sahara/plugins/cdh/v5/resources/hue-hue.json
View File

0
sahara/plugins/cdh/resources/hue-service.json → sahara/plugins/cdh/v5/resources/hue-service.json
View File

0
sahara/plugins/cdh/resources/oozie-oozie.json → sahara/plugins/cdh/v5/resources/oozie-oozie.json
View File

0
sahara/plugins/cdh/resources/oozie-service.json → sahara/plugins/cdh/v5/resources/oozie-service.json
View File

0
sahara/plugins/cdh/resources/priority-one-confs.json → sahara/plugins/cdh/v5/resources/priority-one-confs.json
View File

0
sahara/plugins/cdh/resources/spark-history.json → sahara/plugins/cdh/v5/resources/spark-history.json
View File

0
sahara/plugins/cdh/resources/spark-service.json → sahara/plugins/cdh/v5/resources/spark-service.json
View File

0
sahara/plugins/cdh/resources/yarn-gateway.json → sahara/plugins/cdh/v5/resources/yarn-gateway.json
View File

0
sahara/plugins/cdh/resources/yarn-jobhistory.json → sahara/plugins/cdh/v5/resources/yarn-jobhistory.json
View File

0
sahara/plugins/cdh/resources/yarn-nodemanager.json → sahara/plugins/cdh/v5/resources/yarn-nodemanager.json
View File

0
sahara/plugins/cdh/resources/yarn-resourcemanager.json → sahara/plugins/cdh/v5/resources/yarn-resourcemanager.json
View File

0
sahara/plugins/cdh/resources/yarn-service.json → sahara/plugins/cdh/v5/resources/yarn-service.json
View File

0
sahara/plugins/cdh/resources/zookeeper-server.json → sahara/plugins/cdh/v5/resources/zookeeper-server.json
View File

0
sahara/plugins/cdh/resources/zookeeper-service.json → sahara/plugins/cdh/v5/resources/zookeeper-service.json
View File

177
sahara/plugins/cdh/v5/validation.py Normal file
View File

@ -0,0 +1,177 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_log import log as logging
from sahara.i18n import _
from sahara.plugins.cdh.v5 import plugin_utils as pu
from sahara.plugins import exceptions as ex
from sahara.plugins import utils as u
from sahara.utils import general as gu
LOG = logging.getLogger(__name__)
PU = pu.PluginUtilsV5()
def validate_cluster_creating(cluster):
mng_count = _get_inst_count(cluster, 'CLOUDERA_MANAGER')
if mng_count != 1:
raise ex.InvalidComponentCountException('CLOUDERA_MANAGER',
1, mng_count)
nn_count = _get_inst_count(cluster, 'HDFS_NAMENODE')
if nn_count != 1:
raise ex.InvalidComponentCountException('HDFS_NAMENODE', 1, nn_count)
snn_count = _get_inst_count(cluster, 'HDFS_SECONDARYNAMENODE')
if snn_count != 1:
raise ex.InvalidComponentCountException('HDFS_SECONDARYNAMENODE', 1,
snn_count)
rm_count = _get_inst_count(cluster, 'YARN_RESOURCEMANAGER')
if rm_count not in [0, 1]:
raise ex.InvalidComponentCountException('YARN_RESOURCEMANAGER',
_('0 or 1'), rm_count)
hs_count = _get_inst_count(cluster, 'YARN_JOBHISTORY')
if hs_count not in [0, 1]:
raise ex.InvalidComponentCountException('YARN_JOBHISTORY', _('0 or 1'),
hs_count)
if rm_count > 0 and hs_count < 1:
raise ex.RequiredServiceMissingException(
'YARN_JOBHISTORY', required_by='YARN_RESOURCEMANAGER')
nm_count = _get_inst_count(cluster, 'YARN_NODEMANAGER')
if rm_count == 0:
if nm_count > 0:
raise ex.RequiredServiceMissingException(
'YARN_RESOURCEMANAGER', required_by='YARN_NODEMANAGER')
oo_count = _get_inst_count(cluster, 'OOZIE_SERVER')
dn_count = _get_inst_count(cluster, 'HDFS_DATANODE')
if oo_count not in [0, 1]:
raise ex.InvalidComponentCountException('OOZIE_SERVER', _('0 or 1'),
oo_count)
if oo_count == 1:
if dn_count < 1:
raise ex.RequiredServiceMissingException(
'HDFS_DATANODE', required_by='OOZIE_SERVER')
if nm_count < 1:
raise ex.RequiredServiceMissingException(
'YARN_NODEMANAGER', required_by='OOZIE_SERVER')
if hs_count != 1:
raise ex.RequiredServiceMissingException(
'YARN_JOBHISTORY', required_by='OOZIE_SERVER')
hms_count = _get_inst_count(cluster, 'HIVE_METASTORE')
hvs_count = _get_inst_count(cluster, 'HIVE_SERVER2')
whc_count = _get_inst_count(cluster, 'HIVE_WEBHCAT')
if hms_count and rm_count < 1:
raise ex.RequiredServiceMissingException(
'YARN_RESOURCEMANAGER', required_by='HIVE_METASTORE')
if hms_count and not hvs_count:
raise ex.RequiredServiceMissingException(
'HIVE_SERVER2', required_by='HIVE_METASTORE')
if hvs_count and not hms_count:
raise ex.RequiredServiceMissingException(
'HIVE_METASTORE', required_by='HIVE_SERVER2')
if whc_count and not hms_count:
raise ex.RequiredServiceMissingException(
'HIVE_METASTORE', required_by='WEBHCAT')
hue_count = _get_inst_count(cluster, 'HUE_SERVER')
if hue_count not in [0, 1]:
raise ex.InvalidComponentCountException('HUE_SERVER', _('0 or 1'),
hue_count)
shs_count = _get_inst_count(cluster, 'SPARK_YARN_HISTORY_SERVER')
if shs_count not in [0, 1]:
raise ex.InvalidComponentCountException('SPARK_YARN_HISTORY_SERVER',
_('0 or 1'), shs_count)
if shs_count and not rm_count:
raise ex.RequiredServiceMissingException(
'YARN_RESOURCEMANAGER', required_by='SPARK_YARN_HISTORY_SERVER')
if oo_count < 1 and hue_count:
raise ex.RequiredServiceMissingException(
'OOZIE_SERVER', required_by='HUE_SERVER')
if hms_count < 1 and hue_count:
raise ex.RequiredServiceMissingException(
'HIVE_METASTORE', required_by='HUE_SERVER')
hbm_count = _get_inst_count(cluster, 'HBASE_MASTER')
hbr_count = _get_inst_count(cluster, 'HBASE_REGIONSERVER')
zk_count = _get_inst_count(cluster, 'ZOOKEEPER_SERVER')
if hbm_count >= 1:
if zk_count < 1:
raise ex.RequiredServiceMissingException('ZOOKEEPER',
required_by='HBASE')
if hbr_count < 1:
raise ex.InvalidComponentCountException('HBASE_REGIONSERVER',
_('at least 1'), hbr_count)
elif hbr_count >= 1:
raise ex.InvalidComponentCountException('HBASE_MASTER',
_('at least 1'), hbm_count)
def validate_additional_ng_scaling(cluster, additional):
rm = PU.get_resourcemanager(cluster)
scalable_processes = _get_scalable_processes()
for ng_id in additional:
ng = gu.get_by_id(cluster.node_groups, ng_id)
if not set(ng.node_processes).issubset(scalable_processes):
msg = _("CDH plugin cannot scale nodegroup with processes: "
"%(processes)s")
raise ex.NodeGroupCannotBeScaled(
ng.name, msg % {'processes': ' '.join(ng.node_processes)})
if not rm and 'YARN_NODEMANAGER' in ng.node_processes:
msg = _("CDH plugin cannot scale node group with processes "
"which have no master-processes run in cluster")
raise ex.NodeGroupCannotBeScaled(ng.name, msg)
def validate_existing_ng_scaling(cluster, existing):
scalable_processes = _get_scalable_processes()
dn_to_delete = 0
for ng in cluster.node_groups:
if ng.id in existing:
if ng.count > existing[ng.id] and "datanode" in ng.node_processes:
dn_to_delete += ng.count - existing[ng.id]
if not set(ng.node_processes).issubset(scalable_processes):
msg = _("CDH plugin cannot scale nodegroup with processes: "
"%(processes)s")
raise ex.NodeGroupCannotBeScaled(
ng.name, msg % {'processes': ' '.join(ng.node_processes)})
def _get_scalable_processes():
return ['HDFS_DATANODE', 'YARN_NODEMANAGER']
def _get_inst_count(cluster, process):
return sum([ng.count for ng in u.get_node_groups(cluster, process)])

111
sahara/plugins/cdh/v5/versionhandler.py Normal file
View File

@ -0,0 +1,111 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
from oslo_log import log as logging
from sahara import conductor
from sahara import context
from sahara.plugins.cdh import abstractversionhandler as avm
from sahara.plugins.cdh.v5 import cloudera_utils as cu
from sahara.plugins.cdh.v5 import config_helper as c_helper
from sahara.plugins.cdh.v5 import deploy as dp
from sahara.plugins.cdh.v5 import edp_engine
from sahara.plugins.cdh.v5 import validation as vl
conductor = conductor.API
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
CU = cu.ClouderaUtilsV5()
class VersionHandler(avm.AbstractVersionHandler):
def get_plugin_configs(self):
return c_helper.get_plugin_configs()
def get_node_processes(self):
return {
"CLOUDERA": ['CLOUDERA_MANAGER'],
"HDFS": [],
"NAMENODE": ['HDFS_NAMENODE'],
"DATANODE": ['HDFS_DATANODE'],
"SECONDARYNAMENODE": ['HDFS_SECONDARYNAMENODE'],
"YARN": [],
"RESOURCEMANAGER": ['YARN_RESOURCEMANAGER'],
"NODEMANAGER": ['YARN_NODEMANAGER'],
"JOBHISTORY": ['YARN_JOBHISTORY'],
"OOZIE": ['OOZIE_SERVER'],
"HIVE": [],
"HIVESERVER": ['HIVE_SERVER2'],
"HIVEMETASTORE": ['HIVE_METASTORE'],
"WEBHCAT": ['HIVE_WEBHCAT'],
"HUE": ['HUE_SERVER'],
"SPARK_ON_YARN": ['SPARK_YARN_HISTORY_SERVER'],
"ZOOKEEPER": ['ZOOKEEPER_SERVER'],
"HBASE": [],
"MASTER": ['HBASE_MASTER'],
"REGIONSERVER": ['HBASE_REGIONSERVER'],
}
def validate(self, cluster):
CU.validate_cm_api_libs()
vl.validate_cluster_creating(cluster)
def configure_cluster(self, cluster):
dp.configure_cluster(cluster)
def start_cluster(self, cluster):
dp.start_cluster(cluster)
self._set_cluster_info(cluster)
def decommission_nodes(self, cluster, instances):
dp.decommission_cluster(cluster, instances)
def validate_scaling(self, cluster, existing, additional):
vl.validate_existing_ng_scaling(cluster, existing)
vl.validate_additional_ng_scaling(cluster, additional)
def scale_cluster(self, cluster, instances):
dp.scale_cluster(cluster, instances)
def _set_cluster_info(self, cluster):
mng = CU.pu.get_manager(cluster)
info = {
'Cloudera Manager': {
'Web UI': 'http://%s:7180' % mng.management_ip,
'Username': 'admin',
'Password': 'admin'
}
}
hue = CU.pu.get_hue(cluster)
if hue:
info['Hue Dashboard'] = {
'Web UI': 'http://%s:8888' % hue.management_ip
}
ctx = context.ctx()
conductor.cluster_update(ctx, cluster, {'info': info})
def get_edp_engine(self, cluster, job_type):
if job_type in edp_engine.EdpOozieEngine.get_supported_job_types():
return edp_engine.EdpOozieEngine(cluster)
return None
def get_open_ports(self, node_group):
return dp.get_open_ports(node_group)

0
sahara/plugins/cdh/v5_3_0/__init__.py Normal file
View File

413
sahara/plugins/cdh/v5_3_0/cloudera_utils.py Normal file
View File

@ -0,0 +1,413 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# cm_api client is not present in OS requirements
try:
from cm_api import api_client
from cm_api.endpoints import services
except ImportError:
api_client = None
services = None
import six
from sahara.i18n import _
from sahara.plugins.cdh import cloudera_utils as cu
from sahara.plugins.cdh.v5_3_0 import config_helper as c_helper
from sahara.plugins.cdh.v5_3_0 import plugin_utils as pu
from sahara.plugins.cdh.v5_3_0 import validation as v
from sahara.swift import swift_helper
from sahara.utils import xmlutils
CM_API_PORT = 7180
HDFS_SERVICE_TYPE = 'HDFS'
YARN_SERVICE_TYPE = 'YARN'
OOZIE_SERVICE_TYPE = 'OOZIE'
HIVE_SERVICE_TYPE = 'HIVE'
HUE_SERVICE_TYPE = 'HUE'
SPARK_SERVICE_TYPE = 'SPARK_ON_YARN'
ZOOKEEPER_SERVICE_TYPE = 'ZOOKEEPER'
HBASE_SERVICE_TYPE = 'HBASE'
FLUME_SERVICE_TYPE = 'FLUME'
SENTRY_SERVICE_TYPE = 'SENTRY'
SOLR_SERVICE_TYPE = 'SOLR'
SQOOP_SERVICE_TYPE = 'SQOOP'
KS_INDEXER_SERVICE_TYPE = 'KS_INDEXER'
IMPALA_SERVICE_TYPE = 'IMPALA'
def _merge_dicts(a, b):
res = {}
def update(cfg):
for service, configs in six.iteritems(cfg):
if not res.get(service):
res[service] = {}
res[service].update(configs)
update(a)
update(b)
return res
class ClouderaUtilsV530(cu.ClouderaUtils):
FLUME_SERVICE_NAME = 'flume01'
SOLR_SERVICE_NAME = 'solr01'
SQOOP_SERVICE_NAME = 'sqoop01'
KS_INDEXER_SERVICE_NAME = 'ks_indexer01'
IMPALA_SERVICE_NAME = 'impala01'
SENTRY_SERVICE_NAME = 'sentry01'
CM_API_VERSION = 8
def __init__(self):
cu.ClouderaUtils.__init__(self)
self.pu = pu.PluginUtilsV530()
def get_api_client(self, cluster):
manager_ip = self.pu.get_manager(cluster).management_ip
return api_client.ApiResource(manager_ip,
username=self.CM_DEFAULT_USERNAME,
password=self.CM_DEFAULT_PASSWD,
version=self.CM_API_VERSION)
def get_service_by_role(self, process, cluster=None, instance=None):
cm_cluster = None
if cluster:
cm_cluster = self.get_cloudera_cluster(cluster)
elif instance:
cm_cluster = self.get_cloudera_cluster(instance.node_group.cluster)
else:
raise ValueError(_("'cluster' or 'instance' argument missed"))
if process in ['AGENT']:
return cm_cluster.get_service(self.FLUME_SERVICE_NAME)
elif process in ['SENTRY_SERVER']:
return cm_cluster.get_service(self.SENTRY_SERVICE_NAME)
elif process in ['SQOOP_SERVER']:
return cm_cluster.get_service(self.SQOOP_SERVICE_NAME)
elif process in ['SOLR_SERVER']:
return cm_cluster.get_service(self.SOLR_SERVICE_NAME)
elif process in ['HBASE_INDEXER']:
return cm_cluster.get_service(self.KS_INDEXER_SERVICE_NAME)
elif process in ['CATALOGSERVER', 'STATESTORE', 'IMPALAD', 'LLAMA']:
return cm_cluster.get_service(self.IMPALA_SERVICE_NAME)
else:
return super(ClouderaUtilsV530, self).get_service_by_role(
process, cluster, instance)
@cu.cloudera_cmd
def first_run(self, cluster):
cm_cluster = self.get_cloudera_cluster(cluster)
yield cm_cluster.first_run()
def create_services(self, cluster):
api = self.get_api_client(cluster)
fullversion = ('5.0.0' if cluster.hadoop_version == '5'
else cluster.hadoop_version)
cm_cluster = api.create_cluster(cluster.name,
fullVersion=fullversion)
if len(self.pu.get_zookeepers(cluster)) > 0:
cm_cluster.create_service(self.ZOOKEEPER_SERVICE_NAME,
ZOOKEEPER_SERVICE_TYPE)
cm_cluster.create_service(self.HDFS_SERVICE_NAME, HDFS_SERVICE_TYPE)
cm_cluster.create_service(self.YARN_SERVICE_NAME, YARN_SERVICE_TYPE)
cm_cluster.create_service(self.OOZIE_SERVICE_NAME, OOZIE_SERVICE_TYPE)
if self.pu.get_hive_metastore(cluster):
cm_cluster.create_service(self.HIVE_SERVICE_NAME,
HIVE_SERVICE_TYPE)
if self.pu.get_hue(cluster):
cm_cluster.create_service(self.HUE_SERVICE_NAME, HUE_SERVICE_TYPE)
if self.pu.get_spark_historyserver(cluster):
cm_cluster.create_service(self.SPARK_SERVICE_NAME,
SPARK_SERVICE_TYPE)
if self.pu.get_hbase_master(cluster):
cm_cluster.create_service(self.HBASE_SERVICE_NAME,
HBASE_SERVICE_TYPE)
if len(self.pu.get_flumes(cluster)) > 0:
cm_cluster.create_service(self.FLUME_SERVICE_NAME,
FLUME_SERVICE_TYPE)
if self.pu.get_sentry(cluster):
cm_cluster.create_service(self.SENTRY_SERVICE_NAME,
SENTRY_SERVICE_TYPE)
if len(self.pu.get_solrs(cluster)) > 0:
cm_cluster.create_service(self.SOLR_SERVICE_NAME,
SOLR_SERVICE_TYPE)
if self.pu.get_sqoop(cluster):
cm_cluster.create_service(self.SQOOP_SERVICE_NAME,
SQOOP_SERVICE_TYPE)
if len(self.pu.get_hbase_indexers(cluster)) > 0:
cm_cluster.create_service(self.KS_INDEXER_SERVICE_NAME,
KS_INDEXER_SERVICE_TYPE)
if self.pu.get_catalogserver(cluster):
cm_cluster.create_service(self.IMPALA_SERVICE_NAME,
IMPALA_SERVICE_TYPE)
def configure_services(self, cluster):
cm_cluster = self.get_cloudera_cluster(cluster)
if len(self.pu.get_zookeepers(cluster)) > 0:
zookeeper = cm_cluster.get_service(self.ZOOKEEPER_SERVICE_NAME)
zookeeper.update_config(self._get_configs(ZOOKEEPER_SERVICE_TYPE,
cluster=cluster))
hdfs = cm_cluster.get_service(self.HDFS_SERVICE_NAME)
hdfs.update_config(self._get_configs(HDFS_SERVICE_TYPE,
cluster=cluster))
yarn = cm_cluster.get_service(self.YARN_SERVICE_NAME)
yarn.update_config(self._get_configs(YARN_SERVICE_TYPE,
cluster=cluster))
oozie = cm_cluster.get_service(self.OOZIE_SERVICE_NAME)
oozie.update_config(self._get_configs(OOZIE_SERVICE_TYPE,
cluster=cluster))
if self.pu.get_hive_metastore(cluster):
hive = cm_cluster.get_service(self.HIVE_SERVICE_NAME)
hive.update_config(self._get_configs(HIVE_SERVICE_TYPE,
cluster=cluster))
if self.pu.get_hue(cluster):
hue = cm_cluster.get_service(self.HUE_SERVICE_NAME)
hue.update_config(self._get_configs(HUE_SERVICE_TYPE,
cluster=cluster))
if self.pu.get_spark_historyserver(cluster):
spark = cm_cluster.get_service(self.SPARK_SERVICE_NAME)
spark.update_config(self._get_configs(SPARK_SERVICE_TYPE,
cluster=cluster))
if self.pu.get_hbase_master(cluster):
hbase = cm_cluster.get_service(self.HBASE_SERVICE_NAME)
hbase.update_config(self._get_configs(HBASE_SERVICE_TYPE,
cluster=cluster))
if len(self.pu.get_flumes(cluster)) > 0:
flume = cm_cluster.get_service(self.FLUME_SERVICE_NAME)
flume.update_config(self._get_configs(FLUME_SERVICE_TYPE,
cluster=cluster))
if self.pu.get_sentry(cluster):
sentry = cm_cluster.get_service(self.SENTRY_SERVICE_NAME)
sentry.update_config(self._get_configs(SENTRY_SERVICE_TYPE,
cluster=cluster))
if len(self.pu.get_solrs(cluster)) > 0:
solr = cm_cluster.get_service(self.SOLR_SERVICE_NAME)
solr.update_config(self._get_configs(SOLR_SERVICE_TYPE,
cluster=cluster))
if self.pu.get_sqoop(cluster):
sqoop = cm_cluster.get_service(self.SQOOP_SERVICE_NAME)
sqoop.update_config(self._get_configs(SQOOP_SERVICE_TYPE,
cluster=cluster))
if len(self.pu.get_hbase_indexers(cluster)) > 0:
ks_indexer = cm_cluster.get_service(self.KS_INDEXER_SERVICE_NAME)
ks_indexer.update_config(
self._get_configs(KS_INDEXER_SERVICE_TYPE, cluster=cluster))
if self.pu.get_catalogserver(cluster):
impala = cm_cluster.get_service(self.IMPALA_SERVICE_NAME)
impala.update_config(self._get_configs(IMPALA_SERVICE_TYPE,
cluster=cluster))
def _get_configs(self, service, cluster=None, node_group=None):
def get_hadoop_dirs(mount_points, suffix):
return ','.join([x + suffix for x in mount_points])
all_confs = {}
if cluster:
zk_count = v._get_inst_count(cluster, 'ZOOKEEPER_SERVER')
hbm_count = v._get_inst_count(cluster, 'HBASE_MASTER')
snt_count = v._get_inst_count(cluster, 'SENTRY_SERVER')
ks_count = v._get_inst_count(cluster, 'KEY_VALUE_STORE_INDEXER')
imp_count = v._get_inst_count(cluster, 'IMPALA_CATALOGSERVER')
hive_count = v._get_inst_count(cluster, 'HIVE_METASTORE')
slr_count = v._get_inst_count(cluster, 'SOLR_SERVER')
sqp_count = v._get_inst_count(cluster, 'SQOOP_SERVER')
core_site_safety_valve = ''
if self.pu.c_helper.is_swift_enabled(cluster):
configs = swift_helper.get_swift_configs()
confs = dict((c['name'], c['value']) for c in configs)
core_site_safety_valve = xmlutils.create_elements_xml(confs)
all_confs = {
'HDFS': {
'zookeeper_service':
self.ZOOKEEPER_SERVICE_NAME if zk_count else '',
'dfs_block_local_path_access_user':
'impala' if imp_count else '',
'core_site_safety_valve': core_site_safety_valve
},
'HIVE': {
'mapreduce_yarn_service': self.YARN_SERVICE_NAME,
'sentry_service':
self.SENTRY_SERVICE_NAME if snt_count else '',
'zookeeper_service':
self.ZOOKEEPER_SERVICE_NAME if zk_count else ''
},
'OOZIE': {
'mapreduce_yarn_service': self.YARN_SERVICE_NAME,
'hive_service':
self.HIVE_SERVICE_NAME if hive_count else '',
'zookeeper_service':
self.ZOOKEEPER_SERVICE_NAME if zk_count else ''
},
'YARN': {
'hdfs_service': self.HDFS_SERVICE_NAME,
'zookeeper_service':
self.ZOOKEEPER_SERVICE_NAME if zk_count else ''
},
'HUE': {
'hive_service': self.HIVE_SERVICE_NAME,
'oozie_service': self.OOZIE_SERVICE_NAME,
'sentry_service':
self.SENTRY_SERVICE_NAME if snt_count else '',
'solr_service':
self.SOLR_SERVICE_NAME if slr_count else '',
'zookeeper_service':
self.ZOOKEEPER_SERVICE_NAME if zk_count else '',
'hbase_service':
self.HBASE_SERVICE_NAME if hbm_count else '',
'impala_service':
self.IMPALA_SERVICE_NAME if imp_count else '',
'sqoop_service':
self.SQOOP_SERVICE_NAME if sqp_count else ''
},
'SPARK_ON_YARN': {
'yarn_service': self.YARN_SERVICE_NAME
},
'HBASE': {
'hdfs_service': self.HDFS_SERVICE_NAME,
'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME,
'hbase_enable_indexing': 'true' if ks_count else 'false',
'hbase_enable_replication':
'true' if ks_count else 'false'
},
'FLUME': {
'hdfs_service': self.HDFS_SERVICE_NAME,
'solr_service':
self.SOLR_SERVICE_NAME if slr_count else '',
'hbase_service':
self.HBASE_SERVICE_NAME if hbm_count else ''
},
'SENTRY': {
'hdfs_service': self.HDFS_SERVICE_NAME,
'sentry_server_config_safety_valve': (
c_helper.SENTRY_IMPALA_CLIENT_SAFETY_VALVE
if imp_count else '')
},
'SOLR': {
'hdfs_service': self.HDFS_SERVICE_NAME,
'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME
},
'SQOOP': {
'mapreduce_yarn_service': self.YARN_SERVICE_NAME
},
'KS_INDEXER': {
'hbase_service': self.HBASE_SERVICE_NAME,
'solr_service': self.SOLR_SERVICE_NAME
},
'IMPALA': {
'hdfs_service': self.HDFS_SERVICE_NAME,
'hbase_service':
self.HBASE_SERVICE_NAME if hbm_count else '',
'hive_service': self.HIVE_SERVICE_NAME,
'sentry_service':
self.SENTRY_SERVICE_NAME if snt_count else '',
'zookeeper_service':
self.ZOOKEEPER_SERVICE_NAME if zk_count else ''
}
}
hive_confs = {
'HIVE': {
'hive_metastore_database_type': 'postgresql',
'hive_metastore_database_host':
self.pu.get_manager(cluster).internal_ip,
'hive_metastore_database_port': '7432',
'hive_metastore_database_password':
self.pu.db_helper.get_hive_db_password(cluster)
}
}
hue_confs = {
'HUE': {
'hue_webhdfs': self.pu.get_role_name(
self.pu.get_namenode(cluster), 'NAMENODE')
}
}
sentry_confs = {
'SENTRY': {
'sentry_server_database_type': 'postgresql',
'sentry_server_database_host':
self.pu.get_manager(cluster).internal_ip,
'sentry_server_database_port': '7432',
'sentry_server_database_password':
self.pu.db_helper.get_sentry_db_password(cluster)
}
}
all_confs = _merge_dicts(all_confs, hue_confs)
all_confs = _merge_dicts(all_confs, hive_confs)
all_confs = _merge_dicts(all_confs, sentry_confs)
all_confs = _merge_dicts(all_confs, cluster.cluster_configs)
if node_group:
paths = node_group.storage_paths()
ng_default_confs = {
'NAMENODE': {
'dfs_name_dir_list': get_hadoop_dirs(paths, '/fs/nn')
},
'SECONDARYNAMENODE': {
'fs_checkpoint_dir_list':
get_hadoop_dirs(paths, '/fs/snn')
},
'DATANODE': {
'dfs_data_dir_list': get_hadoop_dirs(paths, '/fs/dn'),
'dfs_datanode_data_dir_perm': 755,
'dfs_datanode_handler_count': 30
},
'NODEMANAGER': {
'yarn_nodemanager_local_dirs':
get_hadoop_dirs(paths, '/yarn/local')
},
'SERVER': {
'maxSessionTimeout': 60000
},
'HIVESERVER2': {
'hiveserver2_enable_impersonation':
'false' if snt_count else 'true',
'hive_hs2_config_safety_valve': (
c_helper.HIVE_SERVER2_SENTRY_SAFETY_VALVE
if snt_count else '')
},
'HIVEMETASTORE': {
'hive_metastore_config_safety_valve': (
c_helper.HIVE_METASTORE_SENTRY_SAFETY_VALVE
if snt_count else '')
}
}
ng_user_confs = self.pu.convert_process_configs(
node_group.node_configs)
all_confs = _merge_dicts(all_confs, ng_user_confs)
all_confs = _merge_dicts(all_confs, ng_default_confs)
return all_confs.get(service, {})

102
sahara/plugins/cdh/config_helper.py → sahara/plugins/cdh/v5_3_0/config_helper.py
View File

@ -18,29 +18,100 @@ import json
from sahara.plugins import provisioning as p
from sahara.utils import files as f
DEFAULT_CDH5_UBUNTU_REPO_LIST_URL = ('http://archive.cloudera.com/cdh5/ubuntu'
'/precise/amd64/cdh/cloudera.list')
CDH5_UBUNTU_REPO = ('deb [arch=amd64] http://archive.cloudera.com/cdh5'
'/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib'
'\ndeb-src http://archive.cloudera.com/cdh5/ubuntu'
'/precise/amd64/cdh precise-cdh5.3.0 contrib')
DEFAULT_CDH5_UBUNTU_REPO_KEY_URL = ('http://archive.cloudera.com/cdh5/ubuntu'
'/precise/amd64/cdh/archive.key')
DEFAULT_CM5_UBUNTU_REPO_LIST_URL = ('http://archive.cloudera.com/cm5/ubuntu'
'/precise/amd64/cm/cloudera.list')
CM5_UBUNTU_REPO = ('deb [arch=amd64] http://archive.cloudera.com/cm5'
'/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib'
'\ndeb-src http://archive.cloudera.com/cm5/ubuntu'
'/precise/amd64/cm precise-cm5.3.0 contrib')
DEFAULT_CM5_UBUNTU_REPO_KEY_URL = ('http://archive.cloudera.com/cm5/ubuntu'
'/precise/amd64/cm/archive.key')
DEFAULT_CDH5_CENTOS_REPO_LIST_URL = ('http://archive.cloudera.com/cdh5/redhat'
'/6/x86_64/cdh/cloudera-cdh5.repo')
CDH5_CENTOS_REPO = ('[cloudera-cdh5]'
'\nname=Cloudera\'s Distribution for Hadoop, Version 5'
'\nbaseurl=http://archive.cloudera.com/cdh5/redhat/6'
'/x86_64/cdh/5.3.0/'
'\ngpgkey = http://archive.cloudera.com/cdh5/redhat/6'
'/x86_64/cdh/RPM-GPG-KEY-cloudera'
'\ngpgcheck = 1')
DEFAULT_CM5_CENTOS_REPO_LIST_URL = ('http://archive.cloudera.com/cm5/redhat'
'/6/x86_64/cm/cloudera-manager.repo')
CM5_CENTOS_REPO = ('[cloudera-manager]'
'\nname=Cloudera Manager'
'\nbaseurl=http://archive.cloudera.com/cm5/redhat/6'
'/x86_64/cm/5.3.0/'
'\ngpgkey = http://archive.cloudera.com/cm5/redhat/6'
'/x86_64/cm/RPM-GPG-KEY-cloudera'
'\ngpgcheck = 1')
DEFAULT_SWIFT_LIB_URL = ('https://repository.cloudera.com/artifactory/repo/org'
'/apache/hadoop/hadoop-openstack/2.3.0-cdh5.1.0'
'/hadoop-openstack-2.3.0-cdh5.1.0.jar')
'/apache/hadoop/hadoop-openstack/2.5.0-cdh5.3.0'
'/hadoop-openstack-2.5.0-cdh5.3.0.jar')
DEFAULT_EXTJS_LIB_URL = 'http://extjs.com/deploy/ext-2.2.zip'
HIVE_SERVER2_SENTRY_SAFETY_VALVE = (
'<property>'
'\n <name>hive.security.authorization.task.factory</name>'
'\n <value>org.apache.sentry.binding.hive.SentryHiveAuthorizationTask'
'FactoryImpl</value>'
'\n</property>'
'\n<property>'
'\n <name>hive.server2.session.hook</name>'
'\n <value>org.apache.sentry.binding.hive.HiveAuthzBindingSessionHook'
'</value>'
'\n</property>'
'\n<property>'
'\n <name>hive.sentry.conf.url</name>'
'\n <value>file:///{{CMF_CONF_DIR}}/sentry-site.xml</value>'
'\n</property>')
HIVE_METASTORE_SENTRY_SAFETY_VALVE = (
'<property>'
'\n <name>hive.metastore.client.impl</name>'
'\n <value>org.apache.sentry.binding.metastore.SentryHiveMetaStore'
'Client</value>'
'\n <description>Sets custom Hive metastore client which Sentry uses'
' to filter out metadata.</description>'
'\n</property>'
'\n<property>'
'\n <name>hive.metastore.pre.event.listeners</name>'
'\n <value>org.apache.sentry.binding.metastore.MetastoreAuthzBinding'
'</value>'
'\n <description>list of comma separated listeners for metastore'
' events.</description>'
'\n</property>'
'\n<property>'
'\n <name>hive.metastore.event.listeners</name>'
'\n <value>org.apache.sentry.binding.metastore.SentryMetastorePost'
'EventListener</value>'
'\n <description>list of comma separated listeners for metastore,'
' post events.</description>'
'\n</property>')
SENTRY_IMPALA_CLIENT_SAFETY_VALVE = (
'<property>'
'\n <name>sentry.service.client.server.rpc-port</name>'
'\n <value>3893</value>'
'\n</property>'
'\n<property>'
'\n <name>sentry.service.client.server.rpc-address</name>'
'\n <value>hostname</value>'
'\n</property>'
'\n<property>'
'\n <name>sentry.service.client.server.rpc-connection-timeout</name>'
'\n <value>200000</value>'
'\n</property>'
'\n<property>'
'\n <name>sentry.service.security.mode</name>'
'\n <value>none</value>'
'\n</property>')
CDH5_REPO_URL = p.Config(
'CDH5 repo list URL', 'general', 'cluster', priority=1,
default_value="")
@ -87,7 +158,7 @@ def _load_json(path_to_file):
return json.loads(data)
path_to_config = 'plugins/cdh/resources/'
path_to_config = 'plugins/cdh/v5_3_0/resources/'
hdfs_confs = _load_json(path_to_config + 'hdfs-service.json')
namenode_confs = _load_json(path_to_config + 'hdfs-namenode.json')
datanode_confs = _load_json(path_to_config + 'hdfs-datanode.json')
@ -98,15 +169,16 @@ resourcemanager_confs = _load_json(
nodemanager_confs = _load_json(path_to_config + 'yarn-nodemanager.json')
jobhistory_confs = _load_json(path_to_config + 'yarn-jobhistory.json')
oozie_service_confs = _load_json(path_to_config + 'oozie-service.json')
oozie_role_confs = _load_json(path_to_config + 'oozie-oozie.json')
oozie_role_confs = _load_json(path_to_config + 'oozie-oozie_server.json')
hive_service_confs = _load_json(path_to_config + 'hive-service.json')
hive_metastore_confs = _load_json(path_to_config + 'hive-metastore.json')
hive_metastore_confs = _load_json(path_to_config + 'hive-hivemetastore.json')
hive_hiveserver_confs = _load_json(path_to_config + 'hive-hiveserver2.json')
hive_webhcat_confs = _load_json(path_to_config + 'hive-webhcat.json')
hue_service_confs = _load_json(path_to_config + 'hue-service.json')
hue_role_confs = _load_json(path_to_config + 'hue-hue.json')
hue_role_confs = _load_json(path_to_config + 'hue-hue_server.json')
spark_service_confs = _load_json(path_to_config + 'spark-service.json')
spark_role_confs = _load_json(path_to_config + 'spark-history.json')
spark_role_confs = _load_json(
path_to_config + 'spark-spark_yarn_history_server.json')
zookeeper_service_confs = _load_json(path_to_config + 'zookeeper-service.json')
zookeeper_server_confs = _load_json(path_to_config + 'zookeeper-server.json')
hbase_confs = _load_json(path_to_config + 'hbase-service.json')

4
sahara/plugins/cdh/db_helper.py → sahara/plugins/cdh/v5_3_0/db_helper.py
View File

@ -41,7 +41,7 @@ def get_hive_db_password(cluster):
def create_hive_database(cluster, remote):
db_password = get_hive_db_password(cluster)
create_db_script = files.get_file_text(
'plugins/cdh/resources/create_hive_db.sql')
'plugins/cdh/v5_3_0/resources/create_hive_db.sql')
create_db_script = create_db_script % db_password
script_name = 'create_hive_db.sql'
remote.write_file_to(script_name, create_db_script)
@ -70,7 +70,7 @@ def get_sentry_db_password(cluster):
def create_sentry_database(cluster, remote):
db_password = get_sentry_db_password(cluster)
create_db_script = files.get_file_text(
'plugins/cdh/resources/create_sentry_db.sql')
'plugins/cdh/v5_3_0/resources/create_sentry_db.sql')
create_db_script = create_db_script % db_password
script_name = 'create_sentry_db.sql'
remote.write_file_to(script_name, create_db_script)

181
sahara/plugins/cdh/v5_3_0/deploy.py Normal file
View File

@ -0,0 +1,181 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from sahara.plugins.cdh import commands as cmd
from sahara.plugins.cdh.v5_3_0 import cloudera_utils as cu
from sahara.plugins import utils as gu
PACKAGES = [
'cloudera-manager-agent',
'cloudera-manager-daemons',
'cloudera-manager-server',
'cloudera-manager-server-db-2',
'flume-ng',
'hadoop-hdfs-datanode',
'hadoop-hdfs-namenode',
'hadoop-hdfs-secondarynamenode',
'hadoop-mapreduce',
'hadoop-mapreduce-historyserver',
'hadoop-yarn-nodemanager',
'hadoop-yarn-resourcemanager',
'hbase',
'hbase-solr',
'hive-hcatalog',
'hive-metastore',
'hive-server2',
'hive-webhcat-server',
'hue',
'impala',
'impala-server',
'impala-state-store',
'impala-catalog',
'ntp',
'oozie',
'oracle-j2sdk1.7',
'sentry',
'solr-server',
'spark-history-server',
'sqoop2',
'unzip',
'zookeeper'
]
CU = cu.ClouderaUtilsV530()
def configure_cluster(cluster):
instances = gu.get_instances(cluster)
if not cmd.is_pre_installed_cdh(CU.pu.get_manager(cluster).remote()):
CU.pu.configure_os(instances)
CU.pu.install_packages(instances, PACKAGES)
CU.pu.start_cloudera_agents(instances)
CU.pu.start_cloudera_manager(cluster)
CU.await_agents(instances)
CU.create_mgmt_service(cluster)
CU.create_services(cluster)
CU.configure_services(cluster)
CU.configure_instances(instances, cluster)
CU.deploy_configs(cluster)
def scale_cluster(cluster, instances):
if not instances:
return
if not cmd.is_pre_installed_cdh(instances[0].remote()):
CU.pu.configure_os(instances)
CU.pu.install_packages(instances, PACKAGES)
CU.pu.start_cloudera_agents(instances)
CU.await_agents(instances)
for instance in instances:
CU.configure_instance(instance)
CU.update_configs(instance)
if 'HDFS_DATANODE' in instance.node_group.node_processes:
CU.refresh_nodes(cluster, 'DATANODE', CU.HDFS_SERVICE_NAME)
CU.pu.configure_swift_to_inst(instance)
if 'HDFS_DATANODE' in instance.node_group.node_processes:
hdfs = CU.get_service_by_role('DATANODE', instance=instance)
CU.start_roles(hdfs, CU.pu.get_role_name(instance, 'DATANODE'))
if 'YARN_NODEMANAGER' in instance.node_group.node_processes:
yarn = CU.get_service_by_role('NODEMANAGER', instance=instance)
CU.start_roles(yarn, CU.pu.get_role_name(instance, 'NODEMANAGER'))
def decommission_cluster(cluster, instances):
dns = []
nms = []
for i in instances:
if 'HDFS_DATANODE' in i.node_group.node_processes:
dns.append(CU.pu.get_role_name(i, 'DATANODE'))
if 'YARN_NODEMANAGER' in i.node_group.node_processes:
nms.append(CU.pu.get_role_name(i, 'NODEMANAGER'))
if dns:
CU.decommission_nodes(cluster, 'DATANODE', dns)
if nms:
CU.decommission_nodes(cluster, 'NODEMANAGER', nms)
CU.delete_instances(cluster, instances)
CU.refresh_nodes(cluster, 'DATANODE', CU.HDFS_SERVICE_NAME)
CU.refresh_nodes(cluster, 'NODEMANAGER', CU.YARN_SERVICE_NAME)
def start_cluster(cluster):
if CU.pu.get_oozie(cluster):
CU.pu.install_extjs(cluster)
if CU.pu.get_hive_metastore(cluster):
CU.pu.configure_hive(cluster)
if CU.pu.get_sentry(cluster):
CU.pu.configure_sentry(cluster)
CU.first_run(cluster)
CU.pu.configure_swift(cluster)
if CU.pu.get_hive_metastore(cluster):
CU.pu.put_hive_hdfs_xml(cluster)
if CU.pu.get_flumes(cluster):
flume = CU.get_service_by_role('AGENT', cluster)
CU.start_service(flume)
def get_open_ports(node_group):
ports = [9000] # for CM agent
ports_map = {
'CLOUDERA_MANAGER': [7180, 7182, 7183, 7432, 7184, 8084, 8086, 10101,
9997, 9996, 8087, 9998, 9999, 8085, 9995, 9994],
'HDFS_NAMENODE': [8020, 8022, 50070, 50470],
'HDFS_SECONDARYNAMENODE': [50090, 50495],
'HDFS_DATANODE': [50010, 1004, 50075, 1006, 50020],
'YARN_RESOURCEMANAGER': [8030, 8031, 8032, 8033, 8088],
'YARN_NODEMANAGER': [8040, 8041, 8042],
'YARN_JOBHISTORY': [10020, 19888],
'HIVE_METASTORE': [9083],
'HIVE_SERVER2': [10000],
'HUE_SERVER': [8888],
'OOZIE_SERVER': [11000, 11001],
'SPARK_YARN_HISTORY_SERVER': [18088],
'ZOOKEEPER_SERVER': [2181, 3181, 4181, 9010],
'HBASE_MASTER': [60000],
'HBASE_REGIONSERVER': [60020],
'FLUME_AGENT': [41414],
'SENTRY_SERVER': [8038],
'SOLR_SERVER': [8983, 8984],
'SQOOP_SERVER': [8005, 12000],
'KEY_VALUE_STORE_INDEXER': [],
'IMPALA_CATALOGSERVER': [25020, 26000],
'IMPALA_STATESTORE': [25010, 24000],
'IMPALAD': [21050, 21000, 23000, 25000, 28000, 22000]
}
for process in node_group.node_processes:
if process in ports_map:
ports.extend(ports_map[process])
return ports

54
sahara/plugins/cdh/v5_3_0/edp_engine.py Normal file
View File

@ -0,0 +1,54 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from sahara.plugins.cdh.v5_3_0 import cloudera_utils as cu
from sahara.plugins import exceptions as ex
from sahara.plugins import utils as u
from sahara.service.edp import hdfs_helper
from sahara.service.edp.oozie import engine as edp_engine
CU = cu.ClouderaUtilsV530()
class EdpOozieEngine(edp_engine.OozieJobEngine):
def get_hdfs_user(self):
return 'hdfs'
def create_hdfs_dir(self, remote, dir_name):
hdfs_helper.create_dir_hadoop2(remote, dir_name, self.get_hdfs_user())
def get_oozie_server_uri(self, cluster):
oozie_ip = CU.pu.get_oozie(cluster).management_ip
return 'http://%s:11000/oozie' % oozie_ip
def get_name_node_uri(self, cluster):
namenode_ip = CU.pu.get_namenode(cluster).fqdn()
return 'hdfs://%s:8020' % namenode_ip
def get_resource_manager_uri(self, cluster):
resourcemanager_ip = CU.pu.get_resourcemanager(cluster).fqdn()
return '%s:8032' % resourcemanager_ip
def get_oozie_server(self, cluster):
return CU.pu.get_oozie(cluster)
def validate_job_execution(self, cluster, job, data):
oo_count = u.get_instances_count(cluster, 'OOZIE_SERVER')
if oo_count != 1:
raise ex.InvalidComponentCountException(
'OOZIE_SERVER', '1', oo_count)
super(EdpOozieEngine, self).validate_job_execution(cluster, job, data)

125
sahara/plugins/cdh/v5_3_0/plugin_utils.py Normal file
View File

@ -0,0 +1,125 @@
# Copyright (c) 2014 Intel Corporation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from sahara.conductor import resource as res
from sahara.plugins.cdh import plugin_utils as pu
from sahara.plugins.cdh.v5_3_0 import config_helper as c_helper
from sahara.plugins.cdh.v5_3_0 import db_helper
from sahara.plugins import utils as u
class PluginUtilsV530(pu.AbstractPluginUtils):
def __init__(self):
self.c_helper = c_helper
self.db_helper = db_helper
def get_role_name(self, instance, service):
# NOTE: role name must match regexp "[_A-Za-z][-_A-Za-z0-9]{0,63}"
shortcuts = {
'AGENT': 'A',
'ALERTPUBLISHER': 'AP',
'CATALOGSERVER': 'ICS',
'DATANODE': 'DN',
'EVENTSERVER': 'ES',
'HBASE_INDEXER': 'LHBI',
'HIVEMETASTORE': 'HVM',
'HIVESERVER2': 'HVS',
'HOSTMONITOR': 'HM',
'IMPALAD': 'ID',
'JOBHISTORY': 'JS',
'MASTER': 'M',
'NAMENODE': 'NN',
'NODEMANAGER': 'NM',
'OOZIE_SERVER': 'OS',
'REGIONSERVER': 'RS',
'RESOURCEMANAGER': 'RM',
'SECONDARYNAMENODE': 'SNN',
'SENTRY_SERVER': 'SNT',
'SERVER': 'S',
'SERVICEMONITOR': 'SM',
'SOLR_SERVER': 'SLR',
'SPARK_YARN_HISTORY_SERVER': 'SHS',
'SQOOP_SERVER': 'S2S',
'STATESTORE': 'ISS',
'WEBHCAT': 'WHC'
}
return '%s_%s' % (shortcuts.get(service, service),
instance.hostname().replace('-', '_'))
def get_sentry(self, cluster):
return u.get_instance(cluster, 'SENTRY_SERVER')
def get_flumes(self, cluster):
return u.get_instances(cluster, 'FLUME_AGENT')
def get_solrs(self, cluster):
return u.get_instances(cluster, 'SOLR_SERVER')
def get_sqoop(self, cluster):
return u.get_instance(cluster, 'SQOOP_SERVER')
def get_hbase_indexers(self, cluster):
return u.get_instances(cluster, 'KEY_VALUE_STORE_INDEXER')
def get_catalogserver(self, cluster):
return u.get_instance(cluster, 'IMPALA_CATALOGSERVER')
def get_statestore(self, cluster):
return u.get_instance(cluster, 'IMPALA_STATESTORE')
def get_impalads(self, cluster):
return u.get_instances(cluster, 'IMPALAD')
def convert_process_configs(self, configs):
p_dict = {
"CLOUDERA": ['MANAGER'],
"NAMENODE": ['NAMENODE'],
"DATANODE": ['DATANODE'],
"SECONDARYNAMENODE": ['SECONDARYNAMENODE'],
"RESOURCEMANAGER": ['RESOURCEMANAGER'],
"NODEMANAGER": ['NODEMANAGER'],
"JOBHISTORY": ['JOBHISTORY'],
"OOZIE": ['OOZIE_SERVER'],
"HIVESERVER": ['HIVESERVER2'],
"HIVEMETASTORE": ['HIVEMETASTORE'],
"WEBHCAT": ['WEBHCAT'],
"HUE": ['HUE_SERVER'],
"SPARK_ON_YARN": ['SPARK_YARN_HISTORY_SERVER'],
"ZOOKEEPER": ['SERVER'],
"MASTER": ['MASTER'],
"REGIONSERVER": ['REGIONSERVER'],
"FLUME": ['AGENT'],
"CATALOGSERVER": ['CATALOGSERVER'],
"STATESTORE": ['STATESTORE'],
"IMPALAD": ['IMPALAD'],
"KS_INDEXER": ['HBASE_INDEXER'],
"SENTRY": ['SENTRY_SERVER'],
"SOLR": ['SOLR_SERVER'],
"SQOOP": ['SQOOP_SERVER']
}
if isinstance(configs, res.Resource):
configs = configs.to_dict()
for k in configs.keys():
if k in p_dict.keys():
item = configs[k]
del configs[k]
newkey = p_dict[k][0]
configs[newkey] = item
return res.Resource(configs)
def configure_sentry(self, cluster):
manager = self.get_manager(cluster)
with manager.remote() as r:
self.db_helper.create_sentry_database(cluster, r)

2
sahara/plugins/cdh/resources/cdh_config.py → sahara/plugins/cdh/v5_3_0/resources/cdh_config.py
View File

@ -54,7 +54,7 @@ def process_service(service, service_name):
for role_cfgs in service.get_all_role_config_groups():
role_cm_cfg = role_cfgs.get_config(view='full')
role_cfg = parse_config(role_cm_cfg)
role_name = role_cfgs.roleType
role_name = role_cfgs.roleType.lower()
write_cfg(role_cfg, '%s-%s.json' % (service_name, role_name))
service_cm_cfg = service.get_config(view='full')[0]

3
sahara/plugins/cdh/v5_3_0/resources/cdh_config.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/bash
tox -evenv -- python $(dirname $0)/cdh_config.py $*

4
sahara/plugins/cdh/v5_3_0/resources/create_hive_db.sql Normal file
View File

@ -0,0 +1,4 @@
CREATE ROLE hive LOGIN PASSWORD '%s';
CREATE DATABASE metastore OWNER hive encoding 'UTF8';
GRANT ALL PRIVILEGES ON DATABASE metastore TO hive;
COMMIT;

0
sahara/plugins/cdh/resources/create_sentry_db.sql → sahara/plugins/cdh/v5_3_0/resources/create_sentry_db.sql
View File

6
sahara/plugins/cdh/resources/flume-agent.json → sahara/plugins/cdh/v5_3_0/resources/flume-agent.json
View File

@ -42,7 +42,7 @@
"value": "USERNAME [a-zA-Z0-9._-]+\nUSER %{USERNAME}\nINT (?:[+-]?(?:[0-9]+))\nBASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+)))\nNUMBER (?:%{BASE10NUM})\nBASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))\nBASE16FLOAT \\b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\\.[0-9A-Fa-f]*)?)|(?:\\.[0-9A-Fa-f]+)))\\b\n\nPOSINT \\b(?:[1-9][0-9]*)\\b\nNONNEGINT \\b(?:[0-9]+)\\b\nWORD \\b\\w+\\b\nNOTSPACE \\S+\nSPACE \\s*\nDATA .*?\nGREEDYDATA .*\n#QUOTEDSTRING (?:(?<!\\\\)(?:\"(?:\\\\.|[^\\\\\"])*\"|(?:'(?:\\\\.|[^\\\\'])*')|(?:`(?:\\\\.|[^\\\\`])*`)))\nQUOTEDSTRING (?>(?<!\\\\)(?>\"(?>\\\\.|[^\\\\\"]+)+\"|\"\"|(?>'(?>\\\\.|[^\\\\']+)+')|''|(?>`(?>\\\\.|[^\\\\`]+)+`)|``))\nUUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}\n\n# Networking\nMAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})\nCISCOMAC (?:(?:[A-Fa-f0-9]{4}\\.){2}[A-Fa-f0-9]{4})\nWINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})\nCOMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})\nIP (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])\nHOSTNAME \\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)\nHOST %{HOSTNAME}\nIPORHOST (?:%{HOSTNAME}|%{IP})\n#HOSTPORT (?:%{IPORHOST=~/\\./}:%{POSINT}) # WH\n\n# paths\nPATH (?:%{UNIXPATH}|%{WINPATH})\nUNIXPATH (?>/(?>[\\w_%!$@:.,-]+|\\\\.)*)+\n#UNIXPATH (?<![\\w\\/])(?:/[^\\/\\s?*]*)+\nLINUXTTY (?>/dev/pts/%{NONNEGINT})\nBSDTTY (?>/dev/tty[pq][a-z0-9])\nTTY (?:%{BSDTTY}|%{LINUXTTY})\nWINPATH (?>[A-Za-z]+:|\\\\)(?:\\\\[^\\\\?*]*)+\nURIPROTO [A-Za-z]+(\\+[A-Za-z+]+)?\nURIHOST %{IPORHOST}(?::%{POSINT:port})?\n# uripath comes loosely from RFC1738, but mostly from what Firefox\n# doesn't turn into %XX\nURIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=#%_\\-]*)+\n#URIPARAM \\?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?\nURIPARAM \\?[A-Za-z0-9$.+!*'|(){},~#%&/=:;_?\\-\\[\\]]*\nURIPATHPARAM %{URIPATH}(?:%{URIPARAM})?\nURI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?\n\n# Months: January, Feb, 3, 03, 12, December\nMONTH \\b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\\b\nMONTHNUM (?:0?[1-9]|1[0-2])\nMONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])\n\n# Days: Monday, Tue, Thu, etc...\nDAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)\n\n# Years?\nYEAR (?>\\d\\d){1,2}\n# Time: HH:MM:SS\n#TIME \\d{2}:\\d{2}(?::\\d{2}(?:\\.\\d+)?)?\n# I'm still on the fence about using grok to perform the time match,\n# since it's probably slower.\n# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\\.%{POSINT})?)?\nHOUR (?:2[0123]|[01]?[0-9])\nMINUTE (?:[0-5][0-9])\n# '60' is a leap second in most time standards and thus is valid.\nSECOND (?:(?:[0-5][0-9]|60)(?:[:.,][0-9]+)?)\nTIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])\n# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)\nDATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}\nDATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}\nISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))\nISO8601_SECOND (?:%{SECOND}|60)\nTIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?\nDATE %{DATE_US}|%{DATE_EU}\nDATESTAMP %{DATE}[- ]%{TIME}\nTZ (?:[PMCE][SD]T)\nDATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}\nDATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}\n\n# Syslog Dates: Month Day HH:MM:SS\nSYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}\nPROG (?:[\\w._/%-]+)\nSYSLOGPROG %{PROG:program}(?:\\[%{POSINT:pid}\\])?\nSYSLOGHOST %{IPORHOST}\nSYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>\nHTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}\n\n# Shortcuts\nQS %{QUOTEDSTRING}\n\n# Log formats\nSYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:\nCOMBINEDAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \\[%{HTTPDATE:timestamp}\\] \"(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent}\n\n# Log Levels\nLOGLEVEL ([T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)"
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that expose an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
@ -114,7 +114,7 @@
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is hit, the oldest data will be deleted.",
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
@ -204,7 +204,7 @@
"value": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor\n license agreements. See the NOTICE file distributed with this work for additional\n information regarding copyright ownership. The ASF licenses this file to\n You under the Apache License, Version 2.0 (the \"License\"); you may not use\n this file except in compliance with the License. You may obtain a copy of\n the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required\n by applicable law or agreed to in writing, software distributed under the\n License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS\n OF ANY KIND, either express or implied. See the License for the specific\n language governing permissions and limitations under the License. -->\n\n<mime-info>\n\n <mime-type type=\"text/space-separated-values\">\n <glob pattern=\"*.ssv\"/>\n </mime-type>\n\n <mime-type type=\"avro/binary\">\n <magic priority=\"50\">\n <match value=\"0x4f626a01\" type=\"string\" offset=\"0\"/> \n </magic>\n <glob pattern=\"*.avro\"/>\n </mime-type>\n\n <mime-type type=\"mytwittertest/json+delimited+length\">\n <magic priority=\"50\">\n <match value=\"[0-9]+(\\r)?\\n\\\\{&quot;\" type=\"regex\" offset=\"0:16\"/> \n </magic>\n </mime-type>\n \n <mime-type type=\"application/hadoop-sequence-file\">\n <magic priority=\"50\">\n <match value=\"SEQ[\\0-\\6]\" type=\"regex\" offset=\"0\"/>\n </magic>\n </mime-type>\n \n</mime-info>"
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span>subdirectory of the role's log directory.",
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null

0
sahara/plugins/cdh/resources/flume-service.json → sahara/plugins/cdh/v5_3_0/resources/flume-service.json
View File

44
sahara/plugins/cdh/v5_3_0/resources/hbase-gateway.json Normal file
View File

@ -0,0 +1,44 @@
[
{
"desc": "For advanced use only, a string to be inserted into the client configuration for <strong>hbase-site.xml</strong>.",
"display_name": "HBase Client Advanced Configuration Snippet (Safety Valve) for hbase-site.xml",
"name": "hbase_client_config_safety_valve",
"value": null
},
{
"desc": "The directory where the client configs will be deployed",
"display_name": "Deploy Directory",
"name": "client_config_root_dir",
"value": "/etc/hbase"
},
{
"desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the client configuration for <strong>hbase-env.sh</strong>",
"display_name": "HBase Client Environment Advanced Configuration Snippet for hbase-env.sh (Safety Valve)",
"name": "hbase_client_env_safety_valve",
"value": null
},
{
"desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.",
"display_name": "Alternatives Priority",
"name": "client_config_priority",
"value": "90"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "These are Java command line arguments. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Client Java Configuration Options",
"name": "hbase_client_java_opts",
"value": "-XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:+CMSIncrementalMode -Djava.net.preferIPv4Stack=true"
},
{
"desc": "Maximum size in bytes for the Java process heap memory. Passed to Java -Xmx.",
"display_name": "Client Java Heap Size in Bytes",
"name": "hbase_client_java_heapsize",
"value": "268435456"
}
]

254
sahara/plugins/cdh/v5_3_0/resources/hbase-hbaserestserver.json Normal file
View File

@ -0,0 +1,254 @@
[
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Log Directory Free Space Monitoring Percentage Thresholds",
"name": "log_directory_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "<p>This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.</p><p>Each rule has some or all of the following fields:</p><ul><li><span class='code'>alert</span> - whether or not events generated from this rule should be promoted to alerts. A value of \"true\" will cause alerts to be generated. If not specified, the default is \"false\".</li><li><span class='code'>rate</span> <strong>(mandatory)</strong> - the maximum number of log messages matching this rule that may be sent as events every minute. If more than <tt>rate</tt> matching log messages are received in a single minute, the extra messages are ignored. If rate is less than 0, the number of messages per minute is unlimited.</li><li><span class='code'>periodminutes</span> - the number of minutes during which the publisher will only publish <tt>rate</tt> events or fewer. If not specified, the default is <strong>one minute</strong></li><li><span class='code'>threshold</span> - apply this rule only to messages with this log4j severity level or above. An example is \"WARN\" for warning level messages or higher.</li><li><span class='code'>content</span> - match only those messages whose contents match this regular expression.</li><li><span class='code'>exceptiontype</span> - match only those messages which are part of an exception message. The exception type must match this regular expression.</li></ul><br/><p>Example:<span class='code'>{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}</span></p><p>This rule will send events to Cloudera Manager for every <span class='code'>StringIndexOutOfBoundsException</span>, up to a maximum of 10 every minute.</p>",
"display_name": "Rules to Extract Events from Log Files",
"name": "log_event_whitelist",
"value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}"
},
{
"desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Java Configuration Options for HBase REST Server",
"name": "hbase_restserver_java_opts",
"value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled"
},
{
"desc": "Whether or not periodic stacks collection is enabled.",
"display_name": "Stacks Collection Enabled",
"name": "stacks_collection_enabled",
"value": "false"
},
{
"desc": "Maximum size of the HBase REST Server thread pool. The server can process this number of concurrent requests. Setting this too high can lead to out of memory errors.",
"display_name": "HBase REST Server Maximum Threads",
"name": "hbase_restserver_threads_max",
"value": "100"
},
{
"desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.",
"display_name": "File Descriptor Monitoring Thresholds",
"name": "hbaserestserver_fd_thresholds",
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hbase-site.xml</strong> for this role only.",
"display_name": "HBase REST Server Advanced Configuration Snippet (Safety Valve) for hbase-site.xml",
"name": "hbase_restserver_config_safety_valve",
"value": null
},
{
"desc": "Directory where HBase REST Server will place its log files.",
"display_name": "HBase REST Server Log Directory",
"name": "hbase_restserver_log_dir",
"value": "/var/log/hbase"
},
{
"desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
"display_name": "Automatically Restart Process",
"name": "process_auto_restart",
"value": "false"
},
{
"desc": "The maximum number of rolled log files to keep for HBase REST Server logs. Typically used by log4j.",
"display_name": "HBase REST Server Maximum Log File Backups",
"name": "max_log_backup_index",
"value": "10"
},
{
"desc": "<p>The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific role. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Role Triggers",
"name": "role_triggers",
"value": "[]"
},
{
"desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Soft Limit",
"name": "rm_memory_soft_limit",
"value": "-1"
},
{
"desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.",
"display_name": "Java Heap Size of HBase REST Server in Bytes",
"name": "hbase_restserver_java_heapsize",
"value": "1073741824"
},
{
"desc": "Enables the health test that the HBase REST Server's process state is consistent with the role configuration",
"display_name": "HBase REST Server Process Health Test",
"name": "hbaserestserver_scm_health_enabled",
"value": "true"
},
{
"desc": "The port that HBase REST Server binds to.",
"display_name": "HBase REST Server Port",
"name": "hbase_restserver_port",
"value": "20550"
},
{
"desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Hard Limit",
"name": "rm_memory_hard_limit",
"value": "-1"
},
{
"desc": "If true, HBase REST Server Web UI will bind to a wildcard address (0.0.0.0). Otherwise it will bind to a host name. Only available in CDH 4.3 and later.",
"display_name": "HBase REST Server Web UI Bind to Wildcard Address",
"name": "hbase_restserver_info_bind_to_wildcard",
"value": "true"
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>log4j.properties</strong> for this role only.",
"display_name": "HBase REST Server Logging Advanced Configuration Snippet (Safety Valve)",
"name": "log4j_safety_valve",
"value": null
},
{
"desc": "The frequency with which stacks will be collected.",
"display_name": "Stacks Collection Frequency",
"name": "stacks_collection_frequency",
"value": "5.0"
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.",
"display_name": "Log Directory Free Space Monitoring Absolute Thresholds",
"name": "log_directory_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "When false, all HTTP methods are permitted (GET/PUT/POST/DELETE). When true, only GET is permitted.",
"display_name": "Enable HBase REST Server Read Only Mode",
"name": "hbase_restserver_readonly",
"value": "false"
},
{
"desc": "The port that HBase REST Server Web UI binds to.",
"display_name": "HBase REST Server Web UI Port",
"name": "hbase_restserver_info_port",
"value": "8085"
},
{
"desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.",
"display_name": "Maximum Process File Descriptors",
"name": "rlimit_fds",
"value": null
},
{
"desc": "When computing the overall HBase REST Server health, consider the host's health.",
"display_name": "HBase REST Server Host Health Test",
"name": "hbaserestserver_host_health_enabled",
"value": "true"
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Health Alerts for this Role",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "The maximum size, in megabytes, per log file for HBase REST Server logs. Typically used by log4j.",
"display_name": "HBase REST Server Max Log Size",
"name": "max_log_size",
"value": "200"
},
{
"desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.",
"display_name": "Kill When Out of Memory",
"name": "oom_sigkill_enabled",
"value": "true"
},
{
"desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.",
"display_name": "Dump Heap When Out of Memory",
"name": "oom_heap_dump_enabled",
"value": "false"
},
{
"desc": "The host name or IP address of the DNS name server which an HBase REST Server should use to determine the host name used for communication and display purposes.",
"display_name": "HBase REST Server DNS Name Server",
"name": "hbase_restserver_dns_nameserver",
"value": null
},
{
"desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.",
"display_name": "Unexpected Exits Thresholds",
"name": "unexpected_exits_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "HBase REST Server will bind to this address.",
"display_name": "HBase REST Server Host Address",
"name": "hbase_restserver_host",
"value": "0.0.0.0"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "The name of the DNS network interface from which an HBase REST Server should report its IP address.",
"display_name": "HBase REST Server DNS Network Interface",
"name": "hbase_restserver_dns_interface",
"value": null
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null
},
{
"desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup I/O Weight",
"name": "rm_io_weight",
"value": "500"
},
{
"desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup CPU Shares",
"name": "rm_cpu_shares",
"value": "1024"
},
{
"desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.",
"display_name": "Heap Dump Directory",
"name": "oom_heap_dump_dir",
"value": "/tmp"
},
{
"desc": "The period to review when computing unexpected exits.",
"display_name": "Unexpected Exits Monitoring Period",
"name": "unexpected_exits_window",
"value": "5"
},
{
"desc": "Minimum size of the HBase REST Server thread pool. The server will maintain at least this number of threads in the pool at all times. The thread pool can grow up to the maximum size set by hbase.rest.threads.max.",
"display_name": "HBase REST Server Minimum Threads",
"name": "hbase_restserver_threads_min",
"value": "2"
},
{
"desc": "The minimum log level for HBase REST Server logs",
"display_name": "HBase REST Server Logging Threshold",
"name": "log_threshold",
"value": "INFO"
}
]

260
sahara/plugins/cdh/v5_3_0/resources/hbase-hbasethriftserver.json Normal file
View File

@ -0,0 +1,260 @@
[
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Log Directory Free Space Monitoring Percentage Thresholds",
"name": "log_directory_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "<p>This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.</p><p>Each rule has some or all of the following fields:</p><ul><li><span class='code'>alert</span> - whether or not events generated from this rule should be promoted to alerts. A value of \"true\" will cause alerts to be generated. If not specified, the default is \"false\".</li><li><span class='code'>rate</span> <strong>(mandatory)</strong> - the maximum number of log messages matching this rule that may be sent as events every minute. If more than <tt>rate</tt> matching log messages are received in a single minute, the extra messages are ignored. If rate is less than 0, the number of messages per minute is unlimited.</li><li><span class='code'>periodminutes</span> - the number of minutes during which the publisher will only publish <tt>rate</tt> events or fewer. If not specified, the default is <strong>one minute</strong></li><li><span class='code'>threshold</span> - apply this rule only to messages with this log4j severity level or above. An example is \"WARN\" for warning level messages or higher.</li><li><span class='code'>content</span> - match only those messages whose contents match this regular expression.</li><li><span class='code'>exceptiontype</span> - match only those messages which are part of an exception message. The exception type must match this regular expression.</li></ul><br/><p>Example:<span class='code'>{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}</span></p><p>This rule will send events to Cloudera Manager for every <span class='code'>StringIndexOutOfBoundsException</span>, up to a maximum of 10 every minute.</p>",
"display_name": "Rules to Extract Events from Log Files",
"name": "log_event_whitelist",
"value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}"
},
{
"desc": "Whether or not periodic stacks collection is enabled.",
"display_name": "Stacks Collection Enabled",
"name": "stacks_collection_enabled",
"value": "false"
},
{
"desc": "The \"core size\" of the thread pool. New threads are created on every connection until this many threads are created.",
"display_name": "HBase Thrift Server Min Worker Threads",
"name": "hbase_thriftserver_min_worker_threads",
"value": "200"
},
{
"desc": "The port that HBase Thrift Server binds to.",
"display_name": "HBase Thrift Server Port",
"name": "hbase_thriftserver_port",
"value": "9090"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hbase-site.xml</strong> for this role only.",
"display_name": "HBase Thrift Server Advanced Configuration Snippet (Safety Valve) for hbase-site.xml",
"name": "hbase_thriftserver_config_safety_valve",
"value": null
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
},
{
"desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
"display_name": "Automatically Restart Process",
"name": "process_auto_restart",
"value": "false"
},
{
"desc": "The maximum number of rolled log files to keep for HBase Thrift Server logs. Typically used by log4j.",
"display_name": "HBase Thrift Server Maximum Log File Backups",
"name": "max_log_backup_index",
"value": "10"
},
{
"desc": "<p>The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific role. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Role Triggers",
"name": "role_triggers",
"value": "[]"
},
{
"desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.",
"display_name": "Java Heap Size of HBase Thrift Server in Bytes",
"name": "hbase_thriftserver_java_heapsize",
"value": "1073741824"
},
{
"desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Soft Limit",
"name": "rm_memory_soft_limit",
"value": "-1"
},
{
"desc": "If true, HBase Thrift Server Web UI will bind to a wildcard address (0.0.0.0). Otherwise it will bind to a host name. Only available in CDH 4.3 and later.",
"display_name": "HBase Thrift Server Web UI Bind to Wildcard Address",
"name": "hbase_thriftserver_info_bind_to_wildcard",
"value": "true"
},
{
"desc": "Address to bind the HBase Thrift Server to. When using the THsHaServer or the TNonblockingServer, always binds to 0.0.0.0 irrespective of this configuration value.",
"display_name": "HBase Thrift Server Bind Address",
"name": "hbase_thriftserver_bindaddress",
"value": "0.0.0.0"
},
{
"desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Hard Limit",
"name": "rm_memory_hard_limit",
"value": "-1"
},
{
"desc": "Type of HBase Thrift Server.",
"display_name": "HBase Thrift Server Type",
"name": "hbase_thriftserver_type",
"value": "threadpool"
},
{
"desc": "The period to review when computing unexpected exits.",
"display_name": "Unexpected Exits Monitoring Period",
"name": "unexpected_exits_window",
"value": "5"
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>log4j.properties</strong> for this role only.",
"display_name": "HBase Thrift Server Logging Advanced Configuration Snippet (Safety Valve)",
"name": "log4j_safety_valve",
"value": null
},
{
"desc": "When computing the overall HBase Thrift Server health, consider the host's health.",
"display_name": "HBase Thrift Server Host Health Test",
"name": "hbasethriftserver_host_health_enabled",
"value": "true"
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.",
"display_name": "Log Directory Free Space Monitoring Absolute Thresholds",
"name": "log_directory_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "Enables the health test that the HBase Thrift Server's process state is consistent with the role configuration",
"display_name": "HBase Thrift Server Process Health Test",
"name": "hbasethriftserver_scm_health_enabled",
"value": "true"
},
{
"desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Java Configuration Options for HBase Thrift Server",
"name": "hbase_thriftserver_java_opts",
"value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled"
},
{
"desc": "Use framed transport. When using the THsHaServer or TNonblockingServer, framed transport is always used irrespective of this configuration value.",
"display_name": "Enable HBase Thrift Server Framed Transport",
"name": "hbase_thriftserver_framed",
"value": "false"
},
{
"desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.",
"display_name": "Maximum Process File Descriptors",
"name": "rlimit_fds",
"value": null
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Health Alerts for this Role",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "The maximum size, in megabytes, per log file for HBase Thrift Server logs. Typically used by log4j.",
"display_name": "HBase Thrift Server Max Log Size",
"name": "max_log_size",
"value": "200"
},
{
"desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.",
"display_name": "Kill When Out of Memory",
"name": "oom_sigkill_enabled",
"value": "true"
},
{
"desc": "The port that HBase Thrift Server Web UI binds to.",
"display_name": "HBase Thrift Server Web UI Port",
"name": "hbase_thriftserver_info_port",
"value": "9095"
},
{
"desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.",
"display_name": "Dump Heap When Out of Memory",
"name": "oom_heap_dump_enabled",
"value": "false"
},
{
"desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.",
"display_name": "Unexpected Exits Thresholds",
"name": "unexpected_exits_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "The host name or IP address of the DNS name server which an HBase Thrift Server should use to determine the host name used for communication and display purposes.",
"display_name": "HBase Thrift Server DNS Name Server",
"name": "hbase_thriftserver_dns_nameserver",
"value": null
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "The name of the DNS network interface from which an HBase Thrift Server should report its IP address.",
"display_name": "HBase Thrift Server DNS Network Interface",
"name": "hbase_thriftserver_dns_interface",
"value": null
},
{
"desc": "Use the TCompactProtocol instead of the default TBinaryProtocol. TCompactProtocol is a binary protocol that is more compact than the default and typically more efficient.",
"display_name": "Enable HBase Thrift Server Compact Protocol",
"name": "hbase_thriftserver_compact",
"value": "false"
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null
},
{
"desc": "Directory where HBase Thrift Server will place its log files.",
"display_name": "HBase Thrift Server Log Directory",
"name": "hbase_thriftserver_log_dir",
"value": "/var/log/hbase"
},
{
"desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup I/O Weight",
"name": "rm_io_weight",
"value": "500"
},
{
"desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.",
"display_name": "File Descriptor Monitoring Thresholds",
"name": "hbasethriftserver_fd_thresholds",
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup CPU Shares",
"name": "rm_cpu_shares",
"value": "1024"
},
{
"desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.",
"display_name": "Heap Dump Directory",
"name": "oom_heap_dump_dir",
"value": "/tmp"
},
{
"desc": "The frequency with which stacks will be collected.",
"display_name": "Stacks Collection Frequency",
"name": "stacks_collection_frequency",
"value": "5.0"
},
{
"desc": "The minimum log level for HBase Thrift Server logs",
"display_name": "HBase Thrift Server Logging Threshold",
"name": "log_threshold",
"value": "INFO"
}
]

326
sahara/plugins/cdh/v5_3_0/resources/hbase-master.json Normal file
View File

@ -0,0 +1,326 @@
[
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Log Directory Free Space Monitoring Percentage Thresholds",
"name": "log_directory_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "<p>This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.</p><p>Each rule has some or all of the following fields:</p><ul><li><span class='code'>alert</span> - whether or not events generated from this rule should be promoted to alerts. A value of \"true\" will cause alerts to be generated. If not specified, the default is \"false\".</li><li><span class='code'>rate</span> <strong>(mandatory)</strong> - the maximum number of log messages matching this rule that may be sent as events every minute. If more than <tt>rate</tt> matching log messages are received in a single minute, the extra messages are ignored. If rate is less than 0, the number of messages per minute is unlimited.</li><li><span class='code'>periodminutes</span> - the number of minutes during which the publisher will only publish <tt>rate</tt> events or fewer. If not specified, the default is <strong>one minute</strong></li><li><span class='code'>threshold</span> - apply this rule only to messages with this log4j severity level or above. An example is \"WARN\" for warning level messages or higher.</li><li><span class='code'>content</span> - match only those messages whose contents match this regular expression.</li><li><span class='code'>exceptiontype</span> - match only those messages which are part of an exception message. The exception type must match this regular expression.</li></ul><br/><p>Example:<span class='code'>{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}</span></p><p>This rule will send events to Cloudera Manager for every <span class='code'>StringIndexOutOfBoundsException</span>, up to a maximum of 10 every minute.</p>",
"display_name": "Rules to Extract Events from Log Files",
"name": "log_event_whitelist",
"value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"IPC Server handler.*ClosedChannelException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"IPC Server Responder, call.*output error\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Daughter regiondir does not exist: .*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"File.*might still be open.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"File.*might still be open.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Moving table .+ state to enabled but was already enabled\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Received OPENED for region.*but region was in the state.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n"
},
{
"desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.",
"display_name": "Health Check Startup Tolerance",
"name": "master_startup_tolerance",
"value": "5"
},
{
"desc": "Whether or not periodic stacks collection is enabled.",
"display_name": "Stacks Collection Enabled",
"name": "stacks_collection_enabled",
"value": "false"
},
{
"desc": "Number of pooled threads to handle region closing in the master.",
"display_name": "Region Closing Threads",
"name": "hbase_master_executor_closeregion_threads",
"value": "5"
},
{
"desc": "When computing the overall Master health, consider the host's health.",
"display_name": "Master Host Health Test",
"name": "master_host_health_enabled",
"value": "true"
},
{
"desc": "Number of pooled threads to handle the recovery of the region servers in the master.",
"display_name": "RegionServer Recovery Threads",
"name": "hbase_master_executor_serverops_threads",
"value": "5"
},
{
"desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup I/O Weight",
"name": "rm_io_weight",
"value": "500"
},
{
"desc": "Directory where Master will place its log files.",
"display_name": "Master Log Directory",
"name": "hbase_master_log_dir",
"value": "/var/log/hbase"
},
{
"desc": "The health test thresholds on the duration of the metrics request to the web server.",
"display_name": "Web Metric Collection Duration",
"name": "master_web_metric_collection_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}"
},
{
"desc": "Time period in seconds to reset long-running metrics (e.g. compactions). This is an HBase specific configuration.",
"display_name": "Extended Period",
"name": "hbase_metrics_extended_period",
"value": "3600"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>log4j.properties</strong> for this role only.",
"display_name": "Master Logging Advanced Configuration Snippet (Safety Valve)",
"name": "log4j_safety_valve",
"value": null
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
},
{
"desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.",
"display_name": "File Descriptor Monitoring Thresholds",
"name": "master_fd_thresholds",
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "The port that the HBase Master binds to.",
"display_name": "HBase Master Port",
"name": "hbase_master_port",
"value": "60000"
},
{
"desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.",
"display_name": "Web Metric Collection",
"name": "master_web_metric_collection_enabled",
"value": "true"
},
{
"desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
"display_name": "Automatically Restart Process",
"name": "process_auto_restart",
"value": "false"
},
{
"desc": "List of org.apache.hadoop.hbase.coprocessor.MasterObserver coprocessors that are loaded by default on the active HMaster process. For any implemented coprocessor methods, the listed classes will be called in order. After implementing your own MasterObserver, just put it in HBase's classpath and add the fully qualified class name here.",
"display_name": "HBase Coprocessor Master Classes",
"name": "hbase_coprocessor_master_classes",
"value": ""
},
{
"desc": "<p>The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific role. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Role Triggers",
"name": "role_triggers",
"value": "[]"
},
{
"desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.",
"display_name": "Java Heap Size of HBase Master in Bytes",
"name": "hbase_master_java_heapsize",
"value": "1073741824"
},
{
"desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Soft Limit",
"name": "rm_memory_soft_limit",
"value": "-1"
},
{
"desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.",
"display_name": "Garbage Collection Duration Thresholds",
"name": "master_gc_duration_thresholds",
"value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}"
},
{
"desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Hard Limit",
"name": "rm_memory_hard_limit",
"value": "-1"
},
{
"desc": "When true, HBase Master will bind to 0.0.0.0. Only available with CDH 4.3 and later.",
"display_name": "HBase Master Bind to Wildcard Address",
"name": "hbase_master_bind_to_wildcard_address",
"value": "true"
},
{
"desc": "Enables the health test that a client can connect to the HBase Master",
"display_name": "HBase Master Canary Health Test",
"name": "master_canary_health_enabled",
"value": "true"
},
{
"desc": "The period to review when computing unexpected exits.",
"display_name": "Unexpected Exits Monitoring Period",
"name": "unexpected_exits_window",
"value": "5"
},
{
"desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into <strong>hadoop-metrics2.properties</strong>.",
"display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)",
"name": "hadoop_metrics2_safety_valve",
"value": null
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
},
{
"desc": "The port for the HBase Master web UI. Set to -1 to disable the HBase Master web UI.",
"display_name": "HBase Master Web UI Port",
"name": "hbase_master_info_port",
"value": "60010"
},
{
"desc": "The period to review when computing the moving average of garbage collection time.",
"display_name": "Garbage Collection Duration Monitoring Period",
"name": "master_gc_duration_window",
"value": "5"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hbase-site.xml</strong> for this role only.",
"display_name": "Master Advanced Configuration Snippet (Safety Valve) for hbase-site.xml",
"name": "hbase_master_config_safety_valve",
"value": null
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.",
"display_name": "Log Directory Free Space Monitoring Absolute Thresholds",
"name": "log_directory_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.",
"display_name": "Maximum Process File Descriptors",
"name": "rlimit_fds",
"value": null
},
{
"desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Java Configuration Options for HBase Master",
"name": "hbase_master_java_opts",
"value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled"
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Health Alerts for this Role",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "The maximum size, in megabytes, per log file for Master logs. Typically used by log4j.",
"display_name": "Master Max Log Size",
"name": "max_log_size",
"value": "200"
},
{
"desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.",
"display_name": "Kill When Out of Memory",
"name": "oom_sigkill_enabled",
"value": "true"
},
{
"desc": "Maximum time an HLog remains in the .oldlogdir directory until an HBase Master thread deletes it.",
"display_name": "Maximum Time to Keep HLogs",
"name": "hbase_master_logcleaner_ttl",
"value": "60000"
},
{
"desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.",
"display_name": "Dump Heap When Out of Memory",
"name": "oom_heap_dump_enabled",
"value": "false"
},
{
"desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.",
"display_name": "Unexpected Exits Thresholds",
"name": "unexpected_exits_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "Number of RPC Server instances spun up on HBase Master.",
"display_name": "HBase Master Handler Count",
"name": "hbase_master_handler_count",
"value": "25"
},
{
"desc": "The address for the HBase Master web UI",
"display_name": "HBase Master Web UI Address",
"name": "hbase_master_info_bindAddress",
"value": null
},
{
"desc": "A comma-separated list of LogCleanerDelegate(s) that are used in LogsCleaner. WAL/HLog cleaner(s) are called in order, so put the log cleaner that prunes the most log files in the front. To implement your own LogCleanerDelegate, add it to HBase's classpath and add the fully-qualified class name here. You should always add the above default log cleaners in the list, unless you have a special reason not to.",
"display_name": "HBase Master Log Cleaner Plugins",
"name": "hbase_master_logcleaner_plugins",
"value": null
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null
},
{
"desc": "The maximum number of rolled log files to keep for Master logs. Typically used by log4j.",
"display_name": "Master Maximum Log File Backups",
"name": "max_log_backup_index",
"value": "10"
},
{
"desc": "The host name or IP address of the DNS name server which an HBase Master should use to determine the host name used for communication and display purposes.",
"display_name": "HBase Master DNS Name Server",
"name": "hbase_master_dns_nameserver",
"value": null
},
{
"desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup CPU Shares",
"name": "rm_cpu_shares",
"value": "1024"
},
{
"desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.",
"display_name": "Heap Dump Directory",
"name": "oom_heap_dump_dir",
"value": "/tmp"
},
{
"desc": "Number of pooled threads to handle region opening in the master.",
"display_name": "Region Opening Threads",
"name": "hbase_master_executor_openregion_threads",
"value": "5"
},
{
"desc": "The frequency with which stacks will be collected.",
"display_name": "Stacks Collection Frequency",
"name": "stacks_collection_frequency",
"value": "5.0"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "The name of the DNS network interface from which an HBase Master should report its IP address.",
"display_name": "HBase Master DNS Network Interface",
"name": "hbase_master_dns_interface",
"value": null
},
{
"desc": "The minimum log level for Master logs",
"display_name": "Master Logging Threshold",
"name": "log_threshold",
"value": "INFO"
},
{
"desc": "Enables the health test that the Master's process state is consistent with the role configuration",
"display_name": "Master Process Health Test",
"name": "master_scm_health_enabled",
"value": "true"
}
]

548
sahara/plugins/cdh/v5_3_0/resources/hbase-regionserver.json Normal file
View File

File diff suppressed because one or more lines are too long

482
sahara/plugins/cdh/v5_3_0/resources/hbase-service.json Normal file
View File

@ -0,0 +1,482 @@
[
{
"desc": "Comma-delimited list of hosts where you want to allow the HBase user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that does not correspond to a host name, such as '_no_host'. <b>Note:</b> This property is used only if HBase REST/Thrift Server Authentication is enabled.",
"display_name": "HBase Proxy User Hosts",
"name": "hbase_proxy_user_hosts_list",
"value": "*"
},
{
"desc": "The user that this service's processes should run as.",
"display_name": "System User",
"name": "process_username",
"value": "hbase"
},
{
"desc": "The frequency in which the log4j event publication appender will retry sending undelivered log events to the Event server, in seconds",
"display_name": "Log Event Retry Frequency",
"name": "log_event_retry_frequency",
"value": "30"
},
{
"desc": "A general client pause time value. Used mostly as a time period to wait before retrying operations such as a failed get or region lookup.",
"display_name": "HBase Client Pause",
"name": "hbase_client_pause",
"value": "100"
},
{
"desc": "Comma-delimited list of groups that you want to allow the HBase user to impersonate. The default '*' allows all groups. To disable entirely, use a string that does not correspond to a group name, such as '_no_group_'. <b>Note:</b> This property is used only if HBase REST/Thrift Server Authentication is enabled.",
"display_name": "HBase Proxy User Groups",
"name": "hbase_proxy_user_groups_list",
"value": "*"
},
{
"desc": "Allow indexing of tables in HBase by Lily HBase Indexer. <strong>Note:</strong> Replication must be enabled for indexing to work.",
"display_name": "Enable Indexing",
"name": "hbase_enable_indexing",
"value": "false"
},
{
"desc": "<p>\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n</p>\n\n<p>\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n</p>\n\n<p>Event trackers are defined in a JSON object like the following:</p>\n\n<pre>\n{\n \"timeToLive\" : [integer],\n \"fields\" : [\n {\n \"type\" : [string],\n \"name\" : [string]\n }\n ]\n}\n</pre>\n\n<p>\nWhere:\n</p>\n\n<ul>\n <li>timeToLive: maximum amount of time an event will be tracked, in\n milliseconds. Must be provided. This defines how long, since it's\n first seen, an event will be tracked. A value of 0 disables tracking.</li>\n\n <li>fields: list of fields to compare when matching events against\n tracked events.</li>\n</ul>\n\n<p>\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n</p>\n\n<ul>\n <li>value: uses the field value for comparison.</li>\n\n <li>username: treats the field value as a user name, and ignores any\n host-specific data. This is useful for environment using Kerberos,\n so that only the principal name and realm are compared.</li>\n</ul>\n\n<p>\nThe following is the list of fields that can used to compare HBase events:\n</p>\n<ul>\n <li>allowed: whether the operation was allowed or denied.</li>\n <li>username: the user performing the action.</li>\n <li>scope: the scopeof the operation.</li>\n <li>family: the column family afftected by the operation.</li>\n <li>qualifier: the qualifier the operation.</li>\n <li>action: the action being performed.</li>\n</ul>\n",
"display_name": "Event Tracker",
"name": "navigator_event_tracker",
"value": "{\n \"comment\" : [\n \"Default event tracker for HBase services.\",\n \"Defines equality by comparing username, action, table name, family \",\n \"and qualifier of the events.\"\n ],\n \"timeToLive\" : 60000,\n \"fields\" : [\n { \"type\": \"value\", \"name\" : \"tableName\" },\n { \"type\": \"value\", \"name\" : \"family\" },\n { \"type\": \"value\", \"name\" : \"qualifier\" },\n { \"type\": \"value\", \"name\" : \"operation\" },\n { \"type\": \"username\", \"name\" : \"username\" }\n ]\n}\n"
},
{
"desc": "Action to take when the audit event queue is full. Drop the event or shutdown the affected process.",
"display_name": "Queue Policy",
"name": "navigator_audit_queue_policy",
"value": "DROP"
},
{
"desc": "Maximum number of errors that the HBase Hbck poller will retain through a given run",
"display_name": "HBase Hbck Poller Maximum Error Count",
"name": "hbase_hbck_poller_max_error_count",
"value": "10000"
},
{
"desc": "Timeout for all HBase RPCs in milliseconds.",
"display_name": "RPC Timeout",
"name": "hbase_rpc_timeout",
"value": "60000"
},
{
"desc": "Timeout (in ms) for the distributed log splitting manager to receive response from a worker.",
"display_name": "SplitLog Manager Timeout",
"name": "hbase_service_splitlog_manager_timeout",
"value": "120000"
},
{
"desc": "<p>Event filters are defined in a JSON object like the following:</p>\n\n<pre>\n{\n \"defaultAction\" : (\"accept\", \"discard\"),\n \"rules\" : [\n {\n \"action\" : (\"accept\", \"discard\"),\n \"fields\" : [\n {\n \"name\" : \"fieldName\",\n \"match\" : \"regex\"\n }\n ]\n }\n ]\n}\n</pre>\n\n<p>\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n</p>\n\n<p>\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n</p>\n\n<p>\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n</p>\n\n<p>\nThe following is the list of fields that can be filtered for HBase events:\n</p>\n<ul>\n <li>allowed: whether the operation was allowed or denied.</li>\n <li>username: the user performing the action.</li>\n <li>tableName: the table affected by the operation.</li>\n <li>family: the column family affected by the operation.</li>\n <li>qualifier: the qualifier the operation.</li>\n <li>action: the action being performed.</li>\n</ul>\n",
"display_name": "Event Filter",
"name": "navigator_audit_event_filter",
"value": "{\n \"comment\" : [\n \"Default filter for HBase services.\",\n \"Discards events that affect the internal -ROOT-, .META. and _acl_ tables.\"\n ],\n \"defaultAction\" : \"accept\",\n \"rules\" : [\n {\n \"action\" : \"discard\",\n \"fields\" : [\n { \"name\" : \"tableName\", \"match\" : \"(?:-ROOT-|.META.|_acl_|hbase:meta|hbase:acl)\" }\n ]\n }\n ]\n}\n"
},
{
"desc": "The tolerance window that will be used in HBase service tests that depend on detection of the active HBase Master.",
"display_name": "HBase Active Master Detection Window",
"name": "hbase_active_master_detecton_window",
"value": "3"
},
{
"desc": "An alert is published if the HBase Hbck tool runs slowly.",
"display_name": "HBase Hbck Slow Run Alert Enabled",
"name": "hbase_hbck_slow_run_alert_enabled",
"value": "true"
},
{
"desc": "Name of the HDFS service that this HBase service instance depends on",
"display_name": "HDFS Service",
"name": "hdfs_service",
"value": null
},
{
"desc": "Period of time, in milliseconds, to pause between connection retries to ZooKeeper. Used together with ${zookeeper.retries} in an exponential backoff fashion when making queries to ZooKeeper.",
"display_name": "ZooKeeper Connection Retry Pause Duration",
"name": "zookeeper_pause",
"value": null
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hbase-site.xml</strong>. Applies to configurations of all roles in this service except client configuration.",
"display_name": "HBase Service Advanced Configuration Snippet (Safety Valve) for hbase-site.xml",
"name": "hbase_service_config_safety_valve",
"value": null
},
{
"desc": "Enable SSL encryption for HBase web UIs.",
"display_name": "Web UI SSL Encryption Enabled",
"name": "hbase_hadoop_ssl_enabled",
"value": "false"
},
{
"desc": "The root znode for HBase in ZooKeeper. All of HBase's ZooKeeper files that are configured with a relative path will go under this node. By default, all of HBase's ZooKeeper file paths are configured with a relative path, so they will all go under this directory unless changed.",
"display_name": "ZooKeeper Znode Parent",
"name": "zookeeper_znode_parent",
"value": "/hbase"
},
{
"desc": "AWS access key Id required to access S3 to store remote snapshots.",
"display_name": "AWS S3 access key Id for remote snapshots",
"name": "hbase_snapshot_s3_access_key_id",
"value": null
},
{
"desc": "When computing the overall HBase cluster health, consider the active HBase Master's health.",
"display_name": "Active Master Health Test",
"name": "hbase_master_health_enabled",
"value": "true"
},
{
"desc": "Maximum number of rolled over audit logs to retain. The logs will not be deleted if they contain audit events that have not yet been propagated to Audit Server.",
"display_name": "Number of Audit Logs to Retain",
"name": "navigator_audit_log_max_backup_index",
"value": "10"
},
{
"desc": "Choose the authentication mechanism used by HBase.",
"display_name": "HBase Secure Authentication",
"name": "hbase_security_authentication",
"value": "simple"
},
{
"desc": "AWS secret access key required to access S3 to store remote snapshots.",
"display_name": "AWS S3 secret access key for remote snapshots",
"name": "hbase_snapshot_s3_secret_access_key",
"value": null
},
{
"desc": "The number of times to retry connections to ZooKeeper. Used for reading and writing root region location. Used together with ${zookeeper.pause} in an exponential backoff fashion when making queries to ZooKeeper.",
"display_name": "ZooKeeper Connection Retries",
"name": "zookeeper_retries",
"value": null
},
{
"desc": "Ratio of Lily HBase Indexers used by each HBase RegionServer while doing replication.",
"display_name": "Replication Source Ratio",
"name": "hbase_replication_source_ratio",
"value": "1.0"
},
{
"desc": "Name of the ZooKeeper service that this HBase service instance depends on.",
"display_name": "ZooKeeper Service",
"name": "zookeeper_service",
"value": null
},
{
"desc": "Maximum size of audit log file in MB before it is rolled over.",
"display_name": "Maximum Audit Log File Size",
"name": "navigator_audit_log_max_file_size",
"value": "100"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>core-site.xml</strong>. Applies to configurations of all roles in this service except client configuration.",
"display_name": "HBase Service Advanced Configuration Snippet (Safety Valve) for core-site.xml",
"name": "hbase_core_site_safety_valve",
"value": null
},
{
"desc": "Password for the server keystore file used for encrypted web UIs.",
"display_name": "SSL Server Keystore File Password",
"name": "ssl_server_keystore_password",
"value": null
},
{
"desc": "Size of the threadpool used for hedged reads in hdfs clients. If a read from a block is slow, a parallel 'hedged' read will be started against a different block replica. The first one to return with a result is used while the other one is cancelled. This 'hedged' read feature helps rein in the outliers. A value of zero disables the feature.",
"display_name": "HDFS Hedged Read Threadpool Size",
"name": "hbase_server_dfs_client_hedged_read_threadpool_size",
"value": "0"
},
{
"desc": "Configures whether the Hbck poller checks HDFS or not. Checking HBase tables and regions information on HDFS can take a while.",
"display_name": "HBase Hbck Check HDFS",
"name": "hbase_hbck_poller_check_hdfs",
"value": "true"
},
{
"desc": "If this is set to \"kerberos\", HBase REST Server will authenticate its clients. HBase Proxy User Hosts and Groups should be configured to allow specific users to access HBase through REST Server.",
"display_name": "HBase REST Authentication",
"name": "hbase_restserver_security_authentication",
"value": "simple"
},
{
"desc": "Tables to exclude in the HBase Region Health Canary which will scan a row from every region.",
"display_name": "HBase Region Health Canary Exclude Tables",
"name": "hbase_region_health_canary_exclude_tables",
"value": ""
},
{
"desc": "Specifies the combined maximum allowed size of a KeyValue instance. This option configures an upper boundary for a single entry saved in a storage file. This option prevents a region from splitting if the data is too large. Set this option to a fraction of the maximum region size. To disable this check, use a value of zero or less.",
"display_name": "Maximum Size of HBase Client KeyValue",
"name": "hbase_client_keyvalue_maxsize",
"value": "10485760"
},
{
"desc": "Path to the directory where audit logs will be written. The directory will be created if it doesn't exist.",
"display_name": "Audit Log Directory",
"name": "audit_event_log_dir",
"value": "/var/log/hbase/audit"
},
{
"desc": "If this is set, HBase Thrift Server authenticates its clients. HBase Proxy User Hosts and Groups should be configured to allow specific users to access HBase through Thrift Server.",
"display_name": "HBase Thrift Authentication",
"name": "hbase_thriftserver_security_authentication",
"value": "none"
},
{
"desc": "Enable snapshots. Disabling snapshots requires deletion of all snapshots before restarting the HBase master; the HBase master will not start if snapshots are disabled and snapshots exist.",
"display_name": "Enable Snapshots",
"name": "hbase_snapshot_enabled",
"value": "true"
},
{
"desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.",
"display_name": "HBase Service Environment Advanced Configuration Snippet (Safety Valve)",
"name": "hbase_service_env_safety_valve",
"value": null
},
{
"desc": "Enables the canary that checks HBase region availability by scanning a row from every region.",
"display_name": "HBase Region Health Canary",
"name": "hbase_region_health_canary_enabled",
"value": "true"
},
{
"desc": "Timeout for graceful shutdown of this HBase service. Once this timeout is reached, any remaining running roles are abruptly shutdown. A value of 0 means no timeout.",
"display_name": "Graceful Shutdown Timeout",
"name": "hbase_graceful_stop_timeout",
"value": "180"
},
{
"desc": "An alert is published if the HBase Hbck tool detects at least this many regions with errors across all tables in this service. If the value is not set, alerts will not be published based on the count of regions with errors.",
"display_name": "HBase Hbck Region Error Count Alert Threshold",
"name": "hbase_hbck_alert_region_error_count_threshold",
"value": null
},
{
"desc": "An alert is published if the HBase Hbck tool finds any errors with matching codes. Possible error codes: UNKNOWN, NO_META_REGION, NULL_ROOT_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META, NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED, MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE, FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS, HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION, ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE",
"display_name": "HBase Hbck Alert Error Codes",
"name": "hbase_hbck_alert_error_codes",
"value": "NO_META_REGION,NULL_ROOT_REGION"
},
{
"desc": "AWS S3 path where remote snapshots should be stored.",
"display_name": "AWS S3 path for remote snapshots",
"name": "hbase_snapshot_s3_path",
"value": null
},
{
"desc": "Maximum number of client retries. Used as a maximum for all operations such as fetching of the root region from the root RegionServer, getting a cell's value, and starting a row update.",
"display_name": "Maximum HBase Client Retries",
"name": "hbase_client_retries_number",
"value": "35"
},
{
"desc": "An alert is published if the HBase Hbck tool detects at least this many errors across all tables in this service. Some errors are not associated with a region, e.g. 'RS_CONNECT_FAILURE'. If the value is not set, alerts will not be published based on the count of errors.",
"display_name": "HBase Hbck Error Count Alert Threshold",
"name": "hbase_hbck_alert_error_count_threshold",
"value": null
},
{
"desc": "Enable collection of audit events from the service's roles.",
"display_name": "Enable Collection",
"name": "navigator_audit_enabled",
"value": "true"
},
{
"desc": "The group that this service's processes should run as.",
"display_name": "System Group",
"name": "process_groupname",
"value": "hbase"
},
{
"desc": "Set to true to use HBase Secure RPC Engine for remote procedure calls (RPC). This is only effective in simple authentication mode. Does not provide authentication for RPC calls, but provides user information in the audit logs. Changing this setting requires a restart of this and all dependent services and redeployment of client configurations, along with a restart of the Service Monitor management role.",
"display_name": "HBase Secure RPC Engine",
"name": "hbase_secure_rpc_engine",
"value": "false"
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Service Level Health Alerts",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "The HDFS directory shared by HBase RegionServers.",
"display_name": "HDFS Root Directory",
"name": "hdfs_rootdir",
"value": "/hbase"
},
{
"desc": "Enable HDFS short circuit read. This allows a client co-located with the DataNode to read HDFS file blocks directly. This gives a performance boost to distributed clients that are aware of locality.",
"display_name": "Enable HDFS Short Circuit Read",
"name": "dfs_client_read_shortcircuit",
"value": "true"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>ssl-server.xml</strong>. Applies to configurations of all roles in this service except client configuration.",
"display_name": "HBase Service Advanced Configuration Snippet (Safety Valve) for ssl-server.xml",
"name": "hbase_ssl_server_safety_valve",
"value": null
},
{
"desc": "Password that protects the private key contained in the server keystore used for encrypted web UIs.",
"display_name": "SSL Server Keystore Key Password",
"name": "ssl_server_keystore_keypassword",
"value": null
},
{
"desc": "Path to ZooKeeper Node holding root region location. This is written by the HBase Master and read by clients and RegionServers. If a relative path is given, the parent folder will be ${zookeeper.znode.parent}. By default, the root location is stored at /hbase/root-region-server.",
"display_name": "ZooKeeper Znode Rootserver",
"name": "zookeeper_znode_rootserver",
"value": "root-region-server"
},
{
"desc": "When computing the overall HBase cluster health, consider the health of the backup HBase Masters.",
"display_name": "Backup Masters Health Test",
"name": "hbase_backup_masters_health_enabled",
"value": "true"
},
{
"desc": "For advanced use only, a list of configuration properties that will be used by the Service Monitor instead of the current client configuration for the service.",
"display_name": "Service Monitor Client Config Overrides",
"name": "smon_client_config_overrides",
"value": "<property><name>zookeeper.recovery.retry</name><value>0</value></property><property><name>zookeeper.recovery.retry.intervalmill</name><value>3000</value></property><property><name>hbase.zookeeper.recoverable.waittime</name><value>1000</value></property><property><name>zookeeper.session.timeout</name><value>30000</value></property><property><name>hbase.rpc.timeout</name><value>10000</value></property><property><name>hbase.client.retries.number</name><value>1</value></property><property><name>hbase.client.rpc.maxattempts</name><value>1</value></property><property><name>hbase.client.operation.timeout</name><value>10000</value></property>"
},
{
"desc": "<p>The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific service. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Service Triggers",
"name": "service_triggers",
"value": "[]"
},
{
"desc": "An alert is published if the HBase region health canary runs slowly.",
"display_name": "HBase Region Health Canary Slow Run Alert Enabled",
"name": "hbase_region_health_canary_slow_run_alert_enabled",
"value": "true"
},
{
"desc": "Set to true to cause the hosting server (Master or RegionServer) to abort if a coprocessor throws a Throwable object that is not IOException or a subclass of IOException. Setting it to true might be useful in development environments where one wants to terminate the server as soon as possible to simplify coprocessor failure analysis.",
"display_name": "HBase Coprocessor Abort on Error",
"name": "hbase_coprocessor_abort_on_error",
"value": "false"
},
{
"desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.",
"display_name": "Enable Log Event Capture",
"name": "catch_events",
"value": "true"
},
{
"desc": "Allow HBase tables to be replicated.",
"display_name": "Enable Replication",
"name": "hbase_enable_replication",
"value": "false"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "Path to the keystore file containing the server certificate and private key used for encrypted web UIs.",
"display_name": "SSL Server Keystore File Location",
"name": "ssl_server_keystore_location",
"value": null
},
{
"desc": "Enable HBase authorization.",
"display_name": "HBase Secure Authorization",
"name": "hbase_security_authorization",
"value": "false"
},
{
"desc": "Name of the scheduler pool to use for MR jobs created during export/import of remote snapshots in AWS S3.",
"display_name": "Scheduler pool for remote snapshots in AWS S3.",
"name": "hbase_snapshot_s3_scheduler_pool",
"value": null
},
{
"desc": "Period of time, in milliseconds, to pause between searches for work. Used as a sleep interval by service threads such as a META scanner and log roller.",
"display_name": "HBase Server Thread Wake Frequency",
"name": "hbase_server_thread_wakefrequency",
"value": "10000"
},
{
"desc": "Number of rows to fetch when calling next on a scanner if it is not served from memory. Higher caching values enable faster scanners but require more memory and some calls of next may take longer when the cache is empty.",
"display_name": "HBase Client Scanner Caching",
"name": "hbase_client_scanner_caching",
"value": "100"
},
{
"desc": "An alert is published if the HBase region health canary detects at least this percentage of total regions are unhealthy. This threshold is used if the explicit count is not set via the hbase_canary_alert_unhealthy_region_count_threshold config.",
"display_name": "HBase Canary Unhealthy Region Percentage Alert Threshold",
"name": "hbase_canary_alert_unhealthy_region_percent_threshold",
"value": "0.1"
},
{
"desc": "The health test thresholds of the overall RegionServer health. The check returns \"Concerning\" health if the percentage of \"Healthy\" RegionServers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" RegionServers falls below the critical threshold.",
"display_name": "Healthy RegionServer Monitoring Thresholds",
"name": "hbase_regionservers_healthy_thresholds",
"value": "{\"critical\":\"90.0\",\"warning\":\"95.0\"}"
},
{
"desc": "For advanced use only, a string to be inserted into the client configuration for <strong>navigator.client.properties</strong>.",
"display_name": "HBASE Client Advanced Configuration Snippet (Safety Valve) for navigator.client.properties",
"name": "navigator_client_config_safety_valve",
"value": null
},
{
"desc": "Duration to wait before starting up a 'hedged' read.",
"display_name": "HDFS Hedged Read Delay Threshold",
"name": "hbase_server_dfs_client_hedged_read_threshold_millis",
"value": "500"
},
{
"desc": "Enables the HBase Hbck Poller so that Hbck reports will be available. Enabling the Hbck poller will increase the amount of memory used by the Service Monitor. Consider increasing the Service Monitor Java heap size by an additional 3KB per region. For example, for a cluster with 10,000 regions, increase the JVM heap size by approximately 30MB.",
"display_name": "HBase Hbck Poller",
"name": "hbase_hbck_poller_enabled",
"value": "false"
},
{
"desc": "List of users or groups, who are allowed full privileges, regardless of stored ACLs, across the cluster. Only used when HBase security is enabled.",
"display_name": "HBase Superusers",
"name": "hbase_superuser",
"value": ""
},
{
"desc": "Maximum number of hlog entries to replicate in one go. If this is large, and a consumer takes a while to process the events, the HBase RPC call will time out.",
"display_name": "Replication Batch Size",
"name": "hbase_replication_source_nb_capacity",
"value": "1000"
},
{
"desc": "An alert is published if the HBase region health canary detects at least this many unhealthy regions. This setting takes precedence over the hbase_canary_alert_unhealthy_region_percent_threshold config.",
"display_name": "HBase Canary Unhealthy Region Count Alert Threshold",
"name": "hbase_canary_alert_unhealthy_region_count_threshold",
"value": null
},
{
"desc": "ZooKeeper session timeout in milliseconds. HBase passes this to the ZooKeeper quorum as the suggested maximum time for a session. See http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions The client sends a requested timeout, the server responds with the timeout that it can give the client.",
"display_name": "ZooKeeper Session Timeout",
"name": "zookeeper_session_timeout",
"value": "60000"
},
{
"desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.",
"display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)",
"name": "smon_derived_configs_safety_valve",
"value": null
},
{
"desc": "Write buffer size in bytes. A larger buffer requires more memory on both the client and the server because the server instantiates the passed write buffer to process it but reduces the number of remote procedure calls (RPC). To estimate the amount of server memory used, multiply the value of 'hbase.client.write.buffer' by the value of 'hbase.regionserver.handler.count'.",
"display_name": "HBase Client Write Buffer",
"name": "hbase_client_write_buffer",
"value": "2097152"
},
{
"desc": "The user the management services will impersonate when connecting to HBase. Defaults to 'hbase', a superuser.",
"display_name": "HBase User to Impersonate",
"name": "hbase_user_to_impersonate",
"value": "hbase"
}
]

44
sahara/plugins/cdh/v5_3_0/resources/hdfs-balancer.json Normal file
View File

@ -0,0 +1,44 @@
[
{
"desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Java Configuration Options for Balancer",
"name": "balancer_java_opts",
"value": ""
},
{
"desc": "<p>This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.</p><p>Each rule has some or all of the following fields:</p><ul><li><span class='code'>alert</span> - whether or not events generated from this rule should be promoted to alerts. A value of \"true\" will cause alerts to be generated. If not specified, the default is \"false\".</li><li><span class='code'>rate</span> <strong>(mandatory)</strong> - the maximum number of log messages matching this rule that may be sent as events every minute. If more than <tt>rate</tt> matching log messages are received in a single minute, the extra messages are ignored. If rate is less than 0, the number of messages per minute is unlimited.</li><li><span class='code'>periodminutes</span> - the number of minutes during which the publisher will only publish <tt>rate</tt> events or fewer. If not specified, the default is <strong>one minute</strong></li><li><span class='code'>threshold</span> - apply this rule only to messages with this log4j severity level or above. An example is \"WARN\" for warning level messages or higher.</li><li><span class='code'>content</span> - match only those messages whose contents match this regular expression.</li><li><span class='code'>exceptiontype</span> - match only those messages which are part of an exception message. The exception type must match this regular expression.</li></ul><br/><p>Example:<span class='code'>{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}</span></p><p>This rule will send events to Cloudera Manager for every <span class='code'>StringIndexOutOfBoundsException</span>, up to a maximum of 10 every minute.</p>",
"display_name": "Rules to Extract Events from Log Files",
"name": "log_event_whitelist",
"value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hdfs-site.xml</strong> for this role only.",
"display_name": "Balancer Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml",
"name": "balancer_config_safety_valve",
"value": null
},
{
"desc": "The policy that should be used to rebalance HDFS storage. The default DataNode policy balances the storage at the DataNode level. This is similar to the balancing policy from prior releases. The BlockPool policy balances the storage at the block pool level as well as at the Datanode level. The BlockPool policy is relevant only to a Federated HDFS service.",
"display_name": "Rebalancing Policy",
"name": "rebalancing_policy",
"value": "DataNode"
},
{
"desc": "The percentage deviation from average utilization, after which a node will be rebalanced. (for example, '10.0' for 10%)",
"display_name": "Rebalancing Threshold",
"name": "rebalancer_threshold",
"value": "10.0"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.",
"display_name": "Java Heap Size of Balancer in Bytes",
"name": "balancer_java_heapsize",
"value": "1073741824"
}
]

416
sahara/plugins/cdh/v5_3_0/resources/hdfs-datanode.json Normal file
View File

@ -0,0 +1,416 @@
[
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Log Directory Free Space Monitoring Percentage Thresholds",
"name": "log_directory_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "<p>This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.</p><p>Each rule has some or all of the following fields:</p><ul><li><span class='code'>alert</span> - whether or not events generated from this rule should be promoted to alerts. A value of \"true\" will cause alerts to be generated. If not specified, the default is \"false\".</li><li><span class='code'>rate</span> <strong>(mandatory)</strong> - the maximum number of log messages matching this rule that may be sent as events every minute. If more than <tt>rate</tt> matching log messages are received in a single minute, the extra messages are ignored. If rate is less than 0, the number of messages per minute is unlimited.</li><li><span class='code'>periodminutes</span> - the number of minutes during which the publisher will only publish <tt>rate</tt> events or fewer. If not specified, the default is <strong>one minute</strong></li><li><span class='code'>threshold</span> - apply this rule only to messages with this log4j severity level or above. An example is \"WARN\" for warning level messages or higher.</li><li><span class='code'>content</span> - match only those messages whose contents match this regular expression.</li><li><span class='code'>exceptiontype</span> - match only those messages which are part of an exception message. The exception type must match this regular expression.</li></ul><br/><p>Example:<span class='code'>{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}</span></p><p>This rule will send events to Cloudera Manager for every <span class='code'>StringIndexOutOfBoundsException</span>, up to a maximum of 10 every minute.</p>",
"display_name": "Rules to Extract Events from Log Files",
"name": "log_event_whitelist",
"value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 5, \"content\":\"Datanode registration failed\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Got a command from standby NN - ignoring command:.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n"
},
{
"desc": "Specifies the maximum number of threads to use for transferring data in and out of the DataNode.",
"display_name": "Maximum Number of Transfer Threads",
"name": "dfs_datanode_max_xcievers",
"value": "4096"
},
{
"desc": "Whether or not periodic stacks collection is enabled.",
"display_name": "Stacks Collection Enabled",
"name": "stacks_collection_enabled",
"value": "false"
},
{
"desc": "Comma-separated list of DataNode plug-ins to be activated. If one plug-in cannot be loaded, all the plug-ins are ignored.",
"display_name": "DateNode Plugins",
"name": "dfs_datanode_plugins_list",
"value": ""
},
{
"desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup I/O Weight",
"name": "rm_io_weight",
"value": "500"
},
{
"desc": "In some workloads, the data read from HDFS is known to be significantly large enough that it is unlikely to be useful to cache it in the operating system buffer cache. In this case, the DataNode may be configured to automatically purge all data from the buffer cache after it is delivered to the client. This may improve performance for some workloads by freeing buffer cache spare usage for more cacheable data. This behavior will always be disabled for workloads that read only short sections of a block (e.g HBase random-IO workloads). This property is supported in CDH3u3 or later deployments.",
"display_name": "Enable purging cache after reads",
"name": "dfs_datanode_drop_cache_behind_reads",
"value": "false"
},
{
"desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup CPU Shares",
"name": "rm_cpu_shares",
"value": "1024"
},
{
"desc": "Comma-delimited list of directories on the local file system where the DataNode stores HDFS block data. Typical values are /data/N/dfs/dn for N = 1, 2, 3... These directories should be mounted using the noatime option and the disks should be configured using JBOD. RAID is not recommended.",
"display_name": "DataNode Data Directory",
"name": "dfs_data_dir_list",
"value": null
},
{
"desc": "The health test thresholds for the weighted average extra time the pause monitor spent paused. Specified as a percentage of elapsed wall clock time.",
"display_name": "Pause Duration Thresholds",
"name": "datanode_pause_duration_thresholds",
"value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}"
},
{
"desc": "The number of volumes that are allowed to fail before a DataNode stops offering service. By default, any volume failure will cause a DataNode to shutdown.",
"display_name": "DataNode Failed Volumes Tolerated",
"name": "dfs_datanode_failed_volumes_tolerated",
"value": "0"
},
{
"desc": "In some workloads, the data written to HDFS is known to be significantly large enough that it is unlikely to be useful to cache it in the operating system buffer cache. In this case, the DataNode may be configured to automatically purge all data from the buffer cache after it is written to disk. This may improve performance for some workloads by freeing buffer cache spare usage for more cacheable data. This property is supported in CDH3u3 or later deployments.",
"display_name": "Enable purging cache after writes",
"name": "dfs_datanode_drop_cache_behind_writes",
"value": "false"
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
},
{
"desc": "If enabled, the DataNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.",
"display_name": "Bind DataNode to Wildcard Address",
"name": "dfs_datanode_bind_wildcard",
"value": "false"
},
{
"desc": "The number of server threads for the DataNode.",
"display_name": "Handler Count",
"name": "dfs_datanode_handler_count",
"value": "3"
},
{
"desc": "When computing the overall DataNode health, consider the host's health.",
"display_name": "DataNode Host Health Test",
"name": "datanode_host_health_enabled",
"value": "true"
},
{
"desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
"display_name": "Automatically Restart Process",
"name": "process_auto_restart",
"value": "true"
},
{
"desc": "The maximum number of rolled log files to keep for DataNode logs. Typically used by log4j.",
"display_name": "DataNode Maximum Log File Backups",
"name": "max_log_backup_index",
"value": "10"
},
{
"desc": "<p>The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific role. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Role Triggers",
"name": "role_triggers",
"value": "[]"
},
{
"desc": "Timeout in seconds for the Hue Thrift server running on each DataNode",
"display_name": "Hue Thrift Server Timeout",
"name": "dfs_thrift_timeout",
"value": "60"
},
{
"desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Soft Limit",
"name": "rm_memory_soft_limit",
"value": "-1"
},
{
"desc": "Whether DataNodes should use DataNode hostnames when connecting to DataNodes for data transfer. This property is supported in CDH3u4 or later deployments.",
"display_name": "Use DataNode Hostname",
"name": "dfs_datanode_use_datanode_hostname",
"value": "false"
},
{
"desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Java Configuration Options for DataNode",
"name": "datanode_java_opts",
"value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled"
},
{
"desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.",
"display_name": "File Descriptor Monitoring Thresholds",
"name": "datanode_fd_thresholds",
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Hard Limit",
"name": "rm_memory_hard_limit",
"value": "-1"
},
{
"desc": "The health test thresholds on the duration of the metrics request to the web server.",
"display_name": "Web Metric Collection Duration",
"name": "datanode_web_metric_collection_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}"
},
{
"desc": "Enables the health test that the DataNode's process state is consistent with the role configuration",
"display_name": "DataNode Process Health Test",
"name": "datanode_scm_health_enabled",
"value": "true"
},
{
"desc": "Port for DataNode's XCeiver Protocol. Combined with the DataNode's hostname to build its address.",
"display_name": "DataNode Transceiver Port",
"name": "dfs_datanode_port",
"value": "50010"
},
{
"desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into <strong>hadoop-metrics2.properties</strong>.",
"display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)",
"name": "hadoop_metrics2_safety_valve",
"value": null
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>log4j.properties</strong> for this role only.",
"display_name": "DataNode Logging Advanced Configuration Snippet (Safety Valve)",
"name": "log4j_safety_valve",
"value": null
},
{
"desc": "DataNode Policy for picking which volume should get a new block. The Available Space policy is only available starting with CDH 4.3.",
"display_name": "DataNode Volume Choosing Policy",
"name": "dfs_datanode_volume_choosing_policy",
"value": "org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy"
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.",
"display_name": "Log Directory Free Space Monitoring Absolute Thresholds",
"name": "log_directory_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "Port for the DataNode HTTP web UI. Combined with the DataNode's hostname to build its HTTP address.",
"display_name": "DataNode HTTP Web UI Port",
"name": "dfs_datanode_http_port",
"value": "50075"
},
{
"desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.",
"display_name": "Maximum Process File Descriptors",
"name": "rlimit_fds",
"value": null
},
{
"desc": "Directory where DataNode will place its log files.",
"display_name": "DataNode Log Directory",
"name": "datanode_log_dir",
"value": "/var/log/hadoop-hdfs"
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Health Alerts for this Role",
"name": "enable_alerts",
"value": "false"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hdfs-site.xml</strong> for this role only.",
"display_name": "DataNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml",
"name": "datanode_config_safety_valve",
"value": null
},
{
"desc": "While reading block files, the DataNode can use the posix_fadvise system call to explicitly page data into the operating system buffer cache ahead of the current reader's position. This can improve performance especially when disks are highly contended. This configuration specifies the number of bytes ahead of the current read position which the DataNode will attempt to read ahead. A value of 0 disables this feature. This property is supported in CDH3u3 or later deployments.",
"display_name": "Number of read ahead bytes",
"name": "dfs_datanode_readahead_bytes",
"value": "4194304"
},
{
"desc": "The maximum size, in megabytes, per log file for DataNode logs. Typically used by log4j.",
"display_name": "DataNode Max Log Size",
"name": "max_log_size",
"value": "200"
},
{
"desc": "The maximum amount of memory a DataNode may use to cache data blocks in memory. Setting it to zero will disable caching.",
"display_name": "Maximum Memory Used for Caching",
"name": "dfs_datanode_max_locked_memory",
"value": "4294967296"
},
{
"desc": "Only used when the DataNode Volume Choosing Policy is set to Available Space. Controls how much DataNode volumes are allowed to differ in terms of bytes of free disk space before they are considered imbalanced. If the free space of all the volumes are within this range of each other, the volumes will be considered balanced and block assignments will be done on a pure round robin basis.",
"display_name": "Available Space Policy Balanced Threshold",
"name": "dfs_datanode_available_space_balanced_threshold",
"value": "10737418240"
},
{
"desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.",
"display_name": "Dump Heap When Out of Memory",
"name": "oom_heap_dump_enabled",
"value": "false"
},
{
"desc": "The period to review when computing unexpected exits.",
"display_name": "Unexpected Exits Monitoring Period",
"name": "unexpected_exits_window",
"value": "5"
},
{
"desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.",
"display_name": "Unexpected Exits Thresholds",
"name": "unexpected_exits_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "The health test thresholds of free space in a DataNode. Specified as a percentage of the capacity on the DataNode.",
"display_name": "DataNode Free Space Monitoring Thresholds",
"name": "datanode_free_space_thresholds",
"value": "{\"critical\":\"10.0\",\"warning\":\"20.0\"}"
},
{
"desc": "Port for the various DataNode Protocols. Combined with the DataNode's hostname to build its IPC port address.",
"display_name": "DataNode Protocol Port",
"name": "dfs_datanode_ipc_port",
"value": "50020"
},
{
"desc": "The period to review when computing the moving average of extra time the pause monitor spent paused.",
"display_name": "Pause Duration Monitoring Period",
"name": "datanode_pause_duration_window",
"value": "5"
},
{
"desc": "Minimum number of running threads for the Hue Thrift server running on each DataNode",
"display_name": "Hue Thrift Server Min Threadcount",
"name": "dfs_thrift_threads_min",
"value": "10"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "If this configuration is enabled, the DataNode will instruct the operating system to enqueue all written data to the disk immediately after it is written. This differs from the usual OS policy which may wait for up to 30 seconds before triggering writeback. This may improve performance for some workloads by smoothing the IO profile for data written to disk. This property is supported in CDH3u3 or later deployments.",
"display_name": "Enable immediate enqueuing of data to disk after writes",
"name": "dfs_datanode_sync_behind_writes",
"value": "false"
},
{
"desc": "The health test thresholds of the number of blocks on a DataNode",
"display_name": "DataNode Block Count Thresholds",
"name": "datanode_block_count_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"500000.0\"}"
},
{
"desc": "Permissions for the directories on the local file system where the DataNode stores its blocks. The permissions must be octal. 755 and 700 are typical values.",
"display_name": "DataNode Data Directory Permissions",
"name": "dfs_datanode_data_dir_perm",
"value": "700"
},
{
"desc": "Reserved space in bytes per volume for non Distributed File System (DFS) use.",
"display_name": "Reserved Space for Non DFS Use",
"name": "dfs_datanode_du_reserved",
"value": "10737418240"
},
{
"desc": "The amount of time to wait for the DataNode to fully start up and connect to the NameNode before enforcing the connectivity check.",
"display_name": "DataNode Connectivity Tolerance at Startup",
"name": "datanode_connectivity_tolerance",
"value": "180"
},
{
"desc": "The base port where the secure DataNode web UI listens. Combined with the DataNode's hostname to build its secure web UI address.",
"display_name": "Secure DataNode Web UI Port (SSL)",
"name": "dfs_datanode_https_port",
"value": "50475"
},
{
"desc": "Maximum amount of bandwidth that each DataNode can use for balancing. Specified in bytes per second.",
"display_name": "DataNode Balancing Bandwidth",
"name": "dfs_balance_bandwidthPerSec",
"value": "10485760"
},
{
"desc": "Maximum number of running threads for the Hue Thrift server running on each DataNode",
"display_name": "Hue Thrift Server Max Threadcount",
"name": "dfs_thrift_threads_max",
"value": "20"
},
{
"desc": "The health test thresholds of transceivers usage in a DataNode. Specified as a percentage of the total configured number of transceivers.",
"display_name": "DataNode Transceivers Usage Thresholds",
"name": "datanode_transceivers_usage_thresholds",
"value": "{\"critical\":\"95.0\",\"warning\":\"75.0\"}"
},
{
"desc": "Only used when the DataNode Volume Choosing Policy is set to Available Space. Controls what percentage of new block allocations will be sent to volumes with more available disk space than others. This setting should be in the range 0.0 - 1.0, though in practice 0.5 - 1.0, since there should be no reason to prefer that volumes with less available disk space receive more block allocations.",
"display_name": "Available Space Policy Balanced Preference",
"name": "dfs_datanode_available_space_balanced_preference",
"value": "0.75"
},
{
"desc": "The health test thresholds of failed volumes in a DataNode.",
"display_name": "DataNode Volume Failures Thresholds",
"name": "datanode_volume_failures_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.",
"display_name": "Heap Dump Directory",
"name": "oom_heap_dump_dir",
"value": "/tmp"
},
{
"desc": "The frequency with which stacks will be collected.",
"display_name": "Stacks Collection Frequency",
"name": "stacks_collection_frequency",
"value": "5.0"
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null
},
{
"desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.",
"display_name": "Kill When Out of Memory",
"name": "oom_sigkill_enabled",
"value": "true"
},
{
"desc": "The minimum log level for DataNode logs",
"display_name": "DataNode Logging Threshold",
"name": "log_threshold",
"value": "INFO"
},
{
"desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.",
"display_name": "Web Metric Collection",
"name": "datanode_web_metric_collection_enabled",
"value": "true"
},
{
"desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.",
"display_name": "Java Heap Size of DataNode in Bytes",
"name": "datanode_java_heapsize",
"value": "1073741824"
},
{
"desc": "Enables the health test that verifies the DataNode is connected to the NameNode",
"display_name": "DataNode Connectivity Health Test",
"name": "datanode_connectivity_health_enabled",
"value": "true"
}
]

200
sahara/plugins/cdh/v5_3_0/resources/hdfs-failovercontroller.json Normal file
View File

@ -0,0 +1,200 @@
[
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Log Directory Free Space Monitoring Percentage Thresholds",
"name": "log_directory_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "Directory where Failover Controller will place its log files.",
"display_name": "Failover Controller Log Directory",
"name": "failover_controller_log_dir",
"value": "/var/log/hadoop-hdfs"
},
{
"desc": "<p>This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.</p><p>Each rule has some or all of the following fields:</p><ul><li><span class='code'>alert</span> - whether or not events generated from this rule should be promoted to alerts. A value of \"true\" will cause alerts to be generated. If not specified, the default is \"false\".</li><li><span class='code'>rate</span> <strong>(mandatory)</strong> - the maximum number of log messages matching this rule that may be sent as events every minute. If more than <tt>rate</tt> matching log messages are received in a single minute, the extra messages are ignored. If rate is less than 0, the number of messages per minute is unlimited.</li><li><span class='code'>periodminutes</span> - the number of minutes during which the publisher will only publish <tt>rate</tt> events or fewer. If not specified, the default is <strong>one minute</strong></li><li><span class='code'>threshold</span> - apply this rule only to messages with this log4j severity level or above. An example is \"WARN\" for warning level messages or higher.</li><li><span class='code'>content</span> - match only those messages whose contents match this regular expression.</li><li><span class='code'>exceptiontype</span> - match only those messages which are part of an exception message. The exception type must match this regular expression.</li></ul><br/><p>Example:<span class='code'>{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}</span></p><p>This rule will send events to Cloudera Manager for every <span class='code'>StringIndexOutOfBoundsException</span>, up to a maximum of 10 every minute.</p>",
"display_name": "Rules to Extract Events from Log Files",
"name": "log_event_whitelist",
"value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}"
},
{
"desc": "Whether or not periodic stacks collection is enabled.",
"display_name": "Stacks Collection Enabled",
"name": "stacks_collection_enabled",
"value": "false"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hdfs-site.xml</strong> for this role only.",
"display_name": "Failover Controller Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml",
"name": "fc_config_safety_valve",
"value": null
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
},
{
"desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
"display_name": "Automatically Restart Process",
"name": "process_auto_restart",
"value": "false"
},
{
"desc": "The maximum number of rolled log files to keep for Failover Controller logs. Typically used by log4j.",
"display_name": "Failover Controller Maximum Log File Backups",
"name": "max_log_backup_index",
"value": "10"
},
{
"desc": "<p>The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific role. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Role Triggers",
"name": "role_triggers",
"value": "[]"
},
{
"desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Soft Limit",
"name": "rm_memory_soft_limit",
"value": "-1"
},
{
"desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.",
"display_name": "Java Heap Size of Failover Controller in Bytes",
"name": "failover_controller_java_heapsize",
"value": "268435456"
},
{
"desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Hard Limit",
"name": "rm_memory_hard_limit",
"value": "-1"
},
{
"desc": "The period to review when computing unexpected exits.",
"display_name": "Unexpected Exits Monitoring Period",
"name": "unexpected_exits_window",
"value": "5"
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>log4j.properties</strong> for this role only.",
"display_name": "Failover Controller Logging Advanced Configuration Snippet (Safety Valve)",
"name": "log4j_safety_valve",
"value": null
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.",
"display_name": "Log Directory Free Space Monitoring Absolute Thresholds",
"name": "log_directory_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.",
"display_name": "Maximum Process File Descriptors",
"name": "rlimit_fds",
"value": null
},
{
"desc": "When computing the overall Failover Controller health, consider the host's health.",
"display_name": "Failover Controller Host Health Test",
"name": "failovercontroller_host_health_enabled",
"value": "true"
},
{
"desc": "The maximum size, in megabytes, per log file for Failover Controller logs. Typically used by log4j.",
"display_name": "Failover Controller Max Log Size",
"name": "max_log_size",
"value": "200"
},
{
"desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.",
"display_name": "Kill When Out of Memory",
"name": "oom_sigkill_enabled",
"value": "true"
},
{
"desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.",
"display_name": "File Descriptor Monitoring Thresholds",
"name": "failovercontroller_fd_thresholds",
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.",
"display_name": "Dump Heap When Out of Memory",
"name": "oom_heap_dump_enabled",
"value": "false"
},
{
"desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.",
"display_name": "Unexpected Exits Thresholds",
"name": "unexpected_exits_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Health Alerts for this Role",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "Enables the health test that the Failover Controller's process state is consistent with the role configuration",
"display_name": "Failover Controller Process Health Test",
"name": "failovercontroller_scm_health_enabled",
"value": "true"
},
{
"desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Java Configuration Options for Failover Controller",
"name": "failover_controller_java_opts",
"value": ""
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null
},
{
"desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup I/O Weight",
"name": "rm_io_weight",
"value": "500"
},
{
"desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup CPU Shares",
"name": "rm_cpu_shares",
"value": "1024"
},
{
"desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.",
"display_name": "Heap Dump Directory",
"name": "oom_heap_dump_dir",
"value": "/tmp"
},
{
"desc": "The frequency with which stacks will be collected.",
"display_name": "Stacks Collection Frequency",
"name": "stacks_collection_frequency",
"value": "5.0"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "The minimum log level for Failover Controller logs",
"display_name": "Failover Controller Logging Threshold",
"name": "log_threshold",
"value": "INFO"
}
]

56
sahara/plugins/cdh/v5_3_0/resources/hdfs-gateway.json Normal file
View File

@ -0,0 +1,56 @@
[
{
"desc": "For advanced use only, a string to be inserted into the client configuration for <strong>hdfs-site.xml</strong>.",
"display_name": "HDFS Client Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml",
"name": "hdfs_client_config_safety_valve",
"value": null
},
{
"desc": "The directory where the client configs will be deployed",
"display_name": "Deploy Directory",
"name": "client_config_root_dir",
"value": "/etc/hadoop"
},
{
"desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the client configuration for <strong>hadoop-env.sh</strong>",
"display_name": "HDFS Client Environment Advanced Configuration Snippet for hadoop-env.sh (Safety Valve)",
"name": "hdfs_client_env_safety_valve",
"value": null
},
{
"desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.",
"display_name": "Alternatives Priority",
"name": "client_config_priority",
"value": "90"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "Maximum size in bytes for the Java process heap memory. Passed to Java -Xmx.",
"display_name": "Client Java Heap Size in Bytes",
"name": "hdfs_client_java_heapsize",
"value": "268435456"
},
{
"desc": "Enable HDFS short circuit read. This allows a client co-located with the DataNode to read HDFS file blocks directly. This gives a performance boost to distributed clients that are aware of locality.",
"display_name": "Enable HDFS Short Circuit Read",
"name": "dfs_client_read_shortcircuit",
"value": "false"
},
{
"desc": "These are Java command line arguments. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Client Java Configuration Options",
"name": "hbase_client_java_opts",
"value": "-Djava.net.preferIPv4Stack=true"
},
{
"desc": "Move deleted files to the trash so that they can be recovered if necessary. This client side configuration takes effect only if the HDFS service-wide trash is disabled (NameNode Filesystem Trash Interval set to 0) and is ignored otherwise. The trash is not automatically emptied when enabled with this configuration.",
"display_name": "Use Trash",
"name": "dfs_client_use_trash",
"value": "false"
}
]

242
sahara/plugins/cdh/v5_3_0/resources/hdfs-httpfs.json Normal file
View File

@ -0,0 +1,242 @@
[
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Log Directory Free Space Monitoring Percentage Thresholds",
"name": "log_directory_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.",
"display_name": "File Descriptor Monitoring Thresholds",
"name": "httpfs_fd_thresholds",
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "Whether or not periodic stacks collection is enabled.",
"display_name": "Stacks Collection Enabled",
"name": "stacks_collection_enabled",
"value": "false"
},
{
"desc": "The user that the HttpFS server process should run as.",
"display_name": "System User",
"name": "httpfs_process_username",
"value": "httpfs"
},
{
"desc": "The group that the HttpFS server process should run as.",
"display_name": "System Group",
"name": "httpfs_process_groupname",
"value": "httpfs"
},
{
"desc": "When computing the overall HttpFS health, consider the host's health.",
"display_name": "HttpFS Host Health Test",
"name": "httpfs_host_health_enabled",
"value": "true"
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
},
{
"desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
"display_name": "Automatically Restart Process",
"name": "process_auto_restart",
"value": "false"
},
{
"desc": "The maximum number of rolled log files to keep for HttpFS logs. Typically used by log4j.",
"display_name": "HttpFS Maximum Log File Backups",
"name": "max_log_backup_index",
"value": "10"
},
{
"desc": "<p>The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific role. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Role Triggers",
"name": "role_triggers",
"value": "[]"
},
{
"desc": "Enables the health test that the HttpFS's process state is consistent with the role configuration",
"display_name": "HttpFS Process Health Test",
"name": "httpfs_scm_health_enabled",
"value": "true"
},
{
"desc": "Password of the keystore used by HttpFS role for SSL.",
"display_name": "HttpFS Keystore Password",
"name": "httpfs_https_keystore_password",
"value": null
},
{
"desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Soft Limit",
"name": "rm_memory_soft_limit",
"value": "-1"
},
{
"desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Hard Limit",
"name": "rm_memory_hard_limit",
"value": "-1"
},
{
"desc": "The period to review when computing unexpected exits.",
"display_name": "Unexpected Exits Monitoring Period",
"name": "unexpected_exits_window",
"value": "5"
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
},
{
"desc": "Directory where HttpFS will place its log files.",
"display_name": "HttpFS Log Directory",
"name": "httpfs_log_dir",
"value": "/var/log/hadoop-httpfs"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>log4j.properties</strong> for this role only.",
"display_name": "HttpFS Logging Advanced Configuration Snippet (Safety Valve)",
"name": "log4j_safety_valve",
"value": null
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.",
"display_name": "Log Directory Free Space Monitoring Absolute Thresholds",
"name": "log_directory_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.",
"display_name": "Maximum Process File Descriptors",
"name": "rlimit_fds",
"value": null
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Health Alerts for this Role",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>httpfs-site.xml</strong> for this role only.",
"display_name": "HttpFS Advanced Configuration Snippet (Safety Valve) for httpfs-site.xml",
"name": "httpfs_config_safety_valve",
"value": null
},
{
"desc": "The maximum size, in megabytes, per log file for HttpFS logs. Typically used by log4j.",
"display_name": "HttpFS Max Log Size",
"name": "max_log_size",
"value": "200"
},
{
"desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.",
"display_name": "Kill When Out of Memory",
"name": "oom_sigkill_enabled",
"value": "true"
},
{
"desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Java Configuration Options for HttpFS",
"name": "httpfs_java_opts",
"value": ""
},
{
"desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.",
"display_name": "Dump Heap When Out of Memory",
"name": "oom_heap_dump_enabled",
"value": "false"
},
{
"desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.",
"display_name": "Unexpected Exits Thresholds",
"name": "unexpected_exits_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "The secret to use for signing client authentication tokens.",
"display_name": "Signature Secret",
"name": "hdfs_httpfs_signature_secret",
"value": "hadoop httpfs secret"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "Location of the keystore file used by HttpFS role for SSL.",
"display_name": "HttpFS Keystore File",
"name": "httpfs_https_keystore_file",
"value": "/var/run/hadoop-httpfs/.keystore"
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null
},
{
"desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.",
"display_name": "Java Heap Size of HttpFS in Bytes",
"name": "httpfs_java_heapsize",
"value": "268435456"
},
{
"desc": "The port for the administration interface.",
"display_name": "Administration Port",
"name": "hdfs_httpfs_admin_port",
"value": "14001"
},
{
"desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup I/O Weight",
"name": "rm_io_weight",
"value": "500"
},
{
"desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup CPU Shares",
"name": "rm_cpu_shares",
"value": "1024"
},
{
"desc": "The HTTP port where the REST interface to HDFS is available.",
"display_name": "HTTP Port",
"name": "hdfs_httpfs_http_port",
"value": "14000"
},
{
"desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.",
"display_name": "Heap Dump Directory",
"name": "oom_heap_dump_dir",
"value": "/tmp"
},
{
"desc": "The frequency with which stacks will be collected.",
"display_name": "Stacks Collection Frequency",
"name": "stacks_collection_frequency",
"value": "5.0"
},
{
"desc": "The minimum log level for HttpFS logs",
"display_name": "HttpFS Logging Threshold",
"name": "log_threshold",
"value": "INFO"
},
{
"desc": "Use SSL for HttpFS.",
"display_name": "Use SSL",
"name": "httpfs_use_ssl",
"value": "false"
}
]

284
sahara/plugins/cdh/v5_3_0/resources/hdfs-journalnode.json Normal file
View File

@ -0,0 +1,284 @@
[
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Log Directory Free Space Monitoring Percentage Thresholds",
"name": "log_directory_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "<p>This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.</p><p>Each rule has some or all of the following fields:</p><ul><li><span class='code'>alert</span> - whether or not events generated from this rule should be promoted to alerts. A value of \"true\" will cause alerts to be generated. If not specified, the default is \"false\".</li><li><span class='code'>rate</span> <strong>(mandatory)</strong> - the maximum number of log messages matching this rule that may be sent as events every minute. If more than <tt>rate</tt> matching log messages are received in a single minute, the extra messages are ignored. If rate is less than 0, the number of messages per minute is unlimited.</li><li><span class='code'>periodminutes</span> - the number of minutes during which the publisher will only publish <tt>rate</tt> events or fewer. If not specified, the default is <strong>one minute</strong></li><li><span class='code'>threshold</span> - apply this rule only to messages with this log4j severity level or above. An example is \"WARN\" for warning level messages or higher.</li><li><span class='code'>content</span> - match only those messages whose contents match this regular expression.</li><li><span class='code'>exceptiontype</span> - match only those messages which are part of an exception message. The exception type must match this regular expression.</li></ul><br/><p>Example:<span class='code'>{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}</span></p><p>This rule will send events to Cloudera Manager for every <span class='code'>StringIndexOutOfBoundsException</span>, up to a maximum of 10 every minute.</p>",
"display_name": "Rules to Extract Events from Log Files",
"name": "log_event_whitelist",
"value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}"
},
{
"desc": "The health test thresholds on the duration of the metrics request to the web server.",
"display_name": "Web Metric Collection Duration",
"name": "journalnode_web_metric_collection_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}"
},
{
"desc": "Whether or not periodic stacks collection is enabled.",
"display_name": "Stacks Collection Enabled",
"name": "stacks_collection_enabled",
"value": "false"
},
{
"desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.",
"display_name": "Java Heap Size of JournalNode in Bytes",
"name": "journalNode_java_heapsize",
"value": "268435456"
},
{
"desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.",
"display_name": "Web Metric Collection",
"name": "journalnode_web_metric_collection_enabled",
"value": "true"
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
},
{
"desc": "Enables the health test that the JournalNode's process state is consistent with the role configuration",
"display_name": "JournalNode Process Health Test",
"name": "journalnode_scm_health_enabled",
"value": "true"
},
{
"desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
"display_name": "Automatically Restart Process",
"name": "process_auto_restart",
"value": "true"
},
{
"desc": "The maximum number of rolled log files to keep for JournalNode logs. Typically used by log4j.",
"display_name": "JournalNode Maximum Log File Backups",
"name": "max_log_backup_index",
"value": "10"
},
{
"desc": "The health check thresholds for monitoring of free space on the filesystem that contains the JournalNode's edits directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Edits Directory Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Edits Directory Free Space Monitoring Percentage Thresholds",
"name": "journalnode_edits_directory_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Soft Limit",
"name": "rm_memory_soft_limit",
"value": "-1"
},
{
"desc": "Directory on the local file system where NameNode edits are written.",
"display_name": "JournalNode Edits Directory",
"name": "dfs_journalnode_edits_dir",
"value": null
},
{
"desc": "The period to review when computing the moving average of garbage collection time.",
"display_name": "Garbage Collection Duration Monitoring Period",
"name": "journalnode_gc_duration_window",
"value": "5"
},
{
"desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Hard Limit",
"name": "rm_memory_hard_limit",
"value": "-1"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hdfs-site.xml</strong> for this role only.",
"display_name": "JournalNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml",
"name": "jn_config_safety_valve",
"value": null
},
{
"desc": "The period to review when computing unexpected exits.",
"display_name": "Unexpected Exits Monitoring Period",
"name": "unexpected_exits_window",
"value": "5"
},
{
"desc": "When computing the overall JournalNode health, consider the host's health.",
"display_name": "JournalNode Host Health Test",
"name": "journalnode_host_health_enabled",
"value": "true"
},
{
"desc": "The amount of time at JournalNode startup allowed for the active NameNode to get in sync with the JournalNode.",
"display_name": "Active NameNode Sync Status Startup Tolerance",
"name": "journalnode_sync_status_startup_tolerance",
"value": "180"
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
},
{
"desc": "The base port where the secure JournalNode web UI listens. Combined with the JournalNode's hostname to build its secure web UI address.",
"display_name": "Secure JournalNode Web UI Port (SSL)",
"name": "dfs_journalnode_https_port",
"value": "8481"
},
{
"desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.",
"display_name": "File Descriptor Monitoring Thresholds",
"name": "journalnode_fd_thresholds",
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>log4j.properties</strong> for this role only.",
"display_name": "JournalNode Logging Advanced Configuration Snippet (Safety Valve)",
"name": "log4j_safety_valve",
"value": null
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.",
"display_name": "Log Directory Free Space Monitoring Absolute Thresholds",
"name": "log_directory_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "The health test thresholds for JournalNode fsync latency",
"display_name": "JournalNode Fsync Latency Thresholds",
"name": "journalnode_fsync_latency_thresholds",
"value": "{\"critical\":\"3000.0\",\"warning\":\"1000.0\"}"
},
{
"desc": "Enables the health check that verifies the active NameNode's sync status to the JournalNode",
"display_name": "Active NameNode Sync Status Health Check",
"name": "journalnode_sync_status_enabled",
"value": "true"
},
{
"desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.",
"display_name": "Maximum Process File Descriptors",
"name": "rlimit_fds",
"value": null
},
{
"desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.",
"display_name": "Garbage Collection Duration Thresholds",
"name": "journalnode_gc_duration_thresholds",
"value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}"
},
{
"desc": "Port for the JournalNode's RPC. Combined with the JournalNode's hostname to build its RPC address.",
"display_name": "JournalNode RPC Port",
"name": "dfs_journalnode_rpc_port",
"value": "8485"
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Health Alerts for this Role",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "The maximum size, in megabytes, per log file for JournalNode logs. Typically used by log4j.",
"display_name": "JournalNode Max Log Size",
"name": "max_log_size",
"value": "200"
},
{
"desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.",
"display_name": "Kill When Out of Memory",
"name": "oom_sigkill_enabled",
"value": "true"
},
{
"desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.",
"display_name": "Dump Heap When Out of Memory",
"name": "oom_heap_dump_enabled",
"value": "false"
},
{
"desc": "If enabled, the JournalNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.",
"display_name": "Bind JournalNode to Wildcard Address",
"name": "journalnode_bind_wildcard",
"value": "false"
},
{
"desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.",
"display_name": "Unexpected Exits Thresholds",
"name": "unexpected_exits_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "The health check thresholds for monitoring of free space on the filesystem that contains the JournalNode's edits directory.",
"display_name": "Edits Directory Free Space Monitoring Absolute Thresholds",
"name": "journalnode_edits_directory_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "<p>The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific role. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Role Triggers",
"name": "role_triggers",
"value": "[]"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "Directory where JournalNode will place its log files.",
"display_name": "JournalNode Log Directory",
"name": "journalnode_log_dir",
"value": "/var/log/hadoop-hdfs"
},
{
"desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Java Configuration Options for JournalNode",
"name": "journalNode_java_opts",
"value": ""
},
{
"desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup I/O Weight",
"name": "rm_io_weight",
"value": "500"
},
{
"desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup CPU Shares",
"name": "rm_cpu_shares",
"value": "1024"
},
{
"desc": "Port for the JournalNode HTTP web UI. Combined with the JournalNode hostname to build its HTTP address.",
"display_name": "JournalNode HTTP Port",
"name": "dfs_journalnode_http_port",
"value": "8480"
},
{
"desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.",
"display_name": "Heap Dump Directory",
"name": "oom_heap_dump_dir",
"value": "/tmp"
},
{
"desc": "The frequency with which stacks will be collected.",
"display_name": "Stacks Collection Frequency",
"name": "stacks_collection_frequency",
"value": "5.0"
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null
},
{
"desc": "The minimum log level for JournalNode logs",
"display_name": "JournalNode Logging Threshold",
"name": "log_threshold",
"value": "INFO"
}
]

488
sahara/plugins/cdh/v5_3_0/resources/hdfs-namenode.json Normal file
View File

@ -0,0 +1,488 @@
[
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Log Directory Free Space Monitoring Percentage Thresholds",
"name": "log_directory_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.",
"display_name": "Health Check Startup Tolerance",
"name": "namenode_startup_tolerance",
"value": "5"
},
{
"desc": "The health test thresholds of failed status directories in a NameNode.",
"display_name": "NameNode Directory Failures Thresholds",
"name": "namenode_directory_failures_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "Name of the journal located on each JournalNode filesystem.",
"display_name": "Quorum-based Storage Journal name",
"name": "dfs_namenode_quorum_journal_name",
"value": null
},
{
"desc": "Whether or not periodic stacks collection is enabled.",
"display_name": "Stacks Collection Enabled",
"name": "stacks_collection_enabled",
"value": "false"
},
{
"desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Java Configuration Options for NameNode",
"name": "namenode_java_opts",
"value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled"
},
{
"desc": "The number of server threads for the NameNode.",
"display_name": "NameNode Handler Count",
"name": "dfs_namenode_handler_count",
"value": "30"
},
{
"desc": "Enables the health test that the NameNode is not in safemode",
"display_name": "NameNode Safemode Health Test",
"name": "namenode_safe_mode_enabled",
"value": "true"
},
{
"desc": "<p>This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.</p><p>Each rule has some or all of the following fields:</p><ul><li><span class='code'>alert</span> - whether or not events generated from this rule should be promoted to alerts. A value of \"true\" will cause alerts to be generated. If not specified, the default is \"false\".</li><li><span class='code'>rate</span> <strong>(mandatory)</strong> - the maximum number of log messages matching this rule that may be sent as events every minute. If more than <tt>rate</tt> matching log messages are received in a single minute, the extra messages are ignored. If rate is less than 0, the number of messages per minute is unlimited.</li><li><span class='code'>periodminutes</span> - the number of minutes during which the publisher will only publish <tt>rate</tt> events or fewer. If not specified, the default is <strong>one minute</strong></li><li><span class='code'>threshold</span> - apply this rule only to messages with this log4j severity level or above. An example is \"WARN\" for warning level messages or higher.</li><li><span class='code'>content</span> - match only those messages whose contents match this regular expression.</li><li><span class='code'>exceptiontype</span> - match only those messages which are part of an exception message. The exception type must match this regular expression.</li></ul><br/><p>Example:<span class='code'>{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}</span></p><p>This rule will send events to Cloudera Manager for every <span class='code'>StringIndexOutOfBoundsException</span>, up to a maximum of 10 every minute.</p>",
"display_name": "Rules to Extract Events from Log Files",
"name": "log_event_whitelist",
"value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"},\n {\"alert\": false, \"rate\": 1, \"threshold\":\"INFO\", \"content\":\"Triggering checkpoint.*\"}\n ]\n}\n"
},
{
"desc": "The base port where the DFS NameNode web UI listens. If the port number is 0, then the server starts on a free port. Combined with the NameNode's hostname to build its HTTP address.",
"display_name": "NameNode Web UI Port",
"name": "dfs_http_port",
"value": "50070"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>dfs_hosts_exclude.txt</strong> for this role only.",
"display_name": "NameNode Advanced Configuration Snippet (Safety Valve) for dfs_hosts_exclude.txt",
"name": "namenode_hosts_exclude_safety_valve",
"value": null
},
{
"desc": "Mountpoints that are mapped to this NameNode's Nameservice.",
"display_name": "Mountpoints",
"name": "nameservice_mountpoints",
"value": "/"
},
{
"desc": "The health test thresholds of the age of the HDFS namespace checkpoint. Specified as a percentage of the configured checkpoint interval.",
"display_name": "Filesystem Checkpoint Age Monitoring Thresholds",
"name": "namenode_checkpoint_age_thresholds",
"value": "{\"critical\":\"400.0\",\"warning\":\"200.0\"}"
},
{
"desc": "The base port where the secure NameNode web UI listens.",
"display_name": "Secure NameNode Web UI Port (SSL)",
"name": "dfs_https_port",
"value": "50470"
},
{
"desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.",
"display_name": "Java Heap Size of Namenode in Bytes",
"name": "namenode_java_heapsize",
"value": "1073741824"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>log4j.properties</strong> for this role only.",
"display_name": "NameNode Logging Advanced Configuration Snippet (Safety Valve)",
"name": "log4j_safety_valve",
"value": null
},
{
"desc": "This determines the total amount of block transfers to begin in parallel at a DataNode for replication, when such a command list is being sent over a DataNode heartbeat by the NameNode. The actual number is obtained by multiplying this value by the total number of live nodes in the cluster. The result number is the number of blocks to transfer immediately, per DataNode heartbeat.",
"display_name": "Replication Work Multiplier Per Iteration",
"name": "dfs_namenode_replication_work_multiplier_per_iteration",
"value": "2"
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
},
{
"desc": "The health test thresholds of the number of transactions since the last HDFS namespace checkpoint. Specified as a percentage of the configured checkpointing transaction limit.",
"display_name": "Filesystem Checkpoint Transactions Monitoring Thresholds",
"name": "namenode_checkpoint_transactions_thresholds",
"value": "{\"critical\":\"400.0\",\"warning\":\"200.0\"}"
},
{
"desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
"display_name": "Automatically Restart Process",
"name": "process_auto_restart",
"value": "false"
},
{
"desc": "Determines extension of safemode in milliseconds after the threshold level is reached.",
"display_name": "Safemode Extension",
"name": "dfs_safemode_extension",
"value": "30000"
},
{
"desc": "Determines where on the local file system the NameNode should store the name table (fsimage). For redundancy, enter a comma-delimited list of directories to replicate the name table in all of the directories. Typical values are /data/N/dfs/nn where N=1..3.",
"display_name": "NameNode Data Directories",
"name": "dfs_name_dir_list",
"value": null
},
{
"desc": "Timeout in seconds for the Hue Thrift server running on the NameNode",
"display_name": "Hue Thrift Server Timeout",
"name": "dfs_thrift_timeout",
"value": "60"
},
{
"desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Soft Limit",
"name": "rm_memory_soft_limit",
"value": "-1"
},
{
"desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.",
"display_name": "Dump Heap When Out of Memory",
"name": "oom_heap_dump_enabled",
"value": "false"
},
{
"desc": "The period to review when computing the moving average of extra time the pause monitor spent paused.",
"display_name": "Pause Duration Monitoring Period",
"name": "namenode_pause_duration_window",
"value": "5"
},
{
"desc": "The health check thresholds of the NameNode's RPC latency.",
"display_name": "NameNode RPC Latency Thresholds",
"name": "namenode_rpc_latency_thresholds",
"value": "{\"critical\":\"5000.0\",\"warning\":\"1000.0\"}"
},
{
"desc": "The health test thresholds on the duration of the metrics request to the web server.",
"display_name": "Web Metric Collection Duration",
"name": "namenode_web_metric_collection_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}"
},
{
"desc": "The number of server threads for the NameNode used for service calls. Only used when NameNode Service RPC Port is configured.",
"display_name": "NameNode Service Handler Count",
"name": "dfs_namenode_service_handler_count",
"value": "30"
},
{
"desc": "Full path to a custom topology script on the host file system. The topology script is used to determine the rack location of nodes. If left blank, a topology script will be provided that uses your hosts' rack information, visible in the \"Hosts\" page.",
"display_name": "Topology Script File Name",
"name": "topology_script_file_name",
"value": null
},
{
"desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.",
"display_name": "Web Metric Collection",
"name": "namenode_web_metric_collection_enabled",
"value": "true"
},
{
"desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Hard Limit",
"name": "rm_memory_hard_limit",
"value": "-1"
},
{
"desc": "The time between two periodic file system checkpoints.",
"display_name": "Filesystem Checkpoint Period",
"name": "fs_checkpoint_period",
"value": "3600"
},
{
"desc": "Specifies the number of DataNodes that must be live before the name node exits safemode. Enter a value less than or equal to 0 to take the number of live DataNodes into account when deciding whether to remain in safemode during startup. Values greater than the number of DataNodes in the cluster will make safemode permanent.",
"display_name": "Safemode Minimum DataNodes",
"name": "dfs_safemode_min_datanodes",
"value": "0"
},
{
"desc": "The port where the NameNode runs the HDFS protocol. Combined with the NameNode's hostname to build its address.",
"display_name": "NameNode Port",
"name": "namenode_port",
"value": "8020"
},
{
"desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.",
"display_name": "Heap Dump Directory",
"name": "oom_heap_dump_dir",
"value": "/tmp"
},
{
"desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup I/O Weight",
"name": "rm_io_weight",
"value": "500"
},
{
"desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into <strong>hadoop-metrics2.properties</strong>.",
"display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)",
"name": "hadoop_metrics2_safety_valve",
"value": null
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hdfs-site.xml</strong> for this role only.",
"display_name": "NameNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml",
"name": "namenode_config_safety_valve",
"value": null
},
{
"desc": "The frequency with which stacks will be collected.",
"display_name": "Stacks Collection Frequency",
"name": "stacks_collection_frequency",
"value": "5.0"
},
{
"desc": "The period to review when computing the moving average of the NameNode's RPC latency.",
"display_name": "NameNode RPC Latency Monitoring Window",
"name": "namenode_rpc_latency_window",
"value": "5"
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.",
"display_name": "Log Directory Free Space Monitoring Absolute Thresholds",
"name": "log_directory_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "Optional port for the service-rpc address which can be used by HDFS daemons instead of sharing the RPC address used by the clients.",
"display_name": "NameNode Service RPC Port",
"name": "dfs_namenode_servicerpc_address",
"value": null
},
{
"desc": "The health test thresholds for the weighted average extra time the pause monitor spent paused. Specified as a percentage of elapsed wall clock time.",
"display_name": "Pause Duration Thresholds",
"name": "namenode_pause_duration_thresholds",
"value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}"
},
{
"desc": "When computing the overall NameNode health, consider the host's health.",
"display_name": "NameNode Host Health Test",
"name": "namenode_host_health_enabled",
"value": "true"
},
{
"desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.",
"display_name": "Maximum Process File Descriptors",
"name": "rlimit_fds",
"value": null
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Health Alerts for this Role",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "The maximum size, in megabytes, per log file for NameNode logs. Typically used by log4j.",
"display_name": "NameNode Max Log Size",
"name": "max_log_size",
"value": "200"
},
{
"desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.",
"display_name": "Kill When Out of Memory",
"name": "oom_sigkill_enabled",
"value": "true"
},
{
"desc": "Enables the health test that the NameNode's process state is consistent with the role configuration",
"display_name": "NameNode Process Health Test",
"name": "namenode_scm_health_enabled",
"value": "true"
},
{
"desc": "The health check thresholds for the number of out-of-sync JournalNodes for this NameNode.",
"display_name": "NameNode Out-Of-Sync JournalNodes Thresholds",
"name": "namenode_out_of_sync_journal_nodes_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "The period to review when computing unexpected exits.",
"display_name": "Unexpected Exits Monitoring Period",
"name": "unexpected_exits_window",
"value": "5"
},
{
"desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.",
"display_name": "Unexpected Exits Thresholds",
"name": "unexpected_exits_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>dfs_hosts_allow.txt</strong> for this role only.",
"display_name": "NameNode Advanced Configuration Snippet (Safety Valve) for dfs_hosts_allow.txt",
"name": "namenode_hosts_allow_safety_valve",
"value": null
},
{
"desc": "Number of minutes between trash checkpoints. Also controls the number of minutes after which a trash checkpoint directory is deleted. To disable the trash feature, enter 0.",
"display_name": "Filesystem Trash Interval",
"name": "fs_trash_interval",
"value": "1440"
},
{
"desc": "The access time for HDFS file is precise upto this value. Setting the value of 0 disables access times for HDFS. When using the NFS Gateway role, make sure this property is enabled.",
"display_name": "Access Time Precision",
"name": "dfs_access_time_precision",
"value": "3600000"
},
{
"desc": "<p>The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific role. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Role Triggers",
"name": "role_triggers",
"value": "[]"
},
{
"desc": "Minimum number of running threads for the Hue Thrift server running on the NameNode",
"display_name": "Hue Thrift Server Min Threadcount",
"name": "dfs_thrift_threads_min",
"value": "10"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "If enabled, the NameNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.",
"display_name": "Bind NameNode to Wildcard Address",
"name": "namenode_bind_wildcard",
"value": "false"
},
{
"desc": "Comma-separated list of NameNode plug-ins to be activated. If one plug-in cannot be loaded, all the plug-ins are ignored.",
"display_name": "NameNode Plugins",
"name": "dfs_namenode_plugins_list",
"value": ""
},
{
"desc": "Directory on a shared storage device, such as a Quorum-based Storage URI or a local directory that is an NFS mount from a NAS, to store the NameNode edits. The value of this configuration is automatically generated to be the Quourm Journal URI if there are JournalNodes and this NameNode is Highly Available.",
"display_name": "Shared Edits Directory",
"name": "dfs_namenode_shared_edits_dir",
"value": null
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null
},
{
"desc": "Enables the health test of the rolling metadata upgrade status of the NameNode. This covers rolling metadata upgrades. Non-rolling metadata upgrades are covered in a separate health test.",
"display_name": "HDFS Rolling Metadata Upgrade Status Health Test",
"name": "namenode_rolling_upgrade_status_enabled",
"value": "true"
},
{
"desc": "This determines the percentage amount of block invalidations (deletes) to do over a single DataNode heartbeat deletion command. The final deletion count is determined by applying this percentage to the number of live nodes in the system. The resultant number is the number of blocks from the deletion list chosen for proper invalidation over a single heartbeat of a single DataNode.",
"display_name": "Invalidate Work Percentage Per Iteration",
"name": "dfs_namenode_invalidate_work_pct_per_iteration",
"value": "0.32"
},
{
"desc": "Enable Automatic Failover to maintain High Availability. Requires a ZooKeeper service and a High Availability NameNode partner.",
"display_name": "Enable Automatic Failover",
"name": "autofailover_enabled",
"value": "false"
},
{
"desc": "The maximum number of rolled log files to keep for NameNode logs. Typically used by log4j.",
"display_name": "NameNode Maximum Log File Backups",
"name": "max_log_backup_index",
"value": "10"
},
{
"desc": "Maximum number of running threads for the Hue Thrift server running on the NameNode",
"display_name": "Hue Thrift Server Max Threadcount",
"name": "dfs_thrift_threads_max",
"value": "20"
},
{
"desc": "Directory where NameNode will place its log files.",
"display_name": "NameNode Log Directory",
"name": "namenode_log_dir",
"value": "/var/log/hadoop-hdfs"
},
{
"desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup CPU Shares",
"name": "rm_cpu_shares",
"value": "1024"
},
{
"desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.",
"display_name": "File Descriptor Monitoring Thresholds",
"name": "namenode_fd_thresholds",
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "Enables the health test of the metadata upgrade status of the NameNode. This covers non-rolling metadata upgrades. Rolling metadata upgrades are covered in a separate health test.",
"display_name": "HDFS Metadata Upgrade Status Health Test",
"name": "namenode_upgrade_status_enabled",
"value": "true"
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystems that contain this role's data directories. Specified as a percentage of the capacity on the filesystem. This setting is not used if a Data Directories Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Data Directories Free Space Monitoring Percentage Thresholds",
"name": "namenode_data_directories_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "Nameservice of this NameNode. The Nameservice represents the interface to this NameNode and its High Availability partner. The Nameservice also represents the namespace associated with a federated NameNode.",
"display_name": "NameNode Nameservice",
"name": "dfs_federation_namenode_nameservice",
"value": null
},
{
"desc": "The number of transactions after which the NameNode or SecondaryNameNode will create a checkpoint of the namespace, regardless of whether the checkpoint period has expired.",
"display_name": "Filesystem Checkpoint Transaction Threshold",
"name": "fs_checkpoint_txns",
"value": "1000000"
},
{
"desc": "The minimum log level for NameNode logs",
"display_name": "NameNode Logging Threshold",
"name": "log_threshold",
"value": "INFO"
},
{
"desc": "Specifies the percentage of blocks that should satisfy the minimal replication requirement defined by dfs.replication.min. Enter a value less than or equal to 0 to wait for any particular percentage of blocks before exiting safemode. Values greater than 1 will make safemode permanent.",
"display_name": "Safemode Threshold Percentage",
"name": "dfs_safemode_threshold_pct",
"value": "0.999"
},
{
"desc": "Directories on the local file system to store the NameNode edits. If not set, the edits are stored in the NameNode's Data Directories. The value of this configuration is automatically generated to be the Quorum-based Storage URI if there are JournalNodes and this NameNode is not Highly Available.",
"display_name": "NameNode Edits Directories",
"name": "dfs_namenode_edits_dir",
"value": null
},
{
"desc": "If set to false and if one of the replicas of the NameNode storage fails, such as temporarily failure of NFS, this directory is not used until the NameNode restarts. If enabled, failed storage is re-checked on every checkpoint and, if it becomes valid, the NameNode will try to restore the edits and fsimage.",
"display_name": "Restore NameNode Directories at Checkpoint Time",
"name": "dfs_name_dir_restore",
"value": "false"
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystems that contain this role's data directories.",
"display_name": "Data Directories Free Space Monitoring Absolute Thresholds",
"name": "namenode_data_directories_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
}
]

230
sahara/plugins/cdh/v5_3_0/resources/hdfs-nfsgateway.json Normal file
View File

@ -0,0 +1,230 @@
[
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Log Directory Free Space Monitoring Percentage Thresholds",
"name": "log_directory_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "<p>This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.</p><p>Each rule has some or all of the following fields:</p><ul><li><span class='code'>alert</span> - whether or not events generated from this rule should be promoted to alerts. A value of \"true\" will cause alerts to be generated. If not specified, the default is \"false\".</li><li><span class='code'>rate</span> <strong>(mandatory)</strong> - the maximum number of log messages matching this rule that may be sent as events every minute. If more than <tt>rate</tt> matching log messages are received in a single minute, the extra messages are ignored. If rate is less than 0, the number of messages per minute is unlimited.</li><li><span class='code'>periodminutes</span> - the number of minutes during which the publisher will only publish <tt>rate</tt> events or fewer. If not specified, the default is <strong>one minute</strong></li><li><span class='code'>threshold</span> - apply this rule only to messages with this log4j severity level or above. An example is \"WARN\" for warning level messages or higher.</li><li><span class='code'>content</span> - match only those messages whose contents match this regular expression.</li><li><span class='code'>exceptiontype</span> - match only those messages which are part of an exception message. The exception type must match this regular expression.</li></ul><br/><p>Example:<span class='code'>{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}</span></p><p>This rule will send events to Cloudera Manager for every <span class='code'>StringIndexOutOfBoundsException</span>, up to a maximum of 10 every minute.</p>",
"display_name": "Rules to Extract Events from Log Files",
"name": "log_event_whitelist",
"value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}"
},
{
"desc": "Whether or not periodic stacks collection is enabled.",
"display_name": "Stacks Collection Enabled",
"name": "stacks_collection_enabled",
"value": "false"
},
{
"desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.",
"display_name": "File Descriptor Monitoring Thresholds",
"name": "nfsgateway_fd_thresholds",
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "NFS clients often reorder writes. As a result, sequential writes can arrive at the NFS Gateway in random order. This directory is used to temporarily save out-of-order writes before writing to HDFS. For each file, the out-of-order writes are dumped after they are accumulated to exceed certain threshold (e.g., 1MB) in memory. Please make sure this directory has enough space. For example, if the application uploads 10 files with each having 100MB, it is recommended that this directory have roughly 1GB of space in case write reorder happens (in the worst case) to every file.",
"display_name": "Temporary Dump Directory",
"name": "dfs_nfs3_dump_dir",
"value": "/tmp/.hdfs-nfs"
},
{
"desc": "Enables the health test that the NFS Gateway's process state is consistent with the role configuration",
"display_name": "NFS Gateway Process Health Test",
"name": "nfsgateway_scm_health_enabled",
"value": "true"
},
{
"desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.",
"display_name": "Java Heap Size of NFS Gateway in Bytes",
"name": "nfsgateway_java_heapsize",
"value": "268435456"
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hdfs-site.xml</strong> for this role only.",
"display_name": "NFS Gateway Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml",
"name": "nfsgateway_config_safety_valve",
"value": null
},
{
"desc": "The port number of the system portmap or rpcbind service. This configuration is used by Cloudera Manager to verify if the system portmap or rpcbind service is running before starting NFS Gateway role. Cloudera Manager does not manage the system portmap or rpcbind service.",
"display_name": "Portmap (or Rpcbind) Port",
"name": "nfs3_portmap_port",
"value": "111"
},
{
"desc": "The NFS Gateway server port.",
"display_name": "NFS Gateway Server Port",
"name": "nfs3_server_port",
"value": "2049"
},
{
"desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Java Configuration Options for NFS Gateway",
"name": "nfsgateway_java_opts",
"value": ""
},
{
"desc": "The maximum number of rolled log files to keep for NFS Gateway logs. Typically used by log4j.",
"display_name": "NFS Gateway Maximum Log File Backups",
"name": "max_log_backup_index",
"value": "10"
},
{
"desc": "<p>The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific role. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Role Triggers",
"name": "role_triggers",
"value": "[]"
},
{
"desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Soft Limit",
"name": "rm_memory_soft_limit",
"value": "-1"
},
{
"desc": "When computing the overall NFS Gateway health, consider the host's health.",
"display_name": "NFS Gateway Host Health Test",
"name": "nfsgateway_host_health_enabled",
"value": "true"
},
{
"desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Hard Limit",
"name": "rm_memory_hard_limit",
"value": "-1"
},
{
"desc": "By default, NFS Gateway exported directories can be mounted by any client. For better access control, update this property with a list of host names and access privileges separated by whitespace characters. Host name format can be a single host, a Java regular expression, or an IPv4 address. The access privilege uses <strong>rw</strong> to specify readwrite and <strong>ro</strong> to specify readonly access. If the access privilege is not provided, the default is read-only. Examples of host name format and access privilege: \"192.168.0.0/22 rw\", \"host.*.example.com\", \"host1.test.org ro\".",
"display_name": "Allowed Hosts and Privileges",
"name": "dfs_nfs_exports_allowed_hosts",
"value": "* rw"
},
{
"desc": "The period to review when computing unexpected exits.",
"display_name": "Unexpected Exits Monitoring Period",
"name": "unexpected_exits_window",
"value": "5"
},
{
"desc": "The port number of the mount daemon implemented inside the NFS Gateway server role.",
"display_name": "NFS Gateway MountD Port",
"name": "nfs3_mountd_port",
"value": "4242"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>log4j.properties</strong> for this role only.",
"display_name": "NFS Gateway Logging Advanced Configuration Snippet (Safety Valve)",
"name": "log4j_safety_valve",
"value": null
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.",
"display_name": "Log Directory Free Space Monitoring Absolute Thresholds",
"name": "log_directory_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.",
"display_name": "Maximum Process File Descriptors",
"name": "rlimit_fds",
"value": null
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Health Alerts for this Role",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
"display_name": "Automatically Restart Process",
"name": "process_auto_restart",
"value": "false"
},
{
"desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.",
"display_name": "Kill When Out of Memory",
"name": "oom_sigkill_enabled",
"value": "true"
},
{
"desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.",
"display_name": "Dump Heap When Out of Memory",
"name": "oom_heap_dump_enabled",
"value": "false"
},
{
"desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.",
"display_name": "Unexpected Exits Thresholds",
"name": "unexpected_exits_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "The maximum size, in megabytes, per log file for NFS Gateway logs. Typically used by log4j.",
"display_name": "NFS Gateway Max Log Size",
"name": "max_log_size",
"value": "200"
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null
},
{
"desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup I/O Weight",
"name": "rm_io_weight",
"value": "500"
},
{
"desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup CPU Shares",
"name": "rm_cpu_shares",
"value": "1024"
},
{
"desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.",
"display_name": "Heap Dump Directory",
"name": "oom_heap_dump_dir",
"value": "/tmp"
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
},
{
"desc": "The frequency with which stacks will be collected.",
"display_name": "Stacks Collection Frequency",
"name": "stacks_collection_frequency",
"value": "5.0"
},
{
"desc": "The minimum log level for NFS Gateway logs",
"display_name": "NFS Gateway Logging Threshold",
"name": "log_threshold",
"value": "INFO"
},
{
"desc": "Directory where NFS Gateway will place its log files.",
"display_name": "NFS Gateway Log Directory",
"name": "nfsgateway_log_dir",
"value": "/var/log/hadoop-hdfs"
}
]

284
sahara/plugins/cdh/v5_3_0/resources/hdfs-secondarynamenode.json Normal file
View File

@ -0,0 +1,284 @@
[
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Log Directory Free Space Monitoring Percentage Thresholds",
"name": "log_directory_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "<p>This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.</p><p>Each rule has some or all of the following fields:</p><ul><li><span class='code'>alert</span> - whether or not events generated from this rule should be promoted to alerts. A value of \"true\" will cause alerts to be generated. If not specified, the default is \"false\".</li><li><span class='code'>rate</span> <strong>(mandatory)</strong> - the maximum number of log messages matching this rule that may be sent as events every minute. If more than <tt>rate</tt> matching log messages are received in a single minute, the extra messages are ignored. If rate is less than 0, the number of messages per minute is unlimited.</li><li><span class='code'>periodminutes</span> - the number of minutes during which the publisher will only publish <tt>rate</tt> events or fewer. If not specified, the default is <strong>one minute</strong></li><li><span class='code'>threshold</span> - apply this rule only to messages with this log4j severity level or above. An example is \"WARN\" for warning level messages or higher.</li><li><span class='code'>content</span> - match only those messages whose contents match this regular expression.</li><li><span class='code'>exceptiontype</span> - match only those messages which are part of an exception message. The exception type must match this regular expression.</li></ul><br/><p>Example:<span class='code'>{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}</span></p><p>This rule will send events to Cloudera Manager for every <span class='code'>StringIndexOutOfBoundsException</span>, up to a maximum of 10 every minute.</p>",
"display_name": "Rules to Extract Events from Log Files",
"name": "log_event_whitelist",
"value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"},\n {\"alert\": false, \"rate\": 1, \"threshold\":\"INFO\", \"content\":\"Triggering checkpoint.*\"}\n ]\n}\n"
},
{
"desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Java Configuration Options for Secondary NameNode",
"name": "secondarynamenode_java_opts",
"value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled"
},
{
"desc": "Whether or not periodic stacks collection is enabled.",
"display_name": "Stacks Collection Enabled",
"name": "stacks_collection_enabled",
"value": "false"
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystems that contain this role's checkpoint directories.",
"display_name": "Checkpoint Directories Free Space Monitoring Absolute Thresholds",
"name": "secondarynamenode_checkpoint_directories_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.",
"display_name": "Java Heap Size of Secondary namenode in Bytes",
"name": "secondary_namenode_java_heapsize",
"value": "1073741824"
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
},
{
"desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
"display_name": "Automatically Restart Process",
"name": "process_auto_restart",
"value": "false"
},
{
"desc": "The maximum number of rolled log files to keep for SecondaryNameNode logs. Typically used by log4j.",
"display_name": "SecondaryNameNode Maximum Log File Backups",
"name": "max_log_backup_index",
"value": "10"
},
{
"desc": "<p>The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific role. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Role Triggers",
"name": "role_triggers",
"value": "[]"
},
{
"desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Soft Limit",
"name": "rm_memory_soft_limit",
"value": "-1"
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.",
"display_name": "Log Directory Free Space Monitoring Absolute Thresholds",
"name": "log_directory_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.",
"display_name": "Dump Heap When Out of Memory",
"name": "oom_heap_dump_enabled",
"value": "false"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hdfs-site.xml</strong> for this role only.",
"display_name": "SecondaryNameNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml",
"name": "secondarynamenode_config_safety_valve",
"value": null
},
{
"desc": "Enables the health test that the SecondaryNameNode's process state is consistent with the role configuration",
"display_name": "SecondaryNameNode Process Health Test",
"name": "secondarynamenode_scm_health_enabled",
"value": "true"
},
{
"desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Hard Limit",
"name": "rm_memory_hard_limit",
"value": "-1"
},
{
"desc": "The time between two periodic file system checkpoints.",
"display_name": "Filesystem Checkpoint Period",
"name": "fs_checkpoint_period",
"value": "3600"
},
{
"desc": "The period to review when computing unexpected exits.",
"display_name": "Unexpected Exits Monitoring Period",
"name": "unexpected_exits_window",
"value": "5"
},
{
"desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into <strong>hadoop-metrics2.properties</strong>.",
"display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)",
"name": "hadoop_metrics2_safety_valve",
"value": null
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
},
{
"desc": "If enabled, the SecondaryNameNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.",
"display_name": "Bind SecondaryNameNode to Wildcard Address",
"name": "secondary_namenode_bind_wildcard",
"value": "false"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>log4j.properties</strong> for this role only.",
"display_name": "SecondaryNameNode Logging Advanced Configuration Snippet (Safety Valve)",
"name": "log4j_safety_valve",
"value": null
},
{
"desc": "The number of transactions after which the NameNode or SecondaryNameNode will create a checkpoint of the namespace, regardless of whether the checkpoint period has expired.",
"display_name": "Filesystem Checkpoint Transaction Threshold",
"name": "fs_checkpoint_txns",
"value": "1000000"
},
{
"desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.",
"display_name": "Web Metric Collection",
"name": "secondarynamenode_web_metric_collection_enabled",
"value": "true"
},
{
"desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.",
"display_name": "Maximum Process File Descriptors",
"name": "rlimit_fds",
"value": null
},
{
"desc": "The health test thresholds on the duration of the metrics request to the web server.",
"display_name": "Web Metric Collection Duration",
"name": "secondarynamenode_web_metric_collection_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}"
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Health Alerts for this Role",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystems that contain this role's checkpoint directories. Specified as a percentage of the capacity on the filesystem. This setting is not used if a Checkpoint Directories Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Checkpoint Directories Free Space Monitoring Percentage Thresholds",
"name": "secondarynamenode_checkpoint_directories_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "When computing the overall SecondaryNameNode health, consider the host's health.",
"display_name": "SecondaryNameNode Host Health Test",
"name": "secondarynamenode_host_health_enabled",
"value": "true"
},
{
"desc": "The maximum size, in megabytes, per log file for SecondaryNameNode logs. Typically used by log4j.",
"display_name": "SecondaryNameNode Max Log Size",
"name": "max_log_size",
"value": "200"
},
{
"desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.",
"display_name": "Kill When Out of Memory",
"name": "oom_sigkill_enabled",
"value": "true"
},
{
"desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.",
"display_name": "Garbage Collection Duration Thresholds",
"name": "secondarynamenode_gc_duration_thresholds",
"value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}"
},
{
"desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.",
"display_name": "Unexpected Exits Thresholds",
"name": "unexpected_exits_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.",
"display_name": "File Descriptor Monitoring Thresholds",
"name": "secondarynamenode_fd_thresholds",
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "The period to review when computing the moving average of garbage collection time.",
"display_name": "Garbage Collection Duration Monitoring Period",
"name": "secondarynamenode_gc_duration_window",
"value": "5"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "Determines where on the local file system the DFS SecondaryNameNode should store the temporary images to merge. For redundancy, enter a comma-delimited list of directories to replicate the image in all of the directories. Typical values are /data/N/dfs/snn for N = 1, 2, 3...",
"display_name": "HDFS Checkpoint Directory",
"name": "fs_checkpoint_dir_list",
"value": null
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null
},
{
"desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup I/O Weight",
"name": "rm_io_weight",
"value": "500"
},
{
"desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup CPU Shares",
"name": "rm_cpu_shares",
"value": "1024"
},
{
"desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.",
"display_name": "Heap Dump Directory",
"name": "oom_heap_dump_dir",
"value": "/tmp"
},
{
"desc": "The frequency with which stacks will be collected.",
"display_name": "Stacks Collection Frequency",
"name": "stacks_collection_frequency",
"value": "5.0"
},
{
"desc": "Nameservice of this SecondaryNameNode",
"display_name": "SecondaryNameNode Nameservice",
"name": "dfs_secondarynamenode_nameservice",
"value": null
},
{
"desc": "The minimum log level for SecondaryNameNode logs",
"display_name": "SecondaryNameNode Logging Threshold",
"name": "log_threshold",
"value": "INFO"
},
{
"desc": "The base port where the secure SecondaryNameNode web UI listens.",
"display_name": "Secure SecondaryNameNode Web UI Port (SSL)",
"name": "dfs_secondary_https_port",
"value": "50495"
},
{
"desc": "Directory where SecondaryNameNode will place its log files.",
"display_name": "SecondaryNameNode Log Directory",
"name": "secondarynamenode_log_dir",
"value": "/var/log/hadoop-hdfs"
},
{
"desc": "The SecondaryNameNode HTTP port. If the port is 0, then the server starts on a free port. Combined with the SecondaryNameNode's hostname to build its HTTP address.",
"display_name": "SecondaryNameNode Web UI Port",
"name": "dfs_secondary_http_port",
"value": "50090"
}
]

698
sahara/plugins/cdh/v5_3_0/resources/hdfs-service.json Normal file
View File

@ -0,0 +1,698 @@
[
{
"desc": "Timeout in milliseconds for the parallel RPCs made in DistributedFileSystem#getFileBlockStorageLocations(). This value is only emitted for Impala.",
"display_name": "HDFS File Block Storage Location Timeout",
"name": "dfs_client_file_block_storage_locations_timeout",
"value": "10000"
},
{
"desc": "The domain to use for the HTTP cookie that stores the authentication token. In order for authentiation to work correctly across all Hadoop nodes' web-consoles the domain must be correctly set. <b>Important:</b> when using IP addresses, browsers ignore cookies with domain settings. For this setting to work properly all nodes in the cluster must be configured to generate URLs with hostname.domain names on it.",
"display_name": "Hadoop HTTP Authentication Cookie Domain",
"name": "hadoop_http_auth_cookie_domain",
"value": ""
},
{
"desc": "The user that this service's processes should run as (except the HttpFS server, which has its own user)",
"display_name": "System User",
"name": "process_username",
"value": "hdfs"
},
{
"desc": "<p>Event filters are defined in a JSON object like the following:</p>\n\n<pre>\n{\n \"defaultAction\" : (\"accept\", \"discard\"),\n \"rules\" : [\n {\n \"action\" : (\"accept\", \"discard\"),\n \"fields\" : [\n {\n \"name\" : \"fieldName\",\n \"match\" : \"regex\"\n }\n ]\n }\n ]\n}\n</pre>\n\n<p>\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n</p>\n\n<p>\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n</p>\n\n<p>\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n</p>\n\n<p>\nThe following is the list of fields that can be filtered for HDFS events:\n</p>\n\n<ul>\n <li>username: the user performing the action.</li>\n <li>ipAddress: the IP from where the request originated.</li>\n <li>command: the HDFS operation being performed.</li>\n <li>src: the source path for the operation.</li>\n <li>dest: the destination path for the operation.</li>\n <li>permissions: the permissions associated with the operation.</li>\n</ul>\n",
"display_name": "Event Filter",
"name": "navigator_audit_event_filter",
"value": "{\n \"comment\" : [\n \"Default filter for HDFS services.\",\n \"Discards events generated by the internal Cloudera and/or HDFS users\",\n \"(hdfs, hbase, mapred and dr.who), and events that affect files in \",\n \"/tmp directory.\"\n ],\n \"defaultAction\" : \"accept\",\n \"rules\" : [\n {\n \"action\" : \"discard\",\n \"fields\" : [\n { \"name\" : \"username\", \"match\" : \"(?:cloudera-scm|hbase|hdfs|mapred|hive|dr.who|solr|impala)(?:/.+)?\" }\n ]\n },\n {\n \"action\" : \"discard\",\n \"fields\" : [\n { \"name\" : \"src\", \"match\" : \"/tmp(?:/.*)?\" }\n ]\n }\n ]\n}\n"
},
{
"desc": "The password for the SSL keystore.",
"display_name": "Hadoop User Group Mapping LDAP SSL Keystore Password",
"name": "hadoop_group_mapping_ldap_keystore_passwd",
"value": ""
},
{
"desc": "Comma-delimited list of hosts where you want to allow the Hue user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.",
"display_name": "Hue Proxy User Hosts",
"name": "hue_proxy_user_hosts_list",
"value": "*"
},
{
"desc": "The service monitor will use this directory to create files to test if the hdfs service is healthy. The directory and files are created with permissions specified by 'HDFS Health Canary Directory Permissions'",
"display_name": "HDFS Health Canary Directory",
"name": "firehose_hdfs_canary_directory",
"value": "/tmp/.cloudera_health_monitoring_canary_files"
},
{
"desc": "Path to the directory where audit logs will be written. The directory will be created if it doesn't exist.",
"display_name": "Audit Log Directory",
"name": "audit_event_log_dir",
"value": "/var/log/hadoop-hdfs/audit"
},
{
"desc": "Class for user to group mapping (get groups for a given user).",
"display_name": "Hadoop User Group Mapping Implementation",
"name": "hadoop_security_group_mapping",
"value": "org.apache.hadoop.security.ShellBasedUnixGroupsMapping"
},
{
"desc": "Allows the oozie superuser to impersonate any members of a comma-delimited list of groups. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.",
"display_name": "Oozie Proxy User Groups",
"name": "oozie_proxy_user_groups_list",
"value": "*"
},
{
"desc": "Comma-separated list of compression codecs that can be used in job or map compression.",
"display_name": "Compression Codecs",
"name": "io_compression_codecs",
"value": "org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec"
},
{
"desc": "Comma-separated list of users authorized to used Hadoop. This is emitted only if authorization is enabled.",
"display_name": "Authorized Users",
"name": "hadoop_authorized_users",
"value": "*"
},
{
"desc": "Enable HDFS short circuit read. This allows a client co-located with the DataNode to read HDFS file blocks directly. This gives a performance boost to distributed clients that are aware of locality.",
"display_name": "Enable HDFS Short Circuit Read",
"name": "dfs_datanode_read_shortcircuit",
"value": "true"
},
{
"desc": "The distinguished name of the user to bind as when connecting to the LDAP server. This may be left blank if the LDAP server supports anonymous binds.",
"display_name": "Hadoop User Group Mapping LDAP Bind User",
"name": "hadoop_group_mapping_ldap_bind_user",
"value": ""
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Service Level Health Alerts",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "Path to the keystore file containing the server certificate and private key used for encrypted shuffle and encrypted web UIs. Applies to configurations of all daemon roles of this service.",
"display_name": "Hadoop SSL Server Keystore File Location",
"name": "ssl_server_keystore_location",
"value": null
},
{
"desc": "Password for the SSL client truststore. Defines a cluster-wide default that can be overridden by individual services.",
"display_name": "Cluster-Wide Default SSL Client Truststore Password",
"name": "ssl_client_truststore_password",
"value": null
},
{
"desc": "The password of the bind user.",
"display_name": "Hadoop User Group Mapping LDAP Bind User Password",
"name": "hadoop_group_mapping_ldap_bind_passwd",
"value": ""
},
{
"desc": "Action to take when the audit event queue is full. Drop the event or shutdown the affected process.",
"display_name": "Queue Policy",
"name": "navigator_audit_queue_policy",
"value": "DROP"
},
{
"desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.",
"display_name": "Enable Log Event Capture",
"name": "catch_events",
"value": "true"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>core-site.xml</strong>. Applies to all roles and client configurations in this HDFS service as well as all its dependent services. Any configs added here will be overridden by their default values in HDFS (which can be found in hdfs-default.xml).",
"display_name": "Cluster-wide Advanced Configuration Snippet (Safety Valve) for core-site.xml",
"name": "core_site_safety_valve",
"value": null
},
{
"desc": "Enable WebHDFS interface",
"display_name": "Enable WebHDFS",
"name": "dfs_webhdfs_enabled",
"value": "true"
},
{
"desc": "The name of the group of superusers.",
"display_name": "Superuser Group",
"name": "dfs_permissions_supergroup",
"value": "supergroup"
},
{
"desc": "Path to the SSL client truststore file. Defines a cluster-wide default that can be overridden by individual services. This truststore must be in JKS format. The truststore contains certificates of trusted servers, or of Certificate Authorities trusted to identify servers. The contents of the truststore can be modified without restarting any roles. By default, changes to its contents are picked up within ten seconds. If not set, the default Java truststore is used to verify certificates.",
"display_name": "Cluster-Wide Default SSL Client Truststore Location",
"name": "ssl_client_truststore_location",
"value": null
},
{
"desc": "Typically, HDFS clients and servers communicate by opening sockets via an IP address. In certain networking configurations, it is preferable to open sockets after doing a DNS lookup on the hostname. Enable this property to open sockets after doing a DNS lookup on the hostname. This property is supported in CDH3u4 or later deployments.",
"display_name": "Use DataNode Hostname",
"name": "dfs_client_use_datanode_hostname",
"value": "false"
},
{
"desc": "Enter a FailoverProxyProvider implementation to configure two URIs to connect to during fail-over. The first configured address is tried first, and on a fail-over event the other address is tried.",
"display_name": "FailoverProxyProvider Class",
"name": "dfs_ha_proxy_provider",
"value": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
},
{
"desc": "The search base for the LDAP connection. This is a distinguished name, and will typically be the root of the LDAP directory.",
"display_name": "Hadoop User Group Mapping Search Base",
"name": "hadoop_group_mapping_ldap_base",
"value": ""
},
{
"desc": "If false, permission checking is turned off for files in HDFS.",
"display_name": "Check HDFS Permissions",
"name": "dfs_permissions",
"value": "true"
},
{
"desc": "Comma-delimited list of groups that you want to allow the Hue user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.",
"display_name": "Hue Proxy User Groups",
"name": "hue_proxy_user_groups_list",
"value": "*"
},
{
"desc": "Comma-separated list of groups authorized to used Hadoop. This is emitted only if authorization is enabled.",
"display_name": "Authorized Groups",
"name": "hadoop_authorized_groups",
"value": ""
},
{
"desc": "Password that protects the private key contained in the server keystore used for encrypted shuffle and encrypted web UIs. Applies to all configurations of daemon roles of this service.",
"display_name": "Hadoop SSL Server Keystore Key Password",
"name": "ssl_server_keystore_keypassword",
"value": null
},
{
"desc": "Comma-delimited list of hosts where you want to allow the oozie user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.",
"display_name": "Oozie Proxy User Hosts",
"name": "oozie_proxy_user_hosts_list",
"value": "*"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "Comma-delimited list of groups that you want to allow the mapred user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.",
"display_name": "Mapred Proxy User Groups",
"name": "mapred_proxy_user_groups_list",
"value": "*"
},
{
"desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.",
"display_name": "HDFS Service Environment Advanced Configuration Snippet (Safety Valve)",
"name": "hdfs_service_env_safety_valve",
"value": null
},
{
"desc": "Additional mapping rules that will be inserted before rules generated from the list of trusted realms and before the default rule. After changing this value and restarting the service, any services depending on this one must be restarted as well. The hadoop.security.auth_to_local property is configured using this information.",
"display_name": "Additional Rules to Map Kerberos Principals to Short Names",
"name": "extra_auth_to_local_rules",
"value": null
},
{
"desc": "Password for the server keystore file used for encrypted shuffle and encrypted web UIs. Applies to configurations of all daemon roles of this service.",
"display_name": "Hadoop SSL Server Keystore File Password",
"name": "ssl_server_keystore_password",
"value": null
},
{
"desc": "Maximum size of audit log file in MB before it is rolled over.",
"display_name": "Maximum Audit Log File Size",
"name": "navigator_audit_log_max_file_size",
"value": "100"
},
{
"desc": "Enables kerberos authentication for hadoop HTTP web-consoles for all roles of this service using the SPNEGO protocol. <b>Note:</b> This is effective only if kerberos is enabled for the HDFS service.",
"display_name": "Enable Kerberos Authentication for HTTP Web-Consoles",
"name": "hadoop_secure_web_ui",
"value": "false"
},
{
"desc": "Comma-delimited list of groups to allow the HDFS user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.",
"display_name": "HDFS Proxy User Groups",
"name": "hdfs_proxy_user_groups_list",
"value": "*"
},
{
"desc": "Quality of protection for secured RPC connections between NameNode and HDFS clients. For effective RPC protection, enable Kerberos authentication.",
"display_name": "Hadoop RPC Protection",
"name": "hadoop_rpc_protection",
"value": "authentication"
},
{
"desc": "Comma-delimited list of groups that you want to allow the Hive user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.",
"display_name": "Hive Proxy User Groups",
"name": "hive_proxy_user_groups_list",
"value": "*"
},
{
"desc": "Comma-separated list of users authorized to perform admin operations on Hadoop. This is emitted only if authorization is enabled.",
"display_name": "Authorized Admin Users",
"name": "hadoop_authorized_admin_users",
"value": "*"
},
{
"desc": "The health check thresholds of the number of missing blocks. Specified as a percentage of the total number of blocks.",
"display_name": "Missing Block Monitoring Thresholds",
"name": "hdfs_missing_blocks_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "The amount of time after NameNode(s) start that the lack of an active NameNode will be tolerated. This is intended to allow either the auto-failover daemon to make a NameNode active, or a specifically issued failover command to take effect. This is an advanced option that does not often need to be changed.",
"display_name": "NameNode Activation Startup Tolerance",
"name": "hdfs_namenode_activation_startup_tolerance",
"value": "180"
},
{
"desc": "Comma-delimited list of groups to allow the HttpFS user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.",
"display_name": "HttpFS Proxy User Groups",
"name": "httpfs_proxy_user_groups_list",
"value": "*"
},
{
"desc": "Allows the flume user to impersonate any members of a comma-delimited list of groups. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.",
"display_name": "Flume Proxy User Groups",
"name": "flume_proxy_user_groups_list",
"value": "*"
},
{
"desc": "Comma-delimited list of hosts where you want to allow the mapred user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.",
"display_name": "Mapred Proxy User Hosts",
"name": "mapred_proxy_user_hosts_list",
"value": "*"
},
{
"desc": "For advanced use only, a list of configuration properties that will be used by the Service Monitor instead of the current client configuration for the service.",
"display_name": "Service Monitor Client Config Overrides",
"name": "smon_client_config_overrides",
"value": "<property><name>dfs.socket.timeout</name><value>3000</value></property><property><name>dfs.datanode.socket.write.timeout</name><value>3000</value></property><property><name>ipc.client.connect.max.retries</name><value>1</value></property><property><name>fs.permissions.umask-mode</name><value>000</value></property>"
},
{
"desc": "<p>The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific service. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Service Triggers",
"name": "service_triggers",
"value": "[]"
},
{
"desc": "Enable SSL encryption for HDFS, MapReduce, and YARN web UIs, as well as encrypted shuffle for MapReduce and Yarn.",
"display_name": "Hadoop SSL Enabled",
"name": "hdfs_hadoop_ssl_enabled",
"value": "false"
},
{
"desc": "List of Kerberos realms that Hadoop services should trust. If empty, defaults to the default_realm property configured in the krb5.conf file. After changing this value and restarting the service, all services depending on this service must also be restarted. Adds mapping rules for each domain to the hadoop.security.auth_to_local property in core-site.xml.",
"display_name": "Trusted Kerberos Realms",
"name": "trusted_realms",
"value": ""
},
{
"desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.",
"display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)",
"name": "smon_derived_configs_safety_valve",
"value": null
},
{
"desc": "Enables the health check that verifies that the failover controllers associated with this service are healthy and running.",
"display_name": "Failover Controllers Healthy",
"name": "failover_controllers_healthy_enabled",
"value": "true"
},
{
"desc": "The attribute of the group object that identifies the users that are members of the group. The default will usually be appropriate for any LDAP installation.",
"display_name": "Hadoop User Group Mapping LDAP Group Membership Attribute",
"name": "hadoop_group_mapping_ldap_member_attr",
"value": "member"
},
{
"desc": "Comma separated list of users allowed to do short circuit read. A short circuit read allows a client co-located with the data to read HDFS file blocks directly from HDFS. If empty, will default to the DataNode process' user.",
"display_name": "DataNode Local Path Access Users",
"name": "dfs_block_local_path_access_user",
"value": null
},
{
"desc": "The timeout, in milliseconds, to use with the Cloudera Manager agent-based fencer.",
"display_name": "Timeout for Cloudera Manager Fencing Strategy",
"name": "dfs_ha_fencing_cloudera_manager_timeout_millis",
"value": "10000"
},
{
"desc": "Enable collection of audit events from the service's roles.",
"display_name": "Enable Collection",
"name": "navigator_audit_enabled",
"value": "true"
},
{
"desc": "Maximum bandwidth used for image transfer in bytes per second. This can help keep normal NameNode operations responsive during checkpointing. A default value of 0 indicates that throttling is disabled.",
"display_name": "FsImage Transfer Bandwidth",
"name": "dfs_image_transfer_bandwidthPerSec",
"value": "0"
},
{
"desc": "The user the management services will impersonate as when connecting to HDFS. Defaults to 'hdfs', a superuser.",
"display_name": "HDFS User to Impersonate",
"name": "hdfs_user_to_impersonate",
"value": "hdfs"
},
{
"desc": "The short name of Hue's Kerberos principal",
"display_name": "Hue's Kerberos Principal Short Name",
"name": "hue_kerberos_principal_shortname",
"value": "hue"
},
{
"desc": "File path to the SSL keystore containing the SSL certificate required by the LDAP server.",
"display_name": "Hadoop User Group Mapping LDAP SSL Keystore",
"name": "hadoop_group_mapping_ldap_keystore",
"value": ""
},
{
"desc": "Comma-delimited list of hosts where you want to allow the HDFS user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.",
"display_name": "HDFS Proxy User Hosts",
"name": "hdfs_proxy_user_hosts_list",
"value": "*"
},
{
"desc": "The minimal block replication.",
"display_name": "Minimal Block Replication",
"name": "dfs_replication_min",
"value": "1"
},
{
"desc": "The maximal block replication.",
"display_name": "Maximal Block Replication",
"name": "dfs_replication_max",
"value": "512"
},
{
"desc": "The service monitor will use these permissions to create the directory and files to test if the hdfs service is healthy. Permissions are specified using the 10-character unix-symbolic format e.g. '-rwxr-xr-x'.",
"display_name": "HDFS Health Canary Directory Permissions",
"name": "firehose_hdfs_canary_directory_permissions",
"value": "-rwxrwxrwx"
},
{
"desc": "Enable authorization",
"display_name": "Hadoop Secure Authorization",
"name": "hadoop_security_authorization",
"value": "false"
},
{
"desc": "The Key Management Server used by HDFS. This must be set in order to use encryption for data at rest.",
"display_name": "KMS Service",
"name": "kms_service",
"value": null
},
{
"desc": "The attribute of the group object that identifies the group name. The default will usually be appropriate for all LDAP systems.",
"display_name": "Hadoop User Group Mapping LDAP Group Name Attribute",
"name": "hadoop_group_mapping_ldap_group_name_attr",
"value": "cn"
},
{
"desc": "Enables DataNode support for the experimental DistributedFileSystem.getFileVBlockStorageLocations API. Applicable to CDH 4.1 and onwards.",
"display_name": "Enable HDFS Block Metadata API",
"name": "dfs_datanode_hdfs_blocks_metadata_enabled",
"value": "true"
},
{
"desc": "The tolerance window that will be used in HDFS service tests that depend on detection of the active NameNode.",
"display_name": "Active NameNode Detection Window",
"name": "hdfs_active_namenode_detecton_window",
"value": "3"
},
{
"desc": "Default block replication. The number of replications to make when the file is created. The default value is used if a replication number is not specified.",
"display_name": "Replication Factor",
"name": "dfs_replication",
"value": "3"
},
{
"desc": "Comma-delimited list of groups that you want to allow the HTTP user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'. This is used by WebHCat.",
"display_name": "HTTP Proxy User Groups",
"name": "HTTP_proxy_user_groups_list",
"value": "*"
},
{
"desc": "The name of the system group shared by all the core Hadoop services.",
"display_name": "Shared Hadoop Group Name",
"name": "hdfs_hadoop_group_name",
"value": "hadoop"
},
{
"desc": "The amount of time to wait for HDFS filesystem image transfer from NameNode to complete.",
"display_name": "FsImage Transfer Timeout",
"name": "dfs_image_transfer_timeout",
"value": "60000"
},
{
"desc": "ACLs (Access Control Lists) enhance the existing HDFS permission model to support controlling file access for arbitrary combinations of users and groups instead of a single owner, single group, and all other users. When ACLs are disabled, the NameNode rejects all attempts to set an ACL.",
"display_name": "Enable Access Control Lists",
"name": "dfs_namenode_acls_enabled",
"value": "false"
},
{
"desc": "An additional filter to use when searching for LDAP users. The default will usually be appropriate for Active Directory installations. If connecting to a generic LDAP server, ''sAMAccountName'' will likely be replaced with ''uid''. {0} is a special string used to denote where the username fits into the filter.",
"display_name": "Hadoop User Group Mapping LDAP User Search Filter",
"name": "hadoop_group_mapping_ldap_user_filter",
"value": "(&(objectClass=user)(sAMAccountName={0}))"
},
{
"desc": "List of fencing methods to use for service fencing. <tt>shell(./cloudera_manager_agent_fencer.py)</tt> is a fencing mechanism designed to use the Cloudera Manager agent. The <tt>sshfence</tt> method uses SSH. If using custom fencers (that may communicate with shared store, power units, or network switches), use the shell to invoke them.",
"display_name": "HDFS High Availability Fencing Methods",
"name": "dfs_ha_fencing_methods",
"value": "shell(./cloudera_manager_agent_fencer.py)"
},
{
"desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the environment of HDFS replication jobs.",
"display_name": "HDFS Replication Advanced Configuration Snippet (Safety Valve)",
"name": "hdfs_replication_env_safety_valve",
"value": null
},
{
"desc": "The default block size in bytes for new HDFS files. Note that this value is also used as the HBase Region Server HLog block size.",
"display_name": "HDFS Block Size",
"name": "dfs_block_size",
"value": "134217728"
},
{
"desc": "Path on the DataNode's local file system to a UNIX domain socket used for communication between the DataNode and local HDFS clients. This socket is used for Short Circuit Reads. Only the HDFS System User and \"root\" should have write access to the parent directory and all of its ancestors. This property is supported in CDH 4.2 or later deployments.",
"display_name": "UNIX Domain Socket path",
"name": "dfs_domain_socket_path",
"value": "/var/run/hdfs-sockets/dn"
},
{
"desc": "Algorithm to encrypt data transfer between DataNodes and clients, and among DataNodes. 3des is more cryptographically secure, but rc4 is substantially faster.",
"display_name": "Data Transfer Encryption Algorithm",
"name": "dfs_encrypt_data_transfer_algorithm",
"value": "rc4"
},
{
"desc": "The health check thresholds of the number of under-replicated blocks. Specified as a percentage of the total number of blocks.",
"display_name": "Under-replicated Block Monitoring Thresholds",
"name": "hdfs_under_replicated_blocks_thresholds",
"value": "{\"critical\":\"40.0\",\"warning\":\"10.0\"}"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hdfs-site.xml</strong>. Applies to configurations of all roles in this service except client configuration.",
"display_name": "HDFS Service Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml",
"name": "hdfs_service_config_safety_valve",
"value": null
},
{
"desc": "Comma-delimited list of hosts where you want to allow the HTTP user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'. This is used by WebHCat.",
"display_name": "HTTP Proxy User Hosts",
"name": "HTTP_proxy_user_hosts_list",
"value": "*"
},
{
"desc": "<p>\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n</p>\n\n<p>\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n</p>\n\n<p>Event trackers are defined in a JSON object like the following:</p>\n\n<pre>\n{\n \"timeToLive\" : [integer],\n \"fields\" : [\n {\n \"type\" : [string],\n \"name\" : [string]\n }\n ]\n}\n</pre>\n\n<p>\nWhere:\n</p>\n\n<ul>\n <li>timeToLive: maximum amount of time an event will be tracked, in\n milliseconds. Must be provided. This defines how long, since it's\n first seen, an event will be tracked. A value of 0 disables tracking.</li>\n\n <li>fields: list of fields to compare when matching events against\n tracked events.</li>\n</ul>\n\n<p>\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n</p>\n\n<ul>\n <li>value: uses the field value for comparison.</li>\n\n <li>username: treats the field value as a user name, and ignores any\n host-specific data. This is useful for environment using Kerberos,\n so that only the principal name and realm are compared.</li>\n</ul>\n\n<p>\nThe following is the list of fields that can be used to compare HDFS events:\n</p>\n\n<ul>\n <li>username: the user performing the action.</li>\n <li>ipAddress: the IP from where the request originated.</li>\n <li>command: the HDFS operation being performed.</li>\n <li>src: the source path for the operation.</li>\n <li>dest: the destination path for the operation.</li>\n <li>permissions: the permissions associated with the operation.</li>\n</ul>\n",
"display_name": "Event Tracker",
"name": "navigator_event_tracker",
"value": "{\n \"comment\" : [\n \"Default event tracker for HDFS services.\",\n \"Defines equality by comparing username, operation and source path \",\n \"of the events.\"\n ],\n \"timeToLive\" : 60000,\n \"fields\" : [\n { \"type\": \"value\", \"name\" : \"src\" },\n { \"type\": \"value\", \"name\" : \"operation\" },\n { \"type\": \"username\", \"name\" : \"username\" }\n ]\n}\n"
},
{
"desc": "Choose the authentication mechanism used by Hadoop",
"display_name": "Hadoop Secure Authentication",
"name": "hadoop_security_authentication",
"value": "simple"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hadoop-policy.xml</strong>. Applies to configurations of all roles in this service except client configuration.",
"display_name": "HDFS Service Advanced Configuration Snippet (Safety Valve) for hadoop-policy.xml",
"name": "hadoop_policy_config_safety_valve",
"value": null
},
{
"desc": "Enable encryption of data transfer between DataNodes and clients, and among DataNodes. For effective data transfer protection, enable Kerberos authentication and choose Privacy Quality of RPC Protection.",
"display_name": "Enable Data Transfer Encryption",
"name": "dfs_encrypt_data_transfer",
"value": "false"
},
{
"desc": "When computing the overall HDFS cluster health, consider the active NameNode's health",
"display_name": "Active NameNode Role Health Check",
"name": "hdfs_namenode_health_enabled",
"value": "true"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>ssl-server.xml</strong>. Applies to configurations of all roles in this service except client configuration.",
"display_name": "HDFS Service Advanced Configuration Snippet (Safety Valve) for ssl-server.xml",
"name": "hdfs_ssl_server_safety_valve",
"value": null
},
{
"desc": "The home directory of the system user on the local filesystem. This setting must reflect the system's configured value - only changing it here will not change the actual home directory.",
"display_name": "System User's Home Directory",
"name": "hdfs_user_home_dir",
"value": "/var/lib/hadoop-hdfs"
},
{
"desc": "SASL protection mode for secured connections to the datanodes when reading or writing data.",
"display_name": "DataNode Data Transfer Protection",
"name": "dfs_data_transfer_protection",
"value": null
},
{
"desc": "When computing the overall HDFS cluster health, consider the health of the standby NameNode.",
"display_name": "Standby NameNode Health Check",
"name": "hdfs_standby_namenodes_health_enabled",
"value": "true"
},
{
"desc": "Comma-delimited list of hosts where you want to allow the flume user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.",
"display_name": "Flume Proxy User Hosts",
"name": "flume_proxy_user_hosts_list",
"value": "*"
},
{
"desc": "The URL for the LDAP server to use for resolving user groups when using LdapGroupsMapping.",
"display_name": "Hadoop User Group Mapping LDAP URL",
"name": "hadoop_group_mapping_ldap_url",
"value": ""
},
{
"desc": "SSH connection timeout, in milliseconds, to use with the built-in sshfence fencer.",
"display_name": "Timeout for SSH Fencing Strategy",
"name": "dfs_ha_fencing_ssh_connect_timeout",
"value": "30000"
},
{
"desc": "Maximum number of rolled over audit logs to retain. The logs will not be deleted if they contain audit events that have not yet been propagated to Audit Server.",
"display_name": "Number of Audit Logs to Retain",
"name": "navigator_audit_log_max_backup_index",
"value": "10"
},
{
"desc": "Enable automatic synchronization of HDFS ACLs with Sentry privileges. HDFS Access Control Lists and Check HDFS Permissions must be enabled when this feature is enabled. Use Sentry Synchronization Path Prefixes to define the HDFS regions where authorization will be enforced using Sentry information. For more information, see <a class=\"bold\" href=\"http://tiny.cloudera.com/sentry-sync-cm5\" target=\"_blank\">Synchronizing HDFS ACLs and Sentry Authorization<i class=\"externalLink\"></i></a>.",
"display_name": "Enable Sentry Synchronization",
"name": "hdfs_sentry_sync_enable",
"value": "false"
},
{
"desc": "Comma-delimited list of hosts where you want to allow the HttpFS user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.",
"display_name": "HttpFS Proxy User Hosts",
"name": "httpfs_proxy_user_hosts_list",
"value": "*"
},
{
"desc": "Name of the ZooKeeper service that this HDFS service instance depends on",
"display_name": "ZooKeeper Service",
"name": "zookeeper_service",
"value": null
},
{
"desc": "The group that this service's processes should run as (except the HttpFS server, which has its own group)",
"display_name": "System Group",
"name": "process_groupname",
"value": "hdfs"
},
{
"desc": "Comma-delimited list of hosts where you want to allow the Hive user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.",
"display_name": "Hive Proxy User Hosts",
"name": "hive_proxy_user_hosts_list",
"value": "*"
},
{
"desc": "The frequency in which the log4j event publication appender will retry sending undelivered log events to the Event server, in seconds",
"display_name": "Log Event Retry Frequency",
"name": "log_event_retry_frequency",
"value": "30"
},
{
"desc": "Default umask for file and directory creation, specified in an octal value (with a leading 0)",
"display_name": "Default Umask",
"name": "dfs_umaskmode",
"value": "022"
},
{
"desc": "The health check thresholds of free space in HDFS. Specified as a percentage of total HDFS capacity.",
"display_name": "HDFS Free Space Monitoring Thresholds",
"name": "hdfs_free_space_thresholds",
"value": "{\"critical\":\"10.0\",\"warning\":\"20.0\"}"
},
{
"desc": "Enables the health check that a client can create, read, write, and delete files",
"display_name": "HDFS Canary Health Check",
"name": "hdfs_canary_health_enabled",
"value": "true"
},
{
"desc": "The health check thresholds of the number of blocks that have at least one corrupt replica. Specified as a percentage of the total number of blocks.",
"display_name": "Blocks With Corrupt Replicas Monitoring Thresholds",
"name": "hdfs_blocks_with_corrupt_replicas_thresholds",
"value": "{\"critical\":\"1.0\",\"warning\":\"0.5\"}"
},
{
"desc": "Comma-separated list of groups authorized to perform admin operations on Hadoop. This is emitted only if authorization is enabled.",
"display_name": "Authorized Admin Groups",
"name": "hadoop_authorized_admin_groups",
"value": ""
},
{
"desc": "An additional filter to use when searching for groups.",
"display_name": "Hadoop User Group Mapping LDAP Group Search Filter",
"name": "hadoop_group_mapping_ldap_group_filter",
"value": "(objectClass=group)"
},
{
"desc": "A list of path prefixes that define the HDFS regions where authorization will be enforced using Sentry information. Only relevant when Sentry Synchronization is enabled.",
"display_name": "Sentry Synchronization Path Prefixes",
"name": "hdfs_sentry_sync_path_prefixes",
"value": "/user/hive/warehouse"
},
{
"desc": "For advanced use only, a string to be inserted into the client configuration for <strong>navigator.client.properties</strong>.",
"display_name": "HDFS Client Advanced Configuration Snippet (Safety Valve) for navigator.client.properties",
"name": "navigator_client_config_safety_valve",
"value": null
},
{
"desc": "The health test thresholds of the overall DataNode health. The check returns \"Concerning\" health if the percentage of \"Healthy\" DataNodes falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" DataNodes falls below the critical threshold.",
"display_name": "Healthy DataNode Monitoring Thresholds",
"name": "hdfs_datanodes_healthy_thresholds",
"value": "{\"critical\":\"90.0\",\"warning\":\"95.0\"}"
},
{
"desc": "The SSH private key files to use with the built-in sshfence fencer. These are to be accessible to the <tt>hdfs</tt> user on the machines running the NameNodes.",
"display_name": "Private Keys for SSH Fencing Strategy",
"name": "dfs_ha_fencing_ssh_private_key_files",
"value": null
},
{
"desc": "Whether or not to use SSL when connecting to the LDAP server.",
"display_name": "Hadoop User Group Mapping LDAP SSL Enabled",
"name": "hadoop_group_mapping_ldap_use_ssl",
"value": "false"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>ssl-client.xml</strong>. Applies cluster-wide, but can be overridden by individual services.",
"display_name": "HDFS Advanced Configuration Snippet (Safety Valve) for ssl-client.xml",
"name": "hdfs_ssl_client_safety_valve",
"value": null
}
]

50
sahara/plugins/cdh/v5_3_0/resources/hive-gateway.json Normal file
View File

@ -0,0 +1,50 @@
[
{
"desc": "For advanced use only, a string to be inserted into the client configuration for <strong>hive-site.xml</strong>.",
"display_name": "Hive Client Advanced Configuration Snippet (Safety Valve) for hive-site.xml",
"name": "hive_client_config_safety_valve",
"value": null
},
{
"desc": "These are Java command line arguments. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Client Java Configuration Options",
"name": "hive_client_java_opts",
"value": "-Djava.net.preferIPv4Stack=true"
},
{
"desc": "Timeout for requests to the Hive Metastore Server. Consider increasing this if you have tables with a lot of metadata and see timeout errors. Used by most Hive Metastore clients such as Hive CLI and HiveServer2, but not by Impala. Impala has a separately configured timeout.",
"display_name": "Hive Metastore Connection Timeout",
"name": "hive_metastore_timeout",
"value": "300"
},
{
"desc": "The directory where the client configs will be deployed",
"display_name": "Deploy Directory",
"name": "client_config_root_dir",
"value": "/etc/hive"
},
{
"desc": "Maximum size in bytes for the Java process heap memory. Passed to Java -Xmx.",
"display_name": "Client Java Heap Size in Bytes",
"name": "hive_client_java_heapsize",
"value": "268435456"
},
{
"desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.",
"display_name": "Alternatives Priority",
"name": "client_config_priority",
"value": "90"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the client configuration for <strong>hive-env.sh</strong>",
"display_name": "Gateway Client Environment Advanced Configuration Snippet for hive-env.sh (Safety Valve)",
"name": "hive_client_env_safety_valve",
"value": null
}
]

230
sahara/plugins/cdh/v5_3_0/resources/hive-hivemetastore.json Normal file
View File

@ -0,0 +1,230 @@
[
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Log Directory Free Space Monitoring Percentage Thresholds",
"name": "log_directory_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "Whether or not periodic stacks collection is enabled.",
"display_name": "Stacks Collection Enabled",
"name": "stacks_collection_enabled",
"value": "false"
},
{
"desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Java Configuration Options for Hive Metastore Server",
"name": "hive_metastore_java_opts",
"value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled"
},
{
"desc": "Maximum number of worker threads in the Hive Metastore Server's thread pool",
"display_name": "Max Hive Metastore Server Threads",
"name": "hive_metastore_max_threads",
"value": "100000"
},
{
"desc": "Enables the health test that checks that basic Hive Metastore operations succeed",
"display_name": "Hive Metastore Canary Health Test",
"name": "metastore_canary_health_enabled",
"value": "true"
},
{
"desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.",
"display_name": "Unexpected Exits Thresholds",
"name": "unexpected_exits_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
},
{
"desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
"display_name": "Automatically Restart Process",
"name": "process_auto_restart",
"value": "false"
},
{
"desc": "The maximum number of rolled log files to keep for Hive Metastore Server logs. Typically used by log4j.",
"display_name": "Hive Metastore Server Maximum Log File Backups",
"name": "max_log_backup_index",
"value": "10"
},
{
"desc": "<p>The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific role. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Role Triggers",
"name": "role_triggers",
"value": "[]"
},
{
"desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Soft Limit",
"name": "rm_memory_soft_limit",
"value": "-1"
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.",
"display_name": "Log Directory Free Space Monitoring Absolute Thresholds",
"name": "log_directory_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Hard Limit",
"name": "rm_memory_hard_limit",
"value": "-1"
},
{
"desc": "Minimum number of worker threads in the Hive Metastore Server's thread pool",
"display_name": "Min Hive Metastore Server Threads",
"name": "hive_metastore_min_threads",
"value": "200"
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
},
{
"desc": "When computing the overall Hive Metastore Server health, consider the host's health.",
"display_name": "Hive Metastore Server Host Health Test",
"name": "hivemetastore_host_health_enabled",
"value": "true"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>log4j.properties</strong> for this role only.",
"display_name": "Hive Metastore Server Logging Advanced Configuration Snippet (Safety Valve)",
"name": "log4j_safety_valve",
"value": null
},
{
"desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.",
"display_name": "Java Heap Size of Hive Metastore Server in Bytes",
"name": "hive_metastore_java_heapsize",
"value": "1073741824"
},
{
"desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.",
"display_name": "File Descriptor Monitoring Thresholds",
"name": "hivemetastore_fd_thresholds",
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "Port on which Hive Metastore Server will listen for connections.",
"display_name": "Hive Metastore Server Port",
"name": "hive_metastore_port",
"value": "9083"
},
{
"desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.",
"display_name": "Maximum Process File Descriptors",
"name": "rlimit_fds",
"value": null
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hive-site.xml</strong> for this role only.",
"display_name": "Hive Metastore Server Advanced Configuration Snippet (Safety Valve) for hive-site.xml",
"name": "hive_metastore_config_safety_valve",
"value": null
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Health Alerts for this Role",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "The maximum size, in megabytes, per log file for Hive Metastore Server logs. Typically used by log4j.",
"display_name": "Hive Metastore Server Max Log Size",
"name": "max_log_size",
"value": "200"
},
{
"desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.",
"display_name": "Kill When Out of Memory",
"name": "oom_sigkill_enabled",
"value": "true"
},
{
"desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.",
"display_name": "Dump Heap When Out of Memory",
"name": "oom_heap_dump_enabled",
"value": "false"
},
{
"desc": "The period to review when computing unexpected exits.",
"display_name": "Unexpected Exits Monitoring Period",
"name": "unexpected_exits_window",
"value": "5"
},
{
"desc": "Enables the health test that the Hive Metastore Server's process state is consistent with the role configuration",
"display_name": "Hive Metastore Server Process Health Test",
"name": "hivemetastore_scm_health_enabled",
"value": "true"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null
},
{
"desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup I/O Weight",
"name": "rm_io_weight",
"value": "500"
},
{
"desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup CPU Shares",
"name": "rm_cpu_shares",
"value": "1024"
},
{
"desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.",
"display_name": "Heap Dump Directory",
"name": "oom_heap_dump_dir",
"value": "/tmp"
},
{
"desc": "Directory where Hive Metastore Server will place its log files.",
"display_name": "Hive Metastore Server Log Directory",
"name": "hive_log_dir",
"value": "/var/log/hive"
},
{
"desc": "The frequency with which stacks will be collected.",
"display_name": "Stacks Collection Frequency",
"name": "stacks_collection_frequency",
"value": "5.0"
},
{
"desc": "The delegation token store implementation class. Use DBTokenStore for Highly Available Metastore Configuration.",
"display_name": "Hive Metastore Delegation Token Store",
"name": "hive_metastore_delegation_token_store",
"value": "org.apache.hadoop.hive.thrift.MemoryTokenStore"
},
{
"desc": "The minimum log level for Hive Metastore Server logs",
"display_name": "Hive Metastore Server Logging Threshold",
"name": "log_threshold",
"value": "INFO"
},
{
"desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.",
"display_name": "Hive Metastore Server Environment Advanced Configuration Snippet (Safety Valve)",
"name": "hive_metastore_env_safety_valve",
"value": null
}
]

272
sahara/plugins/cdh/v5_3_0/resources/hive-hiveserver2.json Normal file
View File

@ -0,0 +1,272 @@
[
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Log Directory Free Space Monitoring Percentage Thresholds",
"name": "log_directory_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "Operation will be closed when it's not accessed for this duration of time, which can be disabled by setting to zero. With a positive value, it's checked for operations in terminal state only (FINISHED, CANCELED, CLOSED, ERROR). With a negative value, it's checked for all of the operations regardless of state. Time in miliseconds.",
"display_name": "Idle Operation Timeout",
"name": "hiveserver2_idle_operation_timeout",
"value": "0"
},
{
"desc": "Whether or not periodic stacks collection is enabled.",
"display_name": "Stacks Collection Enabled",
"name": "stacks_collection_enabled",
"value": "false"
},
{
"desc": "Local Directory where Hive stores jars downloaded for remote file systems (HDFS). If not specified, Hive will use a default location.",
"display_name": "Hive Downloaded Resources Directory",
"name": "hiveserver2_downloaded_resources_dir",
"value": null
},
{
"desc": "HiveServer2 will impersonate the beeline client user when talking to other services such as Mapreduce and Hdfs.",
"display_name": "HiveServer2 Enable Impersonation",
"name": "hiveserver2_enable_impersonation",
"value": "true"
},
{
"desc": "Minimum number of worker threads in HiveServer2's thread pool",
"display_name": "Min HiveServer2 Threads",
"name": "hiveserver2_min_threads",
"value": "5"
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
},
{
"desc": "Port on which HiveServer2 will listen for connections.",
"display_name": "HiveServer2 Port",
"name": "hs2_thrift_address_port",
"value": "10000"
},
{
"desc": "Enable optimization that converts common join into map join based on the input file size.",
"display_name": "Enable MapJoin Optimization",
"name": "hiveserver2_enable_mapjoin",
"value": "true"
},
{
"desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
"display_name": "Automatically Restart Process",
"name": "process_auto_restart",
"value": "false"
},
{
"desc": "The maximum number of rolled log files to keep for HiveServer2 logs. Typically used by log4j.",
"display_name": "HiveServer2 Maximum Log File Backups",
"name": "max_log_backup_index",
"value": "10"
},
{
"desc": "<p>The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific role. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Role Triggers",
"name": "role_triggers",
"value": "[]"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hive-site.xml</strong> for this role only.",
"display_name": "HiveServer2 Advanced Configuration Snippet (Safety Valve) for hive-site.xml",
"name": "hive_hs2_config_safety_valve",
"value": null
},
{
"desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Soft Limit",
"name": "rm_memory_soft_limit",
"value": "-1"
},
{
"desc": "The check interval for session/operation timeout, which can be disabled by setting to zero or a negative value. Time in miliseconds.",
"display_name": "Session Check Interval",
"name": "hiveserver2_session_check_interval",
"value": "0"
},
{
"desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.",
"display_name": "Dump Heap When Out of Memory",
"name": "oom_heap_dump_enabled",
"value": "false"
},
{
"desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.",
"display_name": "HiveServer2 Environment Advanced Configuration Snippet (Safety Valve)",
"name": "hive_hs2_env_safety_valve",
"value": null
},
{
"desc": "Directory in HDFS where Hive writes intermediate data between Map Reduce jobs. If not specified, Hive will use a default location.",
"display_name": "Hive HDFS Scratch Directory",
"name": "hiveserver2_exec_scratchdir",
"value": null
},
{
"desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Hard Limit",
"name": "rm_memory_hard_limit",
"value": "-1"
},
{
"desc": "When enabled, HiveServer2 will log EXPLAIN EXTENDED output for every query at INFO log4j level.",
"display_name": "Enable Explain Logging",
"name": "hiveserver2_enable_explain_output",
"value": "true"
},
{
"desc": "The period to review when computing unexpected exits.",
"display_name": "Unexpected Exits Monitoring Period",
"name": "unexpected_exits_window",
"value": "5"
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>log4j.properties</strong> for this role only.",
"display_name": "HiveServer2 Logging Advanced Configuration Snippet (Safety Valve)",
"name": "log4j_safety_valve",
"value": null
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.",
"display_name": "Log Directory Free Space Monitoring Absolute Thresholds",
"name": "log_directory_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "When computing the overall HiveServer2 health, consider the host's health.",
"display_name": "HiveServer2 Host Health Test",
"name": "hiveserver2_host_health_enabled",
"value": "true"
},
{
"desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.",
"display_name": "Maximum Process File Descriptors",
"name": "rlimit_fds",
"value": null
},
{
"desc": "Session will be closed when it's not accessed for this duration, which can be disabled by setting to zero or a negative value. Time in miliseconds.",
"display_name": "Idle Session Timeout",
"name": "hiveserver2_idle_session_timeout",
"value": "0"
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Health Alerts for this Role",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "The maximum size, in megabytes, per log file for HiveServer2 logs. Typically used by log4j.",
"display_name": "HiveServer2 Max Log Size",
"name": "max_log_size",
"value": "200"
},
{
"desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.",
"display_name": "Kill When Out of Memory",
"name": "oom_sigkill_enabled",
"value": "true"
},
{
"desc": "Local Directory where Hive stores jars and data when performing a map join optimization. If not specified, Hive will use a default location.",
"display_name": "Hive Local Scratch Directory",
"name": "hiveserver2_exec_local_scratchdir",
"value": null
},
{
"desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.",
"display_name": "Unexpected Exits Thresholds",
"name": "unexpected_exits_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null
},
{
"desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup I/O Weight",
"name": "rm_io_weight",
"value": "500"
},
{
"desc": "Enables the health test that the HiveServer2's process state is consistent with the role configuration",
"display_name": "HiveServer2 Process Health Test",
"name": "hiveserver2_scm_health_enabled",
"value": "true"
},
{
"desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.",
"display_name": "Java Heap Size of HiveServer2 in Bytes",
"name": "hiveserver2_java_heapsize",
"value": "268435456"
},
{
"desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup CPU Shares",
"name": "rm_cpu_shares",
"value": "1024"
},
{
"desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.",
"display_name": "Heap Dump Directory",
"name": "oom_heap_dump_dir",
"value": "/tmp"
},
{
"desc": "Directory where HiveServer2 will place its log files.",
"display_name": "HiveServer2 Log Directory",
"name": "hive_log_dir",
"value": "/var/log/hive"
},
{
"desc": "The frequency with which stacks will be collected.",
"display_name": "Stacks Collection Frequency",
"name": "stacks_collection_frequency",
"value": "5.0"
},
{
"desc": "Maximum number of worker threads in HiveServer2's thread pool",
"display_name": "Max HiveServer2 Threads",
"name": "hiveserver2_max_threads",
"value": "100"
},
{
"desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.",
"display_name": "File Descriptor Monitoring Thresholds",
"name": "hiveserver2_fd_thresholds",
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "The minimum log level for HiveServer2 logs",
"display_name": "HiveServer2 Logging Threshold",
"name": "log_threshold",
"value": "INFO"
},
{
"desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Java Configuration Options for HiveServer2",
"name": "hiveserver2_java_opts",
"value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled"
}
]

326
sahara/plugins/cdh/v5_3_0/resources/hive-service.json Normal file
View File

@ -0,0 +1,326 @@
[
{
"desc": "HDFS path to the global policy file for Sentry authorization. This should be a relative path (and not a full HDFS URL). The global policy file must be in Sentry policy file format.",
"display_name": "Sentry Global Policy File",
"name": "hive_sentry_provider_resource",
"value": "/user/hive/sentry/sentry-provider.ini"
},
{
"desc": "Directory containing auxiliary JARs used by Hive. This should be a directory location and not a classpath containing one or more JARs. This directory must be created and managed manually on Hive CLI or HiveServer2 host.",
"display_name": "Hive Auxiliary JARs Directory",
"name": "hive_aux_jars_path_dir",
"value": null
},
{
"desc": "Password for Hive Metastore database",
"display_name": "Hive Metastore Database Password",
"name": "hive_metastore_database_password",
"value": ""
},
{
"desc": "Name of the Sentry service that this Hive service instance depends on. If selected, Hive uses this Sentry service to look up authorization privileges. Before enabling Sentry, read the requirements and configuration steps in <a class=\"bold\" href=\"http://tiny.cloudera.com/sentry-service-cm5\" target=\"_blank\">Setting Up The Sentry Service<i class=\"externalLink\"></i></a>.",
"display_name": "Sentry Service",
"name": "sentry_service",
"value": null
},
{
"desc": "Directory name where Hive Metastore's database is stored (only for Derby)",
"display_name": "Hive Metastore Derby Path",
"name": "hive_metastore_derby_path",
"value": "/var/lib/hive/cloudera_manager/derby/metastore_db"
},
{
"desc": "Hive warehouse directory is the location in HDFS where Hive's tables are stored. Note that Hive's default value for its warehouse directory is '/user/hive/warehouse'.",
"display_name": "Hive Warehouse Directory",
"name": "hive_warehouse_directory",
"value": "/user/hive/warehouse"
},
{
"desc": "SSL keystore password.",
"display_name": "Keystore Password",
"name": "hiveserver2_keystore_password",
"value": null
},
{
"desc": "<p>\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n</p>\n\n<p>\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n</p>\n\n<p>Event trackers are defined in a JSON object like the following:</p>\n\n<pre>\n{\n \"timeToLive\" : [integer],\n \"fields\" : [\n {\n \"type\" : [string],\n \"name\" : [string]\n }\n ]\n}\n</pre>\n\n<p>\nWhere:\n</p>\n\n<ul>\n <li>timeToLive: maximum amount of time an event will be tracked, in\n milliseconds. Must be provided. This defines how long, since it's\n first seen, an event will be tracked. A value of 0 disables tracking.</li>\n\n <li>fields: list of fields to compare when matching events against\n tracked events.</li>\n</ul>\n\n<p>\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n</p>\n\n<ul>\n <li>value: uses the field value for comparison.</li>\n\n <li>userName: treats the field value as a userNname, and ignores any\n host-specific data. This is useful for environment using Kerberos,\n so that only the principal name and realm are compared.</li>\n</ul>\n\n<p>\nThe following is the list of fields that can be used to compare Hive events:\n</p>\n\n<ul>\n <li>username: the user performing the action.</li>\n <li>ipAddress: the IP from where the request originated.</li>\n <li>operation: the Hive operation being performed.</li> \n <li>databaseName: the database affected by the operation.</li>\n <li>tableName: the table affected by the operation.</li> \n</ul>\n\n",
"display_name": "Event Tracker",
"name": "navigator_event_tracker",
"value": null
},
{
"desc": "Action to take when the audit event queue is full. Drop the event or shutdown the affected process.",
"display_name": "Queue Policy",
"name": "navigator_audit_queue_policy",
"value": "DROP"
},
{
"desc": "The user that this service's processes should run as.",
"display_name": "System User",
"name": "process_username",
"value": "hive"
},
{
"desc": "<p>Event filters are defined in a JSON object like the following:</p>\n\n<pre>\n{\n \"defaultAction\" : (\"accept\", \"discard\"),\n \"rules\" : [\n {\n \"action\" : (\"accept\", \"discard\"),\n \"fields\" : [\n {\n \"name\" : \"fieldName\",\n \"match\" : \"regex\"\n }\n ]\n }\n ]\n}\n</pre>\n\n<p>\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n</p>\n\n<p>\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n</p>\n\n<p>\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n</p>\n\n<p>\nThe following is the list of fields that can be filtered for Hive events:\n</p>\n\n<ul>\n <li>userName: the user performing the action.</li>\n <li>ipAddress: the IP from where the request originated.</li>\n <li>operation: the Hive operation being performed.</li> \n <li>databaseName: the databaseName for the operation.</li>\n <li>tableName: the tableName for the operation.</li>\n</ul>\n",
"display_name": "Event Filter",
"name": "navigator_audit_event_filter",
"value": "{\n \"comment\" : [\n \"Default filter for Hive services.\",\n \"Discards events generated by Hive MR jobs in /tmp directory\"\n ],\n \"defaultAction\" : \"accept\",\n \"rules\" : [\n {\n \"action\" : \"discard\",\n \"fields\" : [\n { \"name\" : \"operation\", \"match\" : \"QUERY\" },\n { \"name\" : \"objectType\", \"match\" : \"DFS_DIR\"},\n { \"name\" : \"resourcePath\", \"match\" : \"/tmp/hive-(?:.+)?/hive_(?:.+)?/-mr-.*\" }\n ]\n }\n ]\n}\n"
},
{
"desc": "Max number of reducers to use. If the configuration parameter Hive Reduce Tasks is negative, Hive will limit the number of reducers to the value of this parameter.",
"display_name": "Hive Max Reducers",
"name": "hive_max_reducers",
"value": "999"
},
{
"desc": "Enable support for encrypted client-server communication using Secure Socket Layer (SSL) for HiveServer2 connections. This is only applicable to non-Kerberos environments.",
"display_name": "Enable SSL for HiveServer",
"name": "hiveserver2_enable_ssl",
"value": "false"
},
{
"desc": "Type of Hive Metastore database. Note that Derby is not recommended and Cloudera Impala does not support Derby.",
"display_name": "Hive Metastore Database Type",
"name": "hive_metastore_database_type",
"value": "mysql"
},
{
"desc": "Maximum size of audit log file in MB before it is rolled over.",
"display_name": "Maximum Audit Log File Size",
"name": "navigator_audit_log_max_file_size",
"value": "100"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hive-site.xml</strong>. Applies to configurations of all roles in this service except client configuration.",
"display_name": "Hive Service Advanced Configuration Snippet (Safety Valve) for hive-site.xml",
"name": "hive_service_config_safety_valve",
"value": null
},
{
"desc": "Maximum number of rolled over audit logs to retain. The logs will not be deleted if they contain audit events that have not yet been propagated to Audit Server.",
"display_name": "Number of Audit Logs to Retain",
"name": "navigator_audit_log_max_backup_index",
"value": "10"
},
{
"desc": "Port number of Hive Metastore database",
"display_name": "Hive Metastore Database Port",
"name": "hive_metastore_database_port",
"value": "3306"
},
{
"desc": "Name of the ZooKeeper service that this Hive service instance depends on.",
"display_name": "ZooKeeper Service",
"name": "zookeeper_service",
"value": null
},
{
"desc": "Host name of Hive Metastore database",
"display_name": "Hive Metastore Database Host",
"name": "hive_metastore_database_host",
"value": "localhost"
},
{
"desc": "List of users that are allowed to bypass Sentry Authorization in Hive Metastore. These are usually service users that already ensure that all activity has been authorized, such as hive and impala. Only applies when Hive is using Sentry Service.",
"display_name": "Bypass Sentry Authorization Users",
"name": "sentry_metastore_service_users",
"value": "hive,impala,hue,hdfs"
},
{
"desc": "Prevent Metastore operations in the event of schema version incompatibility. Consider setting this to true to reduce probability of schema corruption during Metastore operations. Note that setting this property to true will also set datanucleus.autoCreateSchema property to false and datanucleus.fixedDatastore property to true. Any values set in Cloudera Manager for these properties will be overridden.",
"display_name": "Strict Hive Metastore Schema Validation",
"name": "hive_metastore_schema_verification",
"value": "true"
},
{
"desc": "Path to the SSL keystore.",
"display_name": "Keystore File Path",
"name": "hiveserver2_keystore_path",
"value": null
},
{
"desc": "Automatically create or upgrade tables in the Hive Metastore database when needed. Consider setting this to false and managing the schema manually.",
"display_name": "Auto Create and Upgrade Hive Metastore Database Schema",
"name": "hive_metastore_database_auto_create_schema",
"value": "false"
},
{
"desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.",
"display_name": "Hive Service Environment Advanced Configuration Snippet (Safety Valve)",
"name": "hive_service_env_safety_valve",
"value": null
},
{
"desc": "Path to the directory where audit logs will be written. The directory will be created if it doesn't exist.",
"display_name": "Audit Log Directory",
"name": "audit_event_log_dir",
"value": "/var/log/hive/audit"
},
{
"desc": "Name of Hive Metastore database",
"display_name": "Hive Metastore Database Name",
"name": "hive_metastore_database_name",
"value": "metastore"
},
{
"desc": "The class to use in Sentry authorization for user to group mapping. Sentry authorization may be configured to use either Hadoop user to group mapping or local groups defined in the policy file. Hadoop user to group mapping may be configured in the Cloudera Manager HDFS service configuration page under the Security section.",
"display_name": "Sentry User to Group Mapping Class",
"name": "hive_sentry_provider",
"value": "org.apache.sentry.provider.file.HadoopGroupResourceAuthorizationProvider"
},
{
"desc": "Use Sentry to enable role-based, fine-grained authorization. This configuration enables Sentry using policy files. To enable Sentry using Sentry service instead, add Sentry service as a dependency to Hive service. <strong>Sentry service provides concurrent and secure access to authorization policy metadata and is the recommended option for enabling Sentry. </strong> Sentry is supported only on CDH 4.4 or later deployments. Before enabling Sentry, read the requirements and configuration steps in <a class=\"bold\" href=\"http://tiny.cloudera.com/sentry-guide-cm5\" target=\"_blank\">Setting Up Hive Authorization with Sentry<i class=\"externalLink\"></i></a>.",
"display_name": "Enable Sentry Authorization using Policy Files",
"name": "sentry_enabled",
"value": "false"
},
{
"desc": "This configuration <strong>overrides</strong> the value set for Hive Proxy User Groups configuration in HDFS service for use by Hive Metastore Server. Specify a comma-delimited list of groups that you want to <strong>allow access to Hive Metastore metadata</strong> and allow the Hive user to impersonate. A value of '*' allows all groups. The default value of empty inherits the value set for Hive Proxy User Groups configuration in HDFS service.",
"display_name": "Hive Metastore Access Control and Proxy User Groups Override",
"name": "hive_proxy_user_groups_list",
"value": null
},
{
"desc": "Default number of reduce tasks per job. Usually set to a prime number close to the number of available hosts. Ignored when mapred.job.tracker is \"local\". Hadoop sets this to 1 by default, while Hive uses -1 as the default. When set to -1, Hive will automatically determine an appropriate number of reducers for each job.",
"display_name": "Hive Reduce Tasks",
"name": "hive_reduce_tasks",
"value": "-1"
},
{
"desc": "Let the table directories inherit the permission of the Warehouse or Database directory instead of being created with the permissions derived from dfs umask. This allows Impala to insert into tables created via Hive.",
"display_name": "Hive Warehouse Subdirectories Inherit Permissions",
"name": "hive_warehouse_subdir_inherit_perms",
"value": "true"
},
{
"desc": "The group that this service's processes should run as.",
"display_name": "System Group",
"name": "process_groupname",
"value": "hive"
},
{
"desc": "The health test thresholds of the overall Hive Metastore Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Hive Metastore Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Hive Metastore Servers falls below the critical threshold.",
"display_name": "Healthy Hive Metastore Server Monitoring Thresholds",
"name": "hive_hivemetastores_healthy_thresholds",
"value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}"
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Service Level Health Alerts",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "The server name used when defining privilege rules in Sentry authorization. Sentry uses this name as an alias for the Hive service. It does not correspond to any physical server name.",
"display_name": "Server Name for Sentry Authorization",
"name": "hive_sentry_server",
"value": "server1"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>core-site.xml</strong>. Applies to configurations of all roles in this service except client configuration.",
"display_name": "Hive Service Advanced Configuration Snippet (Safety Valve) for core-site.xml",
"name": "hive_core_site_safety_valve",
"value": null
},
{
"desc": "The health test thresholds of the overall HiveServer2 health. The check returns \"Concerning\" health if the percentage of \"Healthy\" HiveServer2s falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" HiveServer2s falls below the critical threshold.",
"display_name": "Healthy HiveServer2 Monitoring Thresholds",
"name": "hive_hiveserver2s_healthy_thresholds",
"value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}"
},
{
"desc": "For advanced use only, a list of configuration properties that will be used by the Service Monitor instead of the current client configuration for the service.",
"display_name": "Service Monitor Client Config Overrides",
"name": "smon_client_config_overrides",
"value": "<property><name>hive.metastore.client.socket.timeout</name><value>20</value></property>"
},
{
"desc": "<p>The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific service. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Service Triggers",
"name": "service_triggers",
"value": "[]"
},
{
"desc": "Instead of talking to Hive Metastore Server for Metastore information, Hive clients will talk directly to the Metastore database.",
"display_name": "Bypass Hive Metastore Server",
"name": "hive_bypass_metastore_server",
"value": "false"
},
{
"desc": "Below this size Hive will use a single threaded copy while above this size Hive will use DistCp.",
"display_name": "Hive Copy Large File Size",
"name": "hive_exec_copyfile_maxsize",
"value": "33554432"
},
{
"desc": "User for Hive Metastore database",
"display_name": "Hive Metastore Database User",
"name": "hive_metastore_database_user",
"value": "hive"
},
{
"desc": "Perform DataNucleus validation of metadata during startup. <strong>Note</strong>: when enabled, Hive will log DataNucleus warnings even though Hive will function normally.",
"display_name": "Hive Metastore Database DataNucleus Metadata Validation",
"name": "hive_metastore_database_datanucleus_metadata_validation",
"value": "false"
},
{
"desc": "For advanced use only, a string to be inserted into the client configuration for <strong>navigator.client.properties</strong>.",
"display_name": "Hive Client Advanced Configuration Snippet (Safety Valve) for navigator.client.properties",
"name": "navigator_client_config_safety_valve",
"value": null
},
{
"desc": "The health test thresholds of the overall WebHCat Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" WebHCat Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" WebHCat Servers falls below the critical threshold.",
"display_name": "Healthy WebHCat Server Monitoring Thresholds",
"name": "hive_webhcats_healthy_thresholds",
"value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>sentry-site.xml</strong>. Applies to configurations of all roles in this service except client configuration.",
"display_name": "Hive Service Advanced Configuration Snippet (Safety Valve) for sentry-site.xml",
"name": "hive_server2_sentry_safety_valve",
"value": null
},
{
"desc": "Size per reducer. If the input size is 10GiB and this is set to 1GiB, Hive will use 10 reducers.",
"display_name": "Hive Bytes Per Reducer",
"name": "hive_bytes_per_reducer",
"value": "1073741824"
},
{
"desc": "Enable collection of audit events from the service's roles.",
"display_name": "Enable Collection",
"name": "navigator_audit_enabled",
"value": "true"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "Allows URIs when defining privileges in per-database policy files. <strong>Warning:</strong> Typically, this configuration should be disabled. Enabling it would allow database policy file owner (which is generally not Hive admin user) to grant load privileges to any directory with read access to Hive admin user, including databases controlled by other database policy files.",
"display_name": "Allow URIs in Database Policy File",
"name": "sentry_allow_uri_db_policyfile",
"value": "false"
},
{
"desc": "In unsecure mode, setting this property to true will cause the Metastore Server to execute DFS operations using the client's reported user and group permissions. Cloudera Manager will set this for all clients and servers.",
"display_name": "Set User and Group Information",
"name": "hive_set_ugi",
"value": "true"
},
{
"desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.",
"display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)",
"name": "smon_derived_configs_safety_valve",
"value": null
},
{
"desc": "MapReduce jobs are run against this service.",
"display_name": "MapReduce Service",
"name": "mapreduce_yarn_service",
"value": null
}
]

212
sahara/plugins/cdh/v5_3_0/resources/hive-webhcat.json Normal file
View File

@ -0,0 +1,212 @@
[
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Log Directory Free Space Monitoring Percentage Thresholds",
"name": "log_directory_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "Whether or not periodic stacks collection is enabled.",
"display_name": "Stacks Collection Enabled",
"name": "stacks_collection_enabled",
"value": "false"
},
{
"desc": "Port on which WebHCat Server will listen for connections.",
"display_name": "WebHCat Server Port",
"name": "hive_webhcat_address_port",
"value": "50111"
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
},
{
"desc": "The maximum size, in megabytes, per log file for WebHCat Server logs. Typically used by log4j.",
"display_name": "WebHCat Server Max Log Size",
"name": "max_log_size",
"value": "200"
},
{
"desc": "The maximum number of rolled log files to keep for WebHCat Server logs. Typically used by log4j.",
"display_name": "WebHCat Server Maximum Log File Backups",
"name": "max_log_backup_index",
"value": "10"
},
{
"desc": "<p>The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific role. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Role Triggers",
"name": "role_triggers",
"value": "[]"
},
{
"desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Soft Limit",
"name": "rm_memory_soft_limit",
"value": "-1"
},
{
"desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.",
"display_name": "Java Heap Size of WebHCat Server in Bytes",
"name": "hive_webhcat_java_heapsize",
"value": "268435456"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hive-site.xml</strong> for this role only.",
"display_name": "WebHCat Server Advanced Configuration Snippet (Safety Valve) for hive-site.xml",
"name": "hive_webhcat_hive_config_safety_valve",
"value": null
},
{
"desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Hard Limit",
"name": "rm_memory_hard_limit",
"value": "-1"
},
{
"desc": "The period to review when computing unexpected exits.",
"display_name": "Unexpected Exits Monitoring Period",
"name": "unexpected_exits_window",
"value": "5"
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
},
{
"desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.",
"display_name": "File Descriptor Monitoring Thresholds",
"name": "webhcat_fd_thresholds",
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>log4j.properties</strong> for this role only.",
"display_name": "WebHCat Server Logging Advanced Configuration Snippet (Safety Valve)",
"name": "log4j_safety_valve",
"value": null
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.",
"display_name": "Log Directory Free Space Monitoring Absolute Thresholds",
"name": "log_directory_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.",
"display_name": "WebHCat Server Environment Advanced Configuration Snippet (Safety Valve)",
"name": "hive_webhcat_env_safety_valve",
"value": null
},
{
"desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Java Configuration Options for WebHCat Server",
"name": "hive_webhcat_java_opts",
"value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled"
},
{
"desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.",
"display_name": "Maximum Process File Descriptors",
"name": "rlimit_fds",
"value": null
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Health Alerts for this Role",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
"display_name": "Automatically Restart Process",
"name": "process_auto_restart",
"value": "false"
},
{
"desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.",
"display_name": "Kill When Out of Memory",
"name": "oom_sigkill_enabled",
"value": "true"
},
{
"desc": "Enables the health test that the WebHCat Server's process state is consistent with the role configuration",
"display_name": "WebHCat Server Process Health Test",
"name": "webhcat_scm_health_enabled",
"value": "true"
},
{
"desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.",
"display_name": "Dump Heap When Out of Memory",
"name": "oom_heap_dump_enabled",
"value": "false"
},
{
"desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.",
"display_name": "Unexpected Exits Thresholds",
"name": "unexpected_exits_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>webhcat-site.xml</strong> for this role only.",
"display_name": "WebHCat Server Advanced Configuration Snippet (Safety Valve) for webhcat-site.xml",
"name": "hive_webhcat_config_safety_valve",
"value": null
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "When computing the overall WebHCat Server health, consider the host's health.",
"display_name": "WebHCat Server Host Health Test",
"name": "webhcat_host_health_enabled",
"value": "true"
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null
},
{
"desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup I/O Weight",
"name": "rm_io_weight",
"value": "500"
},
{
"desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup CPU Shares",
"name": "rm_cpu_shares",
"value": "1024"
},
{
"desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.",
"display_name": "Heap Dump Directory",
"name": "oom_heap_dump_dir",
"value": "/tmp"
},
{
"desc": "The frequency with which stacks will be collected.",
"display_name": "Stacks Collection Frequency",
"name": "stacks_collection_frequency",
"value": "5.0"
},
{
"desc": "The minimum log level for WebHCat Server logs",
"display_name": "WebHCat Server Logging Threshold",
"name": "log_threshold",
"value": "INFO"
},
{
"desc": "Directory where WebHCat Server will place its log files.",
"display_name": "WebHCat Server Log Directory",
"name": "hcatalog_log_dir",
"value": "/var/log/hcatalog"
}
]

158
sahara/plugins/cdh/v5_3_0/resources/hue-hue_server.json Normal file
View File

@ -0,0 +1,158 @@
[
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Log Directory Free Space Monitoring Percentage Thresholds",
"name": "log_directory_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.",
"display_name": "File Descriptor Monitoring Thresholds",
"name": "hue_server_fd_thresholds",
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "Enable HTTPS for the Hue web server.",
"display_name": "Enable HTTPS",
"name": "ssl_enable",
"value": "false"
},
{
"desc": "Path to the SSL private key on the host running the Hue web server. This file must be readable by the Hue system user. Hue only supports a key without a passphrase.",
"display_name": "Local Path to SSL Private Key",
"name": "ssl_private_key",
"value": null
},
{
"desc": "Directory where Hue Server will place its log files.",
"display_name": "Hue Server Log Directory",
"name": "hue_server_log_dir",
"value": "/var/log/hue"
},
{
"desc": "Timeout in seconds for Thrift calls to HiveServer2 and Impala.",
"display_name": "HiveServer2 and Impala Thrift Connection Timeout",
"name": "hs2_conn_timeout",
"value": "120"
},
{
"desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
"display_name": "Automatically Restart Process",
"name": "process_auto_restart",
"value": "false"
},
{
"desc": "<p>The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific role. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Role Triggers",
"name": "role_triggers",
"value": "[]"
},
{
"desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Soft Limit",
"name": "rm_memory_soft_limit",
"value": "-1"
},
{
"desc": "Location on HDFS where the jobsub examples and templates are stored.",
"display_name": "Jobsub Examples and Templates Directory",
"name": "hue_server_remote_data_dir",
"value": "/user/hue/jobsub"
},
{
"desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Hard Limit",
"name": "rm_memory_hard_limit",
"value": "-1"
},
{
"desc": "The period to review when computing unexpected exits.",
"display_name": "Unexpected Exits Monitoring Period",
"name": "unexpected_exits_window",
"value": "5"
},
{
"desc": "When computing the overall Hue Server health, consider the host's health.",
"display_name": "Hue Server Host Health Test",
"name": "hue_server_host_health_enabled",
"value": "true"
},
{
"desc": "Port to use to connect to the Hue server.",
"display_name": "Hue HTTP Port",
"name": "hue_http_port",
"value": "8888"
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.",
"display_name": "Log Directory Free Space Monitoring Absolute Thresholds",
"name": "log_directory_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hue_safety_valve_server.ini</strong> for this role only.",
"display_name": "Hue Server Advanced Configuration Snippet (Safety Valve) for hue_safety_valve_server.ini",
"name": "hue_server_hue_safety_valve",
"value": null
},
{
"desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.",
"display_name": "Maximum Process File Descriptors",
"name": "rlimit_fds",
"value": null
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Health Alerts for this Role",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.",
"display_name": "Unexpected Exits Thresholds",
"name": "unexpected_exits_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "Path to the SSL certificate on the host running the Hue web server. This file must be readable by the Hue system user.",
"display_name": "Local Path to SSL Certificate",
"name": "ssl_certificate",
"value": null
},
{
"desc": "If enabled, the Hue server binds to the wildcard address (\"0.0.0.0\") for its ports.",
"display_name": "Bind Hue Server to Wildcard Address",
"name": "hue_server_bind_wildcard",
"value": "false"
},
{
"desc": "Enables the health test that the Hue Server's process state is consistent with the role configuration",
"display_name": "Hue Server Process Health Test",
"name": "hue_server_scm_health_enabled",
"value": "true"
},
{
"desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup I/O Weight",
"name": "rm_io_weight",
"value": "500"
},
{
"desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup CPU Shares",
"name": "rm_cpu_shares",
"value": "1024"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "Random string used for secure hashing in the session store.",
"display_name": "Secret Key",
"name": "secret_key",
"value": null
}
]

110
sahara/plugins/cdh/v5_3_0/resources/hue-kt_renewer.json Normal file
View File

@ -0,0 +1,110 @@
[
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Log Directory Free Space Monitoring Percentage Thresholds",
"name": "log_directory_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Health Alerts for this Role",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "Directory where Kerberos Ticket Renewer will place its log files.",
"display_name": "Kerberos Ticket Renewer Log Directory",
"name": "kt_renewer_log_dir",
"value": "/var/log/hue"
},
{
"desc": "<p>The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific role. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Role Triggers",
"name": "role_triggers",
"value": "[]"
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.",
"display_name": "Log Directory Free Space Monitoring Absolute Thresholds",
"name": "log_directory_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "When computing the overall Kerberos Ticket Renewer health, consider the host's health.",
"display_name": "Kerberos Ticket Renewer Host Health Test",
"name": "kt_renewer_host_health_enabled",
"value": "true"
},
{
"desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Soft Limit",
"name": "rm_memory_soft_limit",
"value": "-1"
},
{
"desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup I/O Weight",
"name": "rm_io_weight",
"value": "500"
},
{
"desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.",
"display_name": "Maximum Process File Descriptors",
"name": "rlimit_fds",
"value": null
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup CPU Shares",
"name": "rm_cpu_shares",
"value": "1024"
},
{
"desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
"display_name": "Automatically Restart Process",
"name": "process_auto_restart",
"value": "false"
},
{
"desc": "Interval in seconds with which Hue's Kerberos ticket will get renewed.",
"display_name": "Hue Keytab Renewal Interval",
"name": "keytab_reinit_frequency",
"value": "3600"
},
{
"desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Hard Limit",
"name": "rm_memory_hard_limit",
"value": "-1"
},
{
"desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.",
"display_name": "File Descriptor Monitoring Thresholds",
"name": "kt_renewer_fd_thresholds",
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "The period to review when computing unexpected exits.",
"display_name": "Unexpected Exits Monitoring Period",
"name": "unexpected_exits_window",
"value": "5"
},
{
"desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.",
"display_name": "Unexpected Exits Thresholds",
"name": "unexpected_exits_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "Enables the health test that the Kerberos Ticket Renewer's process state is consistent with the role configuration",
"display_name": "Kerberos Ticket Renewer Process Health Test",
"name": "kt_renewer_scm_health_enabled",
"value": "true"
}
]

344
sahara/plugins/cdh/v5_3_0/resources/hue-service.json Normal file
View File

@ -0,0 +1,344 @@
[
{
"desc": "Create users in Hue when they try to login with their LDAP credentials. For use when using LdapBackend for Hue authentication.",
"display_name": "Create LDAP users on login",
"name": "create_users_on_login",
"value": "true"
},
{
"desc": "Number of threads used by the Hue web server.",
"display_name": "Hue Web Server Threads",
"name": "cherrypy_server_threads",
"value": "10"
},
{
"desc": "Thrift server to use for HBase app.",
"display_name": "HBase Thrift Server",
"name": "hue_hbase_thrift",
"value": null
},
{
"desc": "Name of the Oozie service that this Hue service instance depends on",
"display_name": "Oozie Service",
"name": "oozie_service",
"value": null
},
{
"desc": "The distinguished name to use as a search base for finding users and groups.",
"display_name": "LDAP Search Base",
"name": "base_dn",
"value": null
},
{
"desc": "Comma-separated list of regular expressions, which match 'host:port' of requested proxy target.",
"display_name": "Whitelist",
"name": "whitelist",
"value": "(localhost|127\\.0\\.0\\.1):(50030|50070|50060|50075)"
},
{
"desc": "Name of the Sentry service that this Hue service instance depends on",
"display_name": "Sentry Service",
"name": "sentry_service",
"value": null
},
{
"desc": "Name of the HBase service that this Hue service instance depends on",
"display_name": "HBase Service",
"name": "hbase_service",
"value": null
},
{
"desc": "Base filter for searching for groups",
"display_name": "LDAP Group Filter",
"name": "group_filter",
"value": null
},
{
"desc": "Time zone name.",
"display_name": "Time Zone",
"name": "time_zone",
"value": "America/Los_Angeles"
},
{
"desc": "The health test thresholds of the overall Kerberos Ticket Renewer health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Kerberos Ticket Renewers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Kerberos Ticket Renewers falls below the critical threshold.",
"display_name": "Healthy Kerberos Ticket Renewer Monitoring Thresholds",
"name": "hue_kt_renewers_healthy_thresholds",
"value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}"
},
{
"desc": "The user that this service's processes should run as.",
"display_name": "System User",
"name": "process_username",
"value": "hue"
},
{
"desc": "The health test thresholds of the overall Hue Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Hue Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Hue Servers falls below the critical threshold.",
"display_name": "Healthy Hue Server Monitoring Thresholds",
"name": "hue_hue_servers_healthy_thresholds",
"value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "The group name attribute in the LDAP schema.",
"display_name": "LDAP Group Name Attribute",
"name": "group_name_attr",
"value": null
},
{
"desc": "HDFS directory used for storing temporary files.",
"display_name": "HDFS Temporary Directory",
"name": "hdfs_tmp_dir",
"value": "/tmp"
},
{
"desc": "Name of the Solr service that this Hue service instance depends on",
"display_name": "Solr Service",
"name": "solr_service",
"value": null
},
{
"desc": "If the database is SQLite3, this is the filename of the database to use, and the directory of this file must be writable by the 'hue' user.",
"display_name": "Hue Database Directory",
"name": "database_dir",
"value": "/var/lib/hue/desktop.db"
},
{
"desc": "The name of a default group that users will be added to at creation time.",
"display_name": "Default User Group",
"name": "default_user_group",
"value": null
},
{
"desc": "When you enable anonymous usage data collection Hue tracks anonymised pages and application versions in order to gather information about each application's usage levels. The data collected does not include any hostnames or IDs. Data collection option is available on CDH 4.4 and later deployments.",
"display_name": "Enable Usage Data Collection",
"name": "usage_data_collection_enable",
"value": "true"
},
{
"desc": "Name of the ZooKeeper service that this Hue service instance depends on",
"display_name": "ZooKeeper Service",
"name": "zookeeper_service",
"value": null
},
{
"desc": "Distinguished name of the user to bind as -- not necessary if the LDAP server supports anonymous searches.",
"display_name": "LDAP Bind User",
"name": "bind_dn",
"value": null
},
{
"desc": "URL of LDAP Server",
"display_name": "LDAP URL",
"name": "ldap_url",
"value": null
},
{
"desc": "Whether to use StartTLS (as opposed to ldaps) to communicate securely with the LDAP server. This is only effective when the LDAP certificate is specified.",
"display_name": "Use StartTLS",
"name": "use_start_tls",
"value": "true"
},
{
"desc": "Name of the Sqoop service that this Hue service instance depends on",
"display_name": "Sqoop Service",
"name": "sqoop_service",
"value": null
},
{
"desc": "Active Directory Domain",
"display_name": "NT Domain",
"name": "nt_domain",
"value": null
},
{
"desc": "Class that defines extra accessor methods for user objects.",
"display_name": "User Augmentor",
"name": "user_augmentor",
"value": "desktop.auth.backend.DefaultUserAugmentor"
},
{
"desc": "File where the database gets dumped to or loaded from.",
"display_name": "Database Dump File",
"name": "database_dump_file",
"value": "/tmp/hue_database_dump.json"
},
{
"desc": "LDAP Username Pattern for use with non-Active Directory LDAP implementations. Must contain the special '&ltusername&gt' string for replacement during authentication.",
"display_name": "LDAP Username Pattern",
"name": "ldap_username_pattern",
"value": null
},
{
"desc": "The password of the bind user.",
"display_name": "LDAP Bind Password",
"name": "bind_password",
"value": null
},
{
"desc": "The group that this service's processes should run as.",
"display_name": "System Group",
"name": "process_groupname",
"value": "hue"
},
{
"desc": "In debug mode, Django displays a detailed traceback when an exception occurs. Note that the debugging information may contain sensitive data. Note also that Django remembers every SQL query it executes while in debug mode, which will rapidly consume memory.",
"display_name": "Enable Django Debug Mode",
"name": "django_debug_enable",
"value": "false"
},
{
"desc": "LDAP certificate for authentication over TLS",
"display_name": "LDAP Certificate",
"name": "ldap_cert",
"value": null
},
{
"desc": "Type of database used for Hue",
"display_name": "Hue Database Type",
"name": "database_type",
"value": "sqlite3"
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Service Level Health Alerts",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "When computing the overall HUE health, consider Beeswax Server's health",
"display_name": "Beeswax Server Role Health Test",
"name": "hue_beeswax_server_health_enabled",
"value": "true"
},
{
"desc": "HTTPFS role or Namenode (if webhdfs is enabled) that hue can use to communicate with HDFS.",
"display_name": "HDFS Web Interface Role",
"name": "hue_webhdfs",
"value": null
},
{
"desc": "Mode of authenticating login credentials. Select desktop.auth.backend.LdapBackend to use LDAP to authenticate login credentials. LDAP requires you to also set the LDAP URL, NT Domain, and optionally LDAP certificate if you are using secure LDAP. Select desktop.auth.backend.PamBackend to use PAM to authenticate login credentials.",
"display_name": "Authentication Backend",
"name": "auth_backend",
"value": "desktop.auth.backend.AllowFirstUserDjangoBackend"
},
{
"desc": "<p>The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific service. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Service Triggers",
"name": "service_triggers",
"value": "[]"
},
{
"desc": "The username to use to log into the Hue database. Not necessary for SQLite3.",
"display_name": "Hue Database Username",
"name": "database_user",
"value": "hue"
},
{
"desc": "Enable debug output in HTTP Internal Server Error (status 500) responses. Note that the debugging information may contain sensitive data. If Enable Django Debug Mode is set, this is automatically enabled.",
"display_name": "Enable Debugging of Internal Server Error Responses",
"name": "http_500_debug_enable",
"value": "false"
},
{
"desc": "Comma-separated list of regular expressions, which match any prefix of 'host:port/path' of requested proxy target. This does not support matching GET parameters.",
"display_name": "Blacklist",
"name": "blacklist",
"value": "()"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>sentry-site.xml</strong>. Applies to configurations of all roles in this service except client configuration.",
"display_name": "Hue Service Advanced Configuration Snippet (Safety Valve) for sentry-site.xml",
"name": "hue_sentry_safety_valve",
"value": null
},
{
"desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.",
"display_name": "Hue Service Environment Advanced Configuration Snippet (Safety Valve)",
"name": "hue_service_env_safety_valve",
"value": null
},
{
"desc": "Name of the Impala service that this Hue service instance depends on",
"display_name": "Impala Service",
"name": "impala_service",
"value": null
},
{
"desc": "Name of Hue database.",
"display_name": "Hue Database Name",
"name": "database_name",
"value": "hue"
},
{
"desc": "The attribute of the group object which identifies the members of the group.",
"display_name": "LDAP Group Membership Attribute",
"name": "group_member_attr",
"value": null
},
{
"desc": "Name of the Hive service that this Hue service instance depends on",
"display_name": "Hive Service",
"name": "hive_service",
"value": null
},
{
"desc": "The username attribute in the LDAP schema.",
"display_name": "LDAP Username Attribute",
"name": "user_name_attr",
"value": null
},
{
"desc": "Default encoding for site data.",
"display_name": "Default Site Encoding",
"name": "default_site_encoding",
"value": "utf"
},
{
"desc": "Search Bind Authentication connects to the LDAP server using credentials provided in the 'bind_dn' and 'bind_password' configurations. If these configurations are not set, then an anonymous search is performed.",
"display_name": "Use Search Bind Authentication",
"name": "search_bind_authentication",
"value": "false"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hue_safety_valve.ini</strong>. Applies to configurations of all roles in this service except client configuration.",
"display_name": "Hue Service Advanced Configuration Snippet (Safety Valve) for hue_safety_valve.ini",
"name": "hue_service_safety_valve",
"value": null
},
{
"desc": "Port on host where the Hue database is running. Not necessary for SQLite3.",
"display_name": "Hue Database Port",
"name": "database_port",
"value": "3306"
},
{
"desc": "Password for Hue database. Not necessary for SQLite3.",
"display_name": "Hue Database Password",
"name": "database_password",
"value": ""
},
{
"desc": "Base filter for searching for users.",
"display_name": "LDAP User Filter",
"name": "user_filter",
"value": null
},
{
"desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.",
"display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)",
"name": "smon_derived_configs_safety_valve",
"value": null
},
{
"desc": "Name of host where the Hue database is running. Not necessary for SQLite3.",
"display_name": "Hue Database Hostname",
"name": "database_host",
"value": "localhost"
}
]

6
sahara/plugins/cdh/resources/impala-catalogserver.json → sahara/plugins/cdh/v5_3_0/resources/impala-catalogserver.json
View File

@ -41,6 +41,12 @@
"name": "logbuflevel",
"value": "0"
},
{
"desc": "If true, loads catalog metadata in the background. If false, metadata is loaded lazily (on access). Only effective in CDH 5+ and Impala 1.2.4+.",
"display_name": "Load Catalog in Background",
"name": "load_catalog_in_background",
"value": "true"
},
{
"desc": "The location of the debug web server's SSL certificate file, in .pem format. If empty, webserver SSL support is not enabled.",
"display_name": "SSL Certificate File Location",

38
sahara/plugins/cdh/resources/impala-impalad.json → sahara/plugins/cdh/v5_3_0/resources/impala-impalad.json
View File

@ -17,12 +17,24 @@
"name": "local_library_dir",
"value": "/var/lib/impala/udfs"
},
{
"desc": "Encrypt and verify the integrity of all data spilled to disk as part of a query. This feature is only supported for Impala 2.0+ and CDH 5.2+ (which includes Impala 2.0).",
"display_name": "Disk Spill Encryption",
"name": "disk_spill_encryption",
"value": "false"
},
{
"desc": "Maximum number of query results a client may request to be cached on a per-query basis to support restarting fetches. This option guards against unreasonably large result caches requested by clients. Requests exceeding this maximum will be rejected.",
"display_name": "Result Cache Maximum Size",
"name": "impalad_result_cache_max_size",
"value": "100000"
},
{
"desc": "Number of seconds to wait between attempts during Llama registration.",
"display_name": "Llama Registration Wait Seconds",
"name": "llama_registration_wait_secs",
"value": "3"
},
{
"desc": "The timeout used by the Cloudera Manager Agent's query monitor when communicating with the Impala Daemon web server, specified in seconds.",
"display_name": "Query Monitoring Timeout",
@ -83,6 +95,12 @@
"name": "impalad_enable_webserver",
"value": "true"
},
{
"desc": "The location on disk of the certificate, in .pem format, used to confirm the authenticity of the LDAP server certificate. If not set, Impala by default trusts all certificates supplied by the LDAP server.",
"display_name": "LDAP Server Certificate",
"name": "impalad_ldap_ca_certificate",
"value": null
},
{
"desc": "Port on which HiveServer2 client requests are served by Impala Daemons.",
"display_name": "Impala Daemon HiveServer2 Port",
@ -113,6 +131,12 @@
"name": "enable_audit_event_log",
"value": "false"
},
{
"desc": "Maximum number of seconds that Impala attempts to register or re-register with Llama. If registration is unsuccessful, Impala cancels the action with an error, which could result in an impalad startup failure or a cancelled query. A setting of -1 seconds means try indefinitely.",
"display_name": "Llama Registration Timeout Seconds",
"name": "llama_registration_timeout_secs",
"value": "30"
},
{
"desc": "When computing the overall Impala Daemon health, consider the host's health.",
"display_name": "Impala Daemon Host Health Test",
@ -198,10 +222,10 @@
"value": "true"
},
{
"desc": "Directory where Impala Daemon will place its log files.",
"display_name": "Impala Daemon Log Directory",
"name": "log_dir",
"value": "/var/log/impalad"
"desc": "Maximum number of times a request to reserve, expand, or release resources is attempted until the request is cancelled.",
"display_name": "Llama Maximum Request Attempts",
"name": "llama_max_request_attempts",
"value": "5"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>hbase-site.xml</strong> for this role only.",
@ -239,6 +263,12 @@
"name": "unexpected_exits_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "Directory where Impala Daemon will place its log files.",
"display_name": "Impala Daemon Log Directory",
"name": "log_dir",
"value": "/var/log/impalad"
},
{
"desc": "The amount of time at Impala Daemon startup allowed for the Impala Daemon to start accepting new queries for processing.",
"display_name": "Impala Daemon Ready Status Startup Tolerance",

6
sahara/plugins/cdh/resources/impala-llama.json → sahara/plugins/cdh/v5_3_0/resources/impala-llama.json
View File

@ -72,7 +72,7 @@
"value": "10"
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that expose an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
@ -150,7 +150,7 @@
"value": "500"
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is hit, the oldest data will be deleted.",
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
@ -324,7 +324,7 @@
"value": "5.0"
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span>subdirectory of the role's log directory.",
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null

126
sahara/plugins/cdh/resources/impala-service.json → sahara/plugins/cdh/v5_3_0/resources/impala-service.json
View File

@ -1,10 +1,22 @@
[
{
"desc": "The health test thresholds for assignment locality. Specified as a percentage of total assignments.",
"display_name": "Assignment Locality Ratio Thresholds",
"name": "impala_assignment_locality_thresholds",
"value": "{\"critical\":\"5.0\",\"warning\":\"80.0\"}"
},
{
"desc": "Name of the HBase service that this Impala service instance depends on",
"display_name": "HBase Service",
"name": "hbase_service",
"value": null
},
{
"desc": "The user that this Impala's processes should run as (except Llama, which has its own user).",
"display_name": "Impala System User (except Llama)",
"name": "process_username",
"value": "impala"
},
{
"desc": "JSON representation of all the configurations that the Fair Scheduler can take on across all schedules. Typically edited using the Pools configuration UI. This configuration only has effect on Impala versions 1.3 or greater.",
"display_name": "Fair Scheduler Allocations",
@ -12,9 +24,9 @@
"value": "{\"defaultMinSharePreemptionTimeout\":null,\"defaultQueueSchedulingPolicy\":null,\"fairSharePreemptionTimeout\":null,\"queueMaxAppsDefault\":null,\"queuePlacementRules\":null,\"queues\":[{\"aclAdministerApps\":null,\"aclSubmitApps\":null,\"minSharePreemptionTimeout\":null,\"name\":\"root\",\"queues\":[{\"aclAdministerApps\":null,\"aclSubmitApps\":null,\"minSharePreemptionTimeout\":null,\"name\":\"default\",\"queues\":[],\"schedulablePropertiesList\":[{\"impalaMaxMemory\":null,\"impalaMaxQueuedQueries\":null,\"impalaMaxRunningQueries\":null,\"maxResources\":null,\"maxRunningApps\":null,\"minResources\":null,\"scheduleName\":\"default\",\"weight\":null}],\"schedulingPolicy\":null}],\"schedulablePropertiesList\":[{\"impalaMaxMemory\":null,\"impalaMaxQueuedQueries\":null,\"impalaMaxRunningQueries\":null,\"maxResources\":null,\"maxRunningApps\":null,\"minResources\":null,\"scheduleName\":\"default\",\"weight\":null}],\"schedulingPolicy\":null}],\"userMaxAppsDefault\":null,\"users\":[]}"
},
{
"desc": "For advanced use only, a string to be inserted into the client configuration for <strong>navigator.client.properties</strong>.",
"display_name": "Impala Client Advanced Configuration Snippet (Safety Valve) for navigator.client.properties",
"name": "navigator_client_config_safety_valve",
"desc": "Name of the Sentry service that this Impala service instance depends on. If selected, Impala uses this Sentry service to look up authorization privileges. Before enabling Sentry, read the requirements and configuration steps in <a class=\"bold\" href=\"http://tiny.cloudera.com/sentry-service-cm5\" target=\"_blank\">Setting Up The Sentry Service<i class=\"externalLink\"></i></a>.",
"display_name": "Sentry Service",
"name": "sentry_service",
"value": null
},
{
@ -30,16 +42,22 @@
"value": null
},
{
"desc": "Name of YARN service to use for resource management integration between Impala and YARN. This service dependency and the existence of a Llama role is required for using said integration.",
"display_name": "YARN Service for Resource Management",
"name": "yarn_service",
"desc": "Use Impala Admission Control to throttle Impala requests. Unless 'Enable Dynamic Resource Pools' is enabled, Impala uses a single, default pool that is configured using the Single Pool configurations below. These features are only supported in Impala 1.3 or later deployments.",
"display_name": "Enable Impala Admission Control",
"name": "all_admission_control_enabled",
"value": "false"
},
{
"desc": "For advanced use only, a string to be inserted into the client configuration for <strong>navigator.client.properties</strong>.",
"display_name": "Impala Client Advanced Configuration Snippet (Safety Valve) for navigator.client.properties",
"name": "navigator_client_config_safety_valve",
"value": null
},
{
"desc": "The user that this Impala's processes should run as (except Llama, which has its own user).",
"display_name": "Impala System User (except Llama)",
"name": "process_username",
"value": "impala"
"desc": "Configures the maximum number of queued queries for admission control when using a single pool. -1 or 0 disables queuing, i.e. incoming requests are rejected if they can not be executed immediately. Ignored when Dynamic Resource Pools for Admission Control is enabled.",
"display_name": "Single Pool Max Queued Queries",
"name": "admission_control_single_pool_max_queued",
"value": "200"
},
{
"desc": "<p>Event filters are defined in a JSON object like the following:</p>\n\n<pre>\n{\n \"defaultAction\" : (\"accept\", \"discard\"),\n \"rules\" : [\n {\n \"action\" : (\"accept\", \"discard\"),\n \"fields\" : [\n {\n \"name\" : \"fieldName\",\n \"match\" : \"regex\"\n }\n ]\n }\n ]\n}\n</pre>\n\n<p>\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n</p>\n\n<p>\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n</p>\n\n<p>\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n</p>\n\n<p>\nThe following is the list of fields that can be filtered for Impala events:\n</p>\n\n<ul>\n <li>userName: the user performing the action.</li>\n <li>ipAddress: the IP from where the request originated.</li>\n <li>operation: the Impala operation being performed.</li> \n <li>databaseName: the databaseName for the operation.</li>\n <li>tableName: the tableName for the operation.</li>\n</ul>\n",
@ -72,16 +90,28 @@
"value": null
},
{
"desc": "Local path to the SSL server certificate file.",
"display_name": "SSL Server Certificate",
"name": "ssl_server_certificate",
"value": null
"desc": "The health test thresholds of the overall Impala Llama ApplicationMaster health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Impala Llama ApplicationMasters falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Impala Llama ApplicationMasters falls below the critical threshold.",
"display_name": "Healthy Impala Llama ApplicationMaster Monitoring Thresholds",
"name": "impala_llamas_healthy_thresholds",
"value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}"
},
{
"desc": "Use Dynamic Resource Pools to configure resource pools used for admission control and resource management for this Impala service. These features are only supported in Impala 1.3 or later deployments.",
"desc": "Maximum amount of time (in milliseconds) that a request waits to be admitted before timing out. Must be a positive integer.",
"display_name": "Admission Control Queue Timeout",
"name": "admission_control_queue_timeout",
"value": "60000"
},
{
"desc": "Use Dynamic Resource Pools to configure Impala admission control or RM for this Impala service. These features are only supported in Impala 1.3 or later deployments.",
"display_name": "Enable Dynamic Resource Pools",
"name": "admission_control_enabled",
"value": "true"
"value": "false"
},
{
"desc": "Name of the Zookeeper service to use for leader election and fencing when Llama is configured for High Availability. This service dependency is required when more than one Llama role is present.",
"display_name": "Zookeeper Service for Llama HA",
"name": "zookeeper_service",
"value": null
},
{
"desc": "Number of minutes between reestablishing our ticket with the Kerberos server.",
@ -96,10 +126,16 @@
"value": "30"
},
{
"desc": "The health test thresholds for assignment locality. Specified as a percentage of total assignments.",
"display_name": "Assignment Locality Ratio Thresholds",
"name": "impala_assignment_locality_thresholds",
"value": "{\"critical\":\"5.0\",\"warning\":\"80.0\"}"
"desc": "The time period over which to compute the assignment locality ratio. Specified in minutes.",
"display_name": "Assignment Locality Monitoring Period",
"name": "impala_assignment_locality_window",
"value": "15"
},
{
"desc": "Configures the max memory of running queries for admission control when using a single pool. -1 or 0 indicates no limit. Ignored when Dynamic Resource Pools for Admission Control is enabled.",
"display_name": "Single Pool Mem Limit",
"name": "admission_control_single_pool_mem_limit",
"value": "-1"
},
{
"desc": "Use Sentry to enable role-based, fine-grained authorization. This configuration enables Sentry using policy files. To enable Sentry using Sentry service instead, add Sentry service as a dependency to Impala service. <strong>Sentry service provides concurrent and secure access to authorization policy metadata and is the recommended option for enabling Sentry. </strong> Sentry is supported only on Impala 1.1 or later deployments.",
@ -150,10 +186,10 @@
"value": "true"
},
{
"desc": "The time period over which to compute the assignment locality ratio. Specified in minutes.",
"display_name": "Assignment Locality Monitoring Period",
"name": "impala_assignment_locality_window",
"value": "15"
"desc": "For advanced use only, key-value pairs (one on each line) to be added (verbatim) to Impala Daemon command-line flags. Applies to all roles in this service. Key names should begin with a hyphen(-). <strong>For example</strong>: -log_filename=foo.log",
"display_name": "Impala Command Line Argument Advanced Configuration Snippet (Safety Valve)",
"name": "impala_cmd_args_safety_valve",
"value": null
},
{
"desc": "Used to generate a core dump to get more information about an Impala crash. Unless otherwise configured system wide using /proc/sys/kernel/core_pattern, the dump is generated in the 'current directory' of the Impala process (usually a subdirectory of the /var/run/cloudera-scm-agent/process directory). The core file can be very large.",
@ -168,15 +204,9 @@
"value": "false"
},
{
"desc": "The health test thresholds of the overall Impala Llama ApplicationMaster health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Impala Llama ApplicationMasters falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Impala Llama ApplicationMasters falls below the critical threshold.",
"display_name": "Healthy Impala Llama ApplicationMaster Monitoring Thresholds",
"name": "impala_llamas_healthy_thresholds",
"value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}"
},
{
"desc": "When computing the overall IMPALA health, consider Impala Catalog Server's health",
"display_name": "Impala Catalog Server Role Health Test",
"name": "impala_catalogserver_health_enabled",
"desc": "Enable HDFS short circuit read. This allows a client co-located with the DataNode to read HDFS file blocks directly. This gives a performance boost to distributed clients that are aware of locality.",
"display_name": "Enable HDFS Short Circuit Read",
"name": "dfs_client_read_shortcircuit",
"value": "true"
},
{
@ -197,6 +227,12 @@
"name": "service_triggers",
"value": "[]"
},
{
"desc": "If true, attempts to establish a TLS (Transport Layer Security) connection with the ldap server. Only supported in Impala 1.4 or CDH 5.1 or higher. Not required when using an LDAP URI with prefix: ldaps://.",
"display_name": "Enable LDAP TLS",
"name": "enable_ldap_tls",
"value": "false"
},
{
"desc": "Controls which queries admin users can see in the queries list view",
"display_name": "Admin Users Query List Visibility Settings",
@ -222,9 +258,15 @@
"value": "true"
},
{
"desc": "For advanced use only, key-value pairs (one on each line) to be added (verbatim) to Impala Daemon command-line flags. Applies to all roles in this service. Key names should begin with a hyphen(-). <strong>For example</strong>: -log_filename=foo.log",
"display_name": "Impala Command Line Argument Advanced Configuration Snippet (Safety Valve)",
"name": "impala_cmd_args_safety_valve",
"desc": "Name of YARN service to use for resource management integration between Impala and YARN. This service dependency and the existence of a Llama role is required for using said integration.",
"display_name": "YARN Service for Resource Management",
"name": "yarn_service",
"value": null
},
{
"desc": "Local path to the SSL server certificate file.",
"display_name": "SSL Server Certificate",
"name": "ssl_server_certificate",
"value": null
},
{
@ -258,9 +300,9 @@
"value": "false"
},
{
"desc": "Enable HDFS short circuit read. This allows a client co-located with the DataNode to read HDFS file blocks directly. This gives a performance boost to distributed clients that are aware of locality.",
"display_name": "Enable HDFS Short Circuit Read",
"name": "dfs_client_read_shortcircuit",
"desc": "When computing the overall IMPALA health, consider Impala Catalog Server's health",
"display_name": "Impala Catalog Server Role Health Test",
"name": "impala_catalogserver_health_enabled",
"value": "true"
},
{
@ -268,5 +310,11 @@
"display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)",
"name": "smon_derived_configs_safety_valve",
"value": null
},
{
"desc": "Configures the maximum number of concurrently running queries for admission control when using a single pool. -1 indicates no limit and 0 indicates all incoming requests will be rejected. Ignored when Dynamic Resource Pools for Admission Control is enabled.",
"display_name": "Single Pool Max Running Queries",
"name": "admission_control_single_pool_max_requests",
"value": "200"
}
]

0
sahara/plugins/cdh/resources/impala-statestore.json → sahara/plugins/cdh/v5_3_0/resources/impala-statestore.json
View File

6
sahara/plugins/cdh/resources/ks_indexer-hbase_indexer.json → sahara/plugins/cdh/v5_3_0/resources/ks_indexer-hbase_indexer.json
View File

@ -18,7 +18,7 @@
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that expose an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
@ -72,7 +72,7 @@
"value": "5"
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is hit, the oldest data will be deleted.",
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
@ -150,7 +150,7 @@
"value": "true"
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span>subdirectory of the role's log directory.",
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null

0
sahara/plugins/cdh/resources/ks_indexer-service.json → sahara/plugins/cdh/v5_3_0/resources/ks_indexer-service.json
View File

380
sahara/plugins/cdh/v5_3_0/resources/oozie-oozie_server.json Normal file
View File

@ -0,0 +1,380 @@
[
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.",
"display_name": "Log Directory Free Space Monitoring Percentage Thresholds",
"name": "log_directory_free_space_percentage_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"never\"}"
},
{
"desc": "Comma-separated list of Oozie plug-ins to be activated. If one plugin cannot be loaded, all the plugins are ignored.",
"display_name": "Oozie Server Plugins",
"name": "oozie_plugins_list",
"value": ""
},
{
"desc": "The period over which to compute the moving average of the callable queue size.",
"display_name": "Oozie Server Callable Queue Monitoring Period",
"name": "oozie_server_callable_queue_window",
"value": "5"
},
{
"desc": "Whether or not periodic stacks collection is enabled.",
"display_name": "Stacks Collection Enabled",
"name": "stacks_collection_enabled",
"value": "false"
},
{
"desc": "Enable SMTP authentication for Oozie email action",
"display_name": "Oozie Email Action SMTP Authentication Enabled",
"name": "oozie_email_smtp_auth",
"value": "false"
},
{
"desc": "The health test thresholds for the weighted average extra time the pause monitor spent paused. Specified as a percentage of elapsed wall clock time.",
"display_name": "Pause Duration Thresholds",
"name": "oozie_server_pause_duration_thresholds",
"value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}"
},
{
"desc": "When computing the overall Oozie Server health, consider the host's health.",
"display_name": "Oozie Server Host Health Test",
"name": "oozie_server_host_health_enabled",
"value": "true"
},
{
"desc": "Password for connecting to the database used by Oozie Server. Does not apply if you are using Derby as the database type.",
"display_name": "Oozie Server Database Password",
"name": "oozie_database_password",
"value": ""
},
{
"desc": "SMTP username for Oozie email action",
"display_name": "Oozie Email Action SMTP Authentication Username",
"name": "oozie_email_smtp_username",
"value": null
},
{
"desc": "Enables the health test that the Oozie Server's process state is consistent with the role configuration",
"display_name": "Oozie Server Process Health Test",
"name": "oozie_server_scm_health_enabled",
"value": "true"
},
{
"desc": "Number of threads used for executing callables",
"display_name": "Number Threads For Executing Callables",
"name": "oozie_service_callablequeueservice_threads",
"value": "10"
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
},
{
"desc": "SMTP password for Oozie email action",
"display_name": "Oozie Email Action SMTP Authentication Password",
"name": "oozie_email_smtp_password",
"value": null
},
{
"desc": "Maximum concurrency for a given callable type. Each command is a callable type: submit, start, run, etc. Each action type is a callable type: MapReduce, SSH, sub-workflow, etc. All commands that use action executors (action-start, action-end. etc.) use the action type as the callable type.",
"display_name": "Maximum concurrency for a given callable type",
"name": "oozie_service_callablequeueservice_callable_concurrency",
"value": "3"
},
{
"desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
"display_name": "Automatically Restart Process",
"name": "process_auto_restart",
"value": "false"
},
{
"desc": "<p>The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific role. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Role Triggers",
"name": "role_triggers",
"value": "[]"
},
{
"desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.",
"display_name": "Java Configuration Options for Oozie Server",
"name": "oozie_java_opts",
"value": ""
},
{
"desc": "The admin port Oozie server runs.",
"display_name": "Oozie Admin Port",
"name": "oozie_admin_port",
"value": "11001"
},
{
"desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Soft Limit",
"name": "rm_memory_soft_limit",
"value": "-1"
},
{
"desc": "The period to review when computing the moving average of extra time the pause monitor spent paused.",
"display_name": "Pause Duration Monitoring Period",
"name": "oozie_server_pause_duration_window",
"value": "5"
},
{
"desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup CPU Shares",
"name": "rm_cpu_shares",
"value": "1024"
},
{
"desc": "If true, enables the Oozie Server web console. ExtJS 2.2 zip archive must be extracted to /var/lib/oozie on the same host as the Oozie Server.",
"display_name": "Enable Oozie Server Web Console",
"name": "oozie_web_console",
"value": "false"
},
{
"desc": "The health test thresholds on the duration of the metrics request to the web server.",
"display_name": "Web Metric Collection Duration",
"name": "oozie_server_web_metric_collection_thresholds",
"value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}"
},
{
"desc": "Username for connecting to the database used by Oozie Server. Does not apply if you are using Derby as the database type.",
"display_name": "Oozie Server Database User",
"name": "oozie_database_user",
"value": "sa"
},
{
"desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.",
"display_name": "Web Metric Collection",
"name": "oozie_server_web_metric_collection_enabled",
"value": "true"
},
{
"desc": "The health test thresholds of the weighted average size of the Oozie Server callable queue over a recent period. See also Oozie Server Callable Queue Monitoring Period.",
"display_name": "Oozie Server Callable Queue Monitoring Threshold",
"name": "oozie_server_callable_queue_threshold",
"value": "{\"critical\":\"95.0\",\"warning\":\"80.0\"}"
},
{
"desc": "Directory where the Oozie Server will place its data. Only applicable when using Derby as the database type.",
"display_name": "Oozie Server Data Directory",
"name": "oozie_data_dir",
"value": "/var/lib/oozie/data"
},
{
"desc": "Password for the keystore.",
"display_name": "Oozie SSL Keystore Password",
"name": "oozie_https_keystore_password",
"value": null
},
{
"desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.",
"display_name": "Cgroup I/O Weight",
"name": "rm_io_weight",
"value": "500"
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
},
{
"desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.",
"display_name": "Cgroup Memory Hard Limit",
"name": "rm_memory_hard_limit",
"value": "-1"
},
{
"desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.",
"display_name": "File Descriptor Monitoring Thresholds",
"name": "oozie_server_fd_thresholds",
"value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>log4j.properties</strong> for this role only.",
"display_name": "Oozie Server Logging Advanced Configuration Snippet (Safety Valve)",
"name": "log4j_safety_valve",
"value": null
},
{
"desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.",
"display_name": "Log Directory Free Space Monitoring Absolute Thresholds",
"name": "log_directory_free_space_absolute_thresholds",
"value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}"
},
{
"desc": "Location of the keystore file on the local file system.",
"display_name": "Oozie SSL Keystore File",
"name": "oozie_https_keystore_file",
"value": "/var/lib/oozie/.keystore"
},
{
"desc": "The from address to be used for mailing all emails for Oozie email action",
"display_name": "Oozie Email Action From Address",
"name": "oozie_email_from_address",
"value": "oozie@localhost"
},
{
"desc": "Comma-separated list of SchemaService workflow extension schemas for additional action types.",
"display_name": "Oozie SchemaService Workflow Extension Schemas",
"name": "oozie_workflow_extension_schemas",
"value": "hive-action-0.2.xsd,sqoop-action-0.2.xsd,email-action-0.1.xsd,distcp-action-0.1.xsd,shell-action-0.1.xsd,ssh-action-0.1.xsd,distcp-action-0.2.xsd,hive-action-0.3.xsd,hive-action-0.4.xsd,hive-action-0.5.xsd,oozie-sla-0.1.xsd,oozie-sla-0.2.xsd,sqoop-action-0.3.xsd,sqoop-action-0.4.xsd,shell-action-0.2.xsd,shell-action-0.3.xsd,ssh-action-0.2.xsd,hive2-action-0.1.xsd"
},
{
"desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.",
"display_name": "Maximum Process File Descriptors",
"name": "rlimit_fds",
"value": null
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Health Alerts for this Role",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "The SMTP server port to use for Oozie email action",
"display_name": "Oozie Email Action SMTP Port",
"name": "oozie_email_smtp_prt",
"value": "25"
},
{
"desc": "Port of Oozie Server",
"display_name": "Oozie HTTP Port",
"name": "oozie_http_port",
"value": "11000"
},
{
"desc": "The maximum number of rolled log files to keep for Oozie Server logs. Typically used by log4j.",
"display_name": "Oozie Server Maximum Log File Backups",
"name": "max_log_backup_index",
"value": "720"
},
{
"desc": "Workflow Status metrics collection interval.",
"display_name": "Workflow Status Metrics Collection Interval",
"name": "oozie_job_metric_collection_interval",
"value": "1"
},
{
"desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.",
"display_name": "Dump Heap When Out of Memory",
"name": "oom_heap_dump_enabled",
"value": "false"
},
{
"desc": "The period to review when computing unexpected exits.",
"display_name": "Unexpected Exits Monitoring Period",
"name": "unexpected_exits_window",
"value": "5"
},
{
"desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.",
"display_name": "Unexpected Exits Thresholds",
"name": "unexpected_exits_thresholds",
"value": "{\"critical\":\"any\",\"warning\":\"never\"}"
},
{
"desc": "Name of the database used by Oozie Server.",
"display_name": "Oozie Server Database Name",
"name": "oozie_database_name",
"value": "oozie"
},
{
"desc": "Directory where Oozie Server will place its log files.",
"display_name": "Oozie Server Log Directory",
"name": "oozie_log_dir",
"value": "/var/log/oozie"
},
{
"desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.",
"display_name": "Java Heap Size of Oozie Server in Bytes",
"name": "oozie_java_heapsize",
"value": "1073741824"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "Whether to use the Codehale based metrics for instrumentation. Enabling this disables the 'instrumentation' REST end-point and enables the 'metrics' REST end-point (&lt;hostname:port&gt;/v2/admin/metrics).",
"display_name": "Enable The Metrics Instrumentation Service",
"name": "oozie_use_metric_instrumentation",
"value": "true"
},
{
"desc": "Comma-separated list of ActionService executor extension classes. Only action types with associated executors can be used in workflows.",
"display_name": "Oozie ActionService Executor Extension Classes",
"name": "oozie_executor_extension_classes",
"value": "org.apache.oozie.action.hadoop.HiveActionExecutor,org.apache.oozie.action.hadoop.SqoopActionExecutor,org.apache.oozie.action.email.EmailActionExecutor,org.apache.oozie.action.hadoop.ShellActionExecutor,org.apache.oozie.action.hadoop.DistcpActionExecutor,org.apache.oozie.action.hadoop.Hive2ActionExecutor"
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null
},
{
"desc": "Type of the database used by Oozie Server.",
"display_name": "Oozie Server Database Type",
"name": "oozie_database_type",
"value": "derby"
},
{
"desc": "Maximum callable queue size",
"display_name": "Maximum Callable Queue Size",
"name": "oozie_service_callablequeueservice_queue_size",
"value": "10000"
},
{
"desc": "Port of the Oozie Server while using SSL.",
"display_name": "Oozie HTTPS Port",
"name": "oozie_https_port",
"value": "11443"
},
{
"desc": "For advanced use only, a string to be inserted into <strong>oozie-site.xml</strong> for this role only.",
"display_name": "Oozie Server Advanced Configuration Snippet (Safety Valve) for oozie-site.xml",
"name": "oozie_config_safety_valve",
"value": null
},
{
"desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.",
"display_name": "Heap Dump Directory",
"name": "oom_heap_dump_dir",
"value": "/tmp"
},
{
"desc": "The frequency with which stacks will be collected.",
"display_name": "Stacks Collection Frequency",
"name": "stacks_collection_frequency",
"value": "5.0"
},
{
"desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.",
"display_name": "Kill When Out of Memory",
"name": "oom_sigkill_enabled",
"value": "true"
},
{
"desc": "The SMTP server host to use for Oozie email action",
"display_name": "Oozie Email Action SMTP Host",
"name": "oozie_email_smtp_host",
"value": "localhost"
},
{
"desc": "The minimum log level for Oozie Server logs",
"display_name": "Oozie Server Logging Threshold",
"name": "log_threshold",
"value": "INFO"
},
{
"desc": "Hostname of the database used by Oozie Server. If the port is non-default for your database type, use host:port notation. Does not apply if you are using Derby as the database type.",
"display_name": "Oozie Server Database Host",
"name": "oozie_database_host",
"value": "localhost"
}
]

128
sahara/plugins/cdh/v5_3_0/resources/oozie-service.json Normal file
View File

@ -0,0 +1,128 @@
[
{
"desc": "Namespace used by this Oozie service in ZooKeeper when High Availability is enabled.",
"display_name": "ZooKeeper Namespace",
"name": "oozie_zookeeper_namespace",
"value": "oozie"
},
{
"desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold",
"display_name": "Enable Service Level Health Alerts",
"name": "enable_alerts",
"value": "true"
},
{
"desc": "List of event listeners used by the Oozie service. Listeners needed for JMS or SLA integration are automatically emitted if they are enabled.",
"display_name": "Oozie Event Listeners",
"name": "oozie_event_listeners",
"value": ""
},
{
"desc": "Enable SSL for Oozie. <b>Note:</b>This is supported only from CDH 4.3 onwards.",
"display_name": "Use SSL",
"name": "oozie_use_ssl",
"value": "false"
},
{
"desc": "URL of the JMS Broker used by the Oozie service in JMS integration is enabled.",
"display_name": "JMS Broker",
"name": "oozie_jms_broker",
"value": "tcp://localhost:61616"
},
{
"desc": "Use ACLs on Znode while a secure ZooKeeper is used for Oozie High Availability. <b>Note:</b> This config is not emitted if ZooKeeper is not secure.",
"display_name": "Use ACLs on Znode",
"name": "oozie_zk_secure",
"value": "true"
},
{
"desc": "Whether to configure Oozie properties needed for JMS integration",
"display_name": "Enable JMS Integration",
"name": "oozie_use_jms",
"value": "false"
},
{
"desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.",
"display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)",
"name": "smon_derived_configs_safety_valve",
"value": null
},
{
"desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.",
"display_name": "Oozie Service Environment Advanced Configuration Snippet (Safety Valve)",
"name": "oozie_env_safety_valve",
"value": null
},
{
"desc": "Name of the ZooKeeper service that this Oozie service instance depends on",
"display_name": "ZooKeeper Service",
"name": "zookeeper_service",
"value": null
},
{
"desc": "Name of the Hive service that this Oozie service instance depends on. This is used to configure Oozie HCat integration.",
"display_name": "Hive Service",
"name": "hive_service",
"value": null
},
{
"desc": "A list of credential class mappings for CredentialsProvider.",
"display_name": "Oozie Credential Classes",
"name": "oozie_credential_classes",
"value": "hcat=org.apache.oozie.action.hadoop.HCatCredentials,hbase=org.apache.oozie.action.hadoop.HbaseCredentials,hive2=org.apache.oozie.action.hadoop.Hive2Credentials"
},
{
"desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.",
"display_name": "Enable Configuration Change Alerts",
"name": "enable_config_alerts",
"value": "false"
},
{
"desc": "Address of the load balancer used if Oozie HA is enabled. Should be specified in host:port format.",
"display_name": "Oozie Load Balancer",
"name": "oozie_load_balancer",
"value": null
},
{
"desc": "The group that this service's processes should run as.",
"display_name": "System Group",
"name": "process_groupname",
"value": "oozie"
},
{
"desc": "Whether to configure Oozie properties needed for SLA integration",
"display_name": "Enable SLA Integration",
"name": "oozie_use_sla",
"value": "false"
},
{
"desc": "Coordinator Job Lookup trigger command is scheduled at this interval (in seconds).",
"display_name": "Coordinator Job Lookup Interval",
"name": "oozie_service_coord_lookup_interval",
"value": "300"
},
{
"desc": "The user that this service's processes should run as.",
"display_name": "System User",
"name": "process_username",
"value": "oozie"
},
{
"desc": "<p>The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.</p><p>Each trigger has all of the following fields:</p><ul><li><code>triggerName</code> <strong>(mandatory)</strong> - the name of the trigger. This value must be unique for the specific service. </li><li><code>triggerExpression</code> <strong>(mandatory)</strong> - a tsquery expression representing the trigger. </li><li><code>streamThreshold</code> <strong>(optional)</strong> - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. </li><li><code>enabled</code> <strong> (optional)</strong> - by default set to 'true'. If set to 'false' the trigger will not be evaluated.</li></ul></p><p>For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:</p><p><pre>[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]</pre></p><p>Consult the trigger rules documentation for more details on how to write triggers using tsquery.</p><p>The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.</p>",
"display_name": "Service Triggers",
"name": "service_triggers",
"value": "[]"
},
{
"desc": "The health test thresholds of the overall Oozie Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Oozie Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Oozie Servers falls below the critical threshold.",
"display_name": "Healthy Oozie Server Monitoring Thresholds",
"name": "oozie_servers_healthy_thresholds",
"value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}"
},
{
"desc": "Service to run MapReduce jobs against",
"display_name": "MapReduce Service",
"name": "mapreduce_yarn_service",
"value": null
}
]

34
sahara/plugins/cdh/v5_3_0/resources/priority-one-confs.json Normal file
View File

@ -0,0 +1,34 @@
[
"dfs_block_size",
"dfs_umaskmode",
"dfs_webhdfs_enabled",
"dfs_permissions",
"io_compression_codecs",
"dfs_datanode_du_reserved",
"dfs_datanode_failed_volumes_tolerated",
"dfs_name_dir_restore",
"fs_trash_interval",
"dfs_safemode_min_datanodes",
"dfs_safemode_extension",
"dfs_access_time_precision",
"yarn_acl_enable",
"yarn_admin_acl",
"yarn_log_aggregation_enable",
"yarn_log_aggregation_retain_seconds",
"mapreduce_jobhistory_max_age_ms",
"mapreduce_jobhistory_cleaner_interval",
"yarn_nodemanager_container_manager_thread_count",
"yarn_nodemanager_delete_thread_count",
"yarn_nodemanager_heartbeat_interval_ms",
"yarn_nodemanager_localizer_cache_cleanup_interval_ms",
"yarn_nodemanager_localizer_client_thread_count",
"yarn_nodemanager_localizer_cache_target_size_mb",
"yarn_nodemanager_localizer_fetch_thread_count",
"yarn_nodemanager_log_retain_seconds",
"yarn_resourcemanager_client_thread_count",
"yarn_resourcemanager_scheduler_client_thread_count",
"yarn_resourcemanager_admin_client_thread_count",
"yarn_resourcemanager_amliveliness_monitor_interval_ms",
"yarn_am_liveness_monitor_expiry_interval_ms",
"yarn_resourcemanager_am_max_retries"
]

6
sahara/plugins/cdh/resources/sentry-sentry_server.json → sahara/plugins/cdh/v5_3_0/resources/sentry-sentry_server.json
View File

@ -18,7 +18,7 @@
"value": "false"
},
{
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that expose an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.",
"display_name": "Stacks Collection Method",
"name": "stacks_collection_method",
"value": "jstack"
@ -72,7 +72,7 @@
"value": "/var/log/sentry"
},
{
"desc": "The amount of stacks data that will be retained. After the retention limit is hit, the oldest data will be deleted.",
"desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.",
"display_name": "Stacks Collection Data Retention",
"name": "stacks_collection_data_retention",
"value": "104857600"
@ -144,7 +144,7 @@
"value": "false"
},
{
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span>subdirectory of the role's log directory.",
"desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a <span class='code'>stacks</span> subdirectory of the role's log directory.",
"display_name": "Stacks Collection Directory",
"name": "stacks_collection_directory",
"value": null

18
sahara/plugins/cdh/resources/sentry-service.json → sahara/plugins/cdh/v5_3_0/resources/sentry-service.json
View File

@ -48,16 +48,16 @@
"value": "localhost"
},
{
"desc": "Path to the directory where audit logs will be written. The directory will be created if it doesn't exist.",
"display_name": "Audit Log Directory",
"name": "audit_event_log_dir",
"value": "/var/log/sentry/audit"
"desc": "Name of Sentry Server database.",
"display_name": "Sentry Server Database Name",
"name": "sentry_server_database_name",
"value": "sentry"
},
{
"desc": "List of users allowed to connect to the Sentry Server. These are usually service users such as hive and impala, and the list does not usually need to include end-users.",
"display_name": "Allowed Connecting Users",
"name": "sentry_service_allow_connect",
"value": "hive,impala,hue"
"value": "hive,impala,hue,hdfs"
},
{
"desc": "The group that this service's processes should run as.",
@ -78,10 +78,10 @@
"value": "true"
},
{
"desc": "Name of Sentry Server database.",
"display_name": "Sentry Server Database Name",
"name": "sentry_server_database_name",
"value": "sentry"
"desc": "Path to the directory where audit logs will be written. The directory will be created if it doesn't exist.",
"display_name": "Audit Log Directory",
"name": "audit_event_log_dir",
"value": "/var/log/sentry/audit"
},
{
"desc": "Action to take when the audit event queue is full. Drop the event or shutdown the affected process.",

Some files were not shown because too many files have changed in this diff Show More