This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"IPC Server handler.*ClosedChannelException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"IPC Server Responder, call.*output error\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Daughter regiondir does not exist: .*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"File.*might still be open.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"File.*might still be open.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Moving table .+ state to enabled but was already enabled\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Received OPENED for region.*but region was in the state.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", - "display_name": "Health Check Startup Tolerance", - "name": "master_startup_tolerance", - "value": "5" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Number of pooled threads to handle region closing in the master.", - "display_name": "Region Closing Threads", - "name": "hbase_master_executor_closeregion_threads", - "value": "5" - }, - { - "desc": "When computing the overall Master health, consider the host's health.", - "display_name": "Master Host Health Test", - "name": "master_host_health_enabled", - "value": "true" - }, - { - "desc": "Number of pooled threads to handle the recovery of the region servers in the master.", - "display_name": "RegionServer Recovery Threads", - "name": "hbase_master_executor_serverops_threads", - "value": "5" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Directory where Master will place its log files.", - "display_name": "Master Log Directory", - "name": "hbase_master_log_dir", - "value": "/var/log/hbase" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "master_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Time period in seconds to reset long-running metrics (e.g. compactions). This is an HBase specific configuration.", - "display_name": "Extended Period", - "name": "hbase_metrics_extended_period", - "value": "3600" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Master Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that expose an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "master_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The port that the HBase Master binds to.", - "display_name": "HBase Master Port", - "name": "hbase_master_port", - "value": "60000" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "master_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "List of org.apache.hadoop.hbase.coprocessor.MasterObserver coprocessors that are loaded by default on the active HMaster process. For any implemented coprocessor methods, the listed classes will be called in order. After implementing your own MasterObserver, just put it in HBase's classpath and add the fully qualified class name here.", - "display_name": "HBase Coprocessor Master Classes", - "name": "hbase_coprocessor_master_classes", - "value": "" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of HBase Master in Bytes", - "name": "hbase_master_java_heapsize", - "value": "1073741824" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "master_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "When true, HBase Master will bind to 0.0.0.0. Only available with CDH 4.3 and later.", - "display_name": "HBase Master Bind to Wildcard Address", - "name": "hbase_master_bind_to_wildcard_address", - "value": "true" - }, - { - "desc": "Enables the health test that a client can connect to the HBase Master", - "display_name": "HBase Master Canary Health Test", - "name": "master_canary_health_enabled", - "value": "true" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is hit, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The port for the HBase Master web UI. Set to -1 to disable the HBase Master web UI.", - "display_name": "HBase Master Web UI Port", - "name": "hbase_master_info_port", - "value": "60010" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "master_gc_duration_window", - "value": "5" - }, - { - "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", - "display_name": "Master Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", - "name": "hbase_master_config_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for HBase Master", - "name": "hbase_master_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Master logs. Typically used by log4j.", - "display_name": "Master Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "Maximum time an HLog remains in the .oldlogdir directory until an HBase Master thread deletes it.", - "display_name": "Maximum Time to Keep HLogs", - "name": "hbase_master_logcleaner_ttl", - "value": "60000" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Number of RPC Server instances spun up on HBase Master.", - "display_name": "HBase Master Handler Count", - "name": "hbase_master_handler_count", - "value": "25" - }, - { - "desc": "The address for the HBase Master web UI", - "display_name": "HBase Master Web UI Address", - "name": "hbase_master_info_bindAddress", - "value": null - }, - { - "desc": "A comma-separated list of LogCleanerDelegate(s) that are used in LogsCleaner. WAL/HLog cleaner(s) are called in order, so put the log cleaner that prunes the most log files in the front. To implement your own LogCleanerDelegate, add it to HBase's classpath and add the fully-qualified class name here. You should always add the above default log cleaners in the list, unless you have a special reason not to.", - "display_name": "HBase Master Log Cleaner Plugins", - "name": "hbase_master_logcleaner_plugins", - "value": null - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stackssubdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "The maximum number of rolled log files to keep for Master logs. Typically used by log4j.", - "display_name": "Master Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The host name or IP address of the DNS name server which an HBase Master should use to determine the host name used for communication and display purposes.", - "display_name": "HBase Master DNS Name Server", - "name": "hbase_master_dns_nameserver", - "value": null - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "Number of pooled threads to handle region opening in the master.", - "display_name": "Region Opening Threads", - "name": "hbase_master_executor_openregion_threads", - "value": "5" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The name of the DNS network interface from which an HBase Master should report its IP address.", - "display_name": "HBase Master DNS Network Interface", - "name": "hbase_master_dns_interface", - "value": null - }, - { - "desc": "The minimum log level for Master logs", - "display_name": "Master Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Enables the health test that the Master's process state is consistent with the role configuration", - "display_name": "Master Process Health Test", - "name": "master_scm_health_enabled", - "value": "true" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/hbase-regionserver.json b/sahara/plugins/cdh/v5/resources/hbase-regionserver.json deleted file mode 100644 index b6a5f94d..00000000 --- a/sahara/plugins/cdh/v5/resources/hbase-regionserver.json +++ /dev/null @@ -1,542 +0,0 @@ -[ - { - "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", - "display_name": "RegionServer Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", - "name": "hbase_regionserver_config_safety_valve", - "value": null - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Maximum number of HStoreFiles to compact per minor compaction.", - "display_name": "Maximum Number of HStoreFiles Compaction", - "name": "hbase_hstore_compaction_max", - "value": null - }, - { - "desc": "The address for the HBase RegionServer web UI", - "display_name": "HBase RegionServer Web UI Address", - "name": "hbase_regionserver_info_bindAddress", - "value": null - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that expose an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "Number of threads to use while loading and unloading regions to or from a RegionServer. Can be used to increase the speed of decommissioning or rolling restart operations.", - "display_name": "Region Mover Threads", - "name": "hbase_regionserver_regionmover_thread_count", - "value": "1" - }, - { - "desc": "The health test thresholds of the average size of the HBase RegionServer flush queue over a recent period. See HBase RegionServer Flush Queue Monitoring Period.", - "display_name": "HBase RegionServer Flush Queue Monitoring Thresholds", - "name": "regionserver_flush_queue_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10.0\"}" - }, - { - "desc": "The amount of time to wait for the HBase Region Server to fully start up and connect to the HBase Master before enforcing the connectivity check.", - "display_name": "HBase Region Server Connectivity Tolerance at Startup", - "name": "regionserver_connectivity_tolerance", - "value": "180" - }, - { - "desc": "The hashing algorithm for use in HashFunction. Two values are supported: 'murmur' (for MurmurHash) and 'jenkins' (for JenkinsHash).", - "display_name": "HBase Hash Type", - "name": "hbase_hash_type", - "value": "murmur" - }, - { - "desc": "The port for the HBase RegionServer web UI. Set to -1 to disable RegionServer web UI.", - "display_name": "HBase RegionServer Web UI port", - "name": "hbase_regionserver_info_port", - "value": "60030" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "When memstores are being forced to flush to make room in memory, keep flushing until this amount is reached. If this amount is equal to 'hbase.regionserver.global.memstore.upperLimit', then minimum possible flushing will occur when updates are blocked due to memstore limiting.", - "display_name": "Low Watermark for Memstore Flush", - "name": "hbase_regionserver_global_memstore_lowerLimit", - "value": "0.38" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for HBase RegionServer", - "name": "hbase_regionserver_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "RegionServer Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds of the latency that the RegionServer sees for HDFS read operations", - "display_name": "HBase RegionServer HDFS Read Latency Thresholds", - "name": "regionserver_read_latency_thresholds", - "value": "{\"critical\":\"100.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds of the size used by the HStoreFile index. Specified as a percentage of the total heap size.", - "display_name": "Percentage of Heap Used by HStoreFile Index", - "name": "regionserver_store_file_idx_size_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10.0\"}" - }, - { - "desc": "Maximum number of Write-Ahead Log (WAL) files. This value multiplied by HDFS Block Size (dfs.blocksize) is the size of the WAL that will need to be replayed when a server crashes. This value is inversely proportional to the frequency of flushes to disk.", - "display_name": "Maximum number of Write-Ahead Log (WAL) files", - "name": "hbase_regionserver_maxlogs", - "value": "32" - }, - { - "desc": "List of coprocessors that are loaded by default on all tables. For any override coprocessor method, these classes will be called in order. After implementing your own coprocessor, just put it in HBase's classpath and add the fully qualified class name here. A coprocessor can also be loaded on demand by setting HTableDescriptor.", - "display_name": "HBase Coprocessor Region Classes", - "name": "hbase_coprocessor_region_classes", - "value": "" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "regionserver_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "Blocks writes if the size of the memstore increases to the value of 'hbase.hregion.block.memstore' multiplied by the value of 'hbase.hregion.flush.size' bytes. This setting is useful for preventing runaway memstore during spikes in update traffic. Without an upper-bound, memstore fills such that when it flushes, the resultant process of flushing files take a long time to compact or split, or worse, an \"out of memory\" error occurs.", - "display_name": "HBase Memstore Block Multiplier", - "name": "hbase_hregion_memstore_block_multiplier", - "value": "2" - }, - { - "desc": "Sync the HLog to HDFS after this interval, in milliseconds, if it has not accumulated the number of HLog Entries specified to trigger a sync.", - "display_name": "Synch Interval of HLog Entries", - "name": "hbase_regionserver_optionallogflushinterval", - "value": "1000" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \"IPC Server handler.*took.*appending an edit to hlog.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"ABORTING region server serverName.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"DFSOutputStream ResponseProcessor exception.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Error Recovery for block blk.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Failed init\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Problem renewing lease for DFSClient.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"remote error telling master we are up\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Session.*for server.*closing socket connection and attempting reconnect.\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Starting .*compaction on region (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_REGION_STARTED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^completed compaction on region (.+) after (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_REGION_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Starting compaction on (.+) in region (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_COLUMN_FAMILY_STARTED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"COLUMN_FAMILY\", \"group1\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^completed compaction: regionName=(.+), storeName=(.+), fileCount=(.+), fileSize=(.+), priority=(.+), time=(.+); duration=(.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_COLUMN_FAMILY_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"COLUMN_FAMILY\", \"group2\": \"FILE_COUNT\", \"group3\": \"FILE_SIZE\", \"group4\": \"PRIORITY\", \"group5\": \"REQUEST_TIME_NANOS\", \"group6\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Completed compaction: Request = regionName=(.+), storeName=(.+), fileCount=(.+), fileSize=(.+), priority=(.+), time=(.+); duration=(.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_COLUMN_FAMILY_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"COLUMN_FAMILY\", \"group2\": \"FILE_COUNT\", \"group3\": \"FILE_SIZE\", \"group4\": \"PRIORITY\", \"group5\": \"REQUEST_TIME_NANOS\", \"group6\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^aborted compaction: regionName=(.+), storeName=(.+), fileCount=(.+), fileSize=(.+), priority=(.+), time=(.+); duration=(.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_COLUMN_FAMILY_ABORTED\", \"attribute:SEVERITY\": \"IMPORTANT\", \"group0\": \"REGION\", \"group1\": \"COLUMN_FAMILY\", \"group2\": \"FILE_COUNT\", \"group3\": \"FILE_SIZE\", \"group4\": \"PRIORITY\", \"group5\": \"REQUEST_TIME_NANOS\", \"group6\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Finished memstore flush of .+ for region (.+) in (.+), sequenceid=(.+), compaction requested=(.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_FLUSH_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"DURATION\", \"group2\": \"SEQUENCE_ID\", \"group3\": \"COMPACTION_REQUESTED\" },\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Flush of region (.+) due to global heap pressure$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_FLUSH_DUE_TO_HEAP_PRESSURE\", \"attribute:SEVERITY\": \"IMPORTANT\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"WARN\", \"content\": \"^Region (.+) has too many store files; delaying flush up to 90000ms$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_FLUSH_DELAYED_TOO_MANY_STORE_FILES\", \"attribute:SEVERITY\": \"CRITICAL\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Starting split of region (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_SPLIT_STARTED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Running rollback/cleanup of failed split of (.+);.+$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_SPLIT_ABORTED\", \"attribute:SEVERITY\": \"IMPORTANT\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Region split, hbase:meta updated, and report to master. Parent=(.+), new regions: (.+,.*,.+), (.+,.*,.+). Split took (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_SPLIT_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"DAUGHTER_REGIONS\", \"group2\": \"DAUGHTER_REGIONS\", \"group3\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Region split, META updated, and report to master. Parent=(.+), new regions: (.+,.*,.+), (.+,.*,.+). Split took (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_SPLIT_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"DAUGHTER_REGIONS\", \"group2\": \"DAUGHTER_REGIONS\", \"group3\": \"DURATION\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" - }, - { - "desc": "The host name or IP address of the DNS name server which a RegionServer should use to determine the host name used by the HBase Master for communication and display purposes.", - "display_name": "RegionServer DNS Nameserver", - "name": "hbase_regionserver_dns_nameserver", - "value": null - }, - { - "desc": "The health test thresholds of the total size of RegionServer's memstores. Specified as a percentage of the configured upper limit. See Maximum Size of All Memstores in RegionServer.", - "display_name": "HBase RegionServer Memstore Size Thresholds", - "name": "regionserver_memstore_size_thresholds", - "value": "{\"critical\":\"100.0\",\"warning\":\"95.0\"}" - }, - { - "desc": "Percentage of maximum heap (-Xmx setting) to allocate to block cache used by HFile/StoreFile. To disable, set this value to 0 .", - "display_name": "HFile Block Cache Size", - "name": "hfile_block_cache_size", - "value": "0.4" - }, - { - "desc": "The maximum size, in megabytes, per log file for RegionServer logs. Typically used by log4j.", - "display_name": "RegionServer Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Memstore will be flushed to disk if size of the memstore exceeds this value in number of bytes. This value is checked by a thread that runs the frequency specified by hbase.server.thread.wakefrequency.", - "display_name": "HBase Memstore Flush Size", - "name": "hbase_hregion_memstore_flush_size", - "value": "134217728" - }, - { - "desc": "The HLog file reader implementation.", - "display_name": "HLog Reader Implementation", - "name": "hbase_regionserver_hlog_reader_impl", - "value": null - }, - { - "desc": "If this number of HStoreFiles in any one HStore is exceeded, then a compaction is run to rewrite all HStoreFiles files as one HStoreFile. (One HStoreFile is written per flush of memstore.) You can delay compaction by specifying a larger number, but the compaction will take longer when it does run. During a compaction, updates cannot be flushed to disk. Long compactions require memory sufficient to carry the logging of all updates across the duration of the compaction. If too large, clients timeout during compaction.", - "display_name": "HStore Compaction Threshold", - "name": "hbase_hstore_compactionThreshold", - "value": "3" - }, - { - "desc": "The period over which to compute the moving average of the HDFS sync latency of the HBase RegionServer.", - "display_name": "HBase RegionServer HDFS Sync Latency Monitoring Period", - "name": "regionserver_sync_latency_window", - "value": "5" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is hit, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The maximum size byte array that should come from the MSLAB, in bytes.", - "display_name": "Maximum Byte Array from MSLAB Allocation Scheme", - "name": "hbase_hregion_memstore_mslab_max_allocation", - "value": "262144" - }, - { - "desc": "The health test thresholds for the latency of HDFS write operations that the RegionServer detects", - "display_name": "HBase RegionServer HDFS Sync Latency Thresholds", - "name": "regionserver_sync_latency_thresholds", - "value": "{\"critical\":\"5000.0\",\"warning\":\"500.0\"}" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Directory where RegionServer will place its log files.", - "display_name": "RegionServer Log Directory", - "name": "hbase_regionserver_log_dir", - "value": "/var/log/hbase" - }, - { - "desc": "The period of time that an HRegion will block updates after reaching the HStoreFile limit that is specified by 'hbase.hstore.blockingStoreFiles'. After this time has elapsed, the HRegion will stop blocking updates even if a compaction has not been completed.", - "display_name": "HStore Blocking Wait Time", - "name": "hbase_hstore_blockingWaitTime", - "value": "90000" - }, - { - "desc": "Timeout for Canary to perform its checks.", - "display_name": "Canary Timeout", - "name": "hbase_regionserver_canary_timeout", - "value": "15" - }, - { - "desc": "Duration between consecutive checks done by the Canary.", - "display_name": "Canary Interval", - "name": "hbase_regionserver_canary_interval", - "value": "6" - }, - { - "desc": "The lease period, in milliseconds, for the HBase RegionServer. Clients must report in within this period or else they are considered dead.", - "display_name": "HBase RegionServer Lease Period", - "name": "hbase_regionserver_lease_period", - "value": "60000" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "The minimum log level for RegionServer logs", - "display_name": "RegionServer Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "regionserver_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "The name of the DNS Network Interface from which a RegionServer should report its IP address.", - "display_name": "RegionServer DNS Network Interface", - "name": "hbase_regionserver_dns_interface", - "value": null - }, - { - "desc": "The number of reservation blocks that are used to prevent unstable RegionServers caused by an OOME.", - "display_name": "RegionServer Reservation Blocks", - "name": "hbase_regionserver_nbreservationblocks", - "value": "4" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "regionserver_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The size of the chunks allocated by MSLAB, in bytes.", - "display_name": "Chunk Size Allocated by MSLAB Allocation Scheme", - "name": "hbase_hregion_memstore_mslab_chunksize", - "value": "2097152" - }, - { - "desc": "If the memstores in a region are this size or larger when closing, run a pre-flush process to clear out memstores before putting up the region closed flag and taking the region offline. On close, a flush process is run under the close flag up to empty memory. During this time, the region is offline and no writes are taken. If the memstore content is large, the flush process could take a long time to complete. The pre-flush process cleans out the bulk of the memstore before putting up the close flag and taking the region offline, so that the flush process that runs under the close flag has little to do.", - "display_name": "HBase Memstore Pre-close Flush Size", - "name": "hbase_hregion_preclose_flush_size", - "value": "5242880" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The period over which to compute the moving average of the HDFS read latency of the HBase RegionServer.", - "display_name": "HBase RegionServer HDFS Read Latency Monitoring Period", - "name": "regionserver_read_latency_window", - "value": "5" - }, - { - "desc": "An interface that is assignable to HRegionInterface. Used in HBase Client for opening a proxy to a remote HBase RegionServer.", - "display_name": "HBase RegionServer Interface Class", - "name": "hbase_regionserver_class", - "value": null - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "When true, HBase RegionServer will bind to 0.0.0.0. Only available in CDH 4.3 and later.", - "display_name": "HBase RegionServer Bind to Wildcard Address", - "name": "hbase_regionserver_bind_to_wildcard_address", - "value": "true" - }, - { - "desc": "The time, in milliseconds, between 'major' compactions of all HStoreFiles in a region. To disable automated major compactions, set this value to 0.", - "display_name": "HBase HRegion Major Compaction", - "name": "hbase_hregion_majorcompaction", - "value": "604800000" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Enable MemStore-Local Allocation Buffer (MSLAB) Allocation Scheme. Note: This feature is experimental in CDH3.", - "display_name": "Enable MSLAB Allocation Scheme", - "name": "hbase_hregion_memstore_mslab_enabled", - "value": "true" - }, - { - "desc": "Enables the health test that the RegionServer is connected to the Master", - "display_name": "HBase RegionServer to Master Connectivity Test", - "name": "regionserver_master_connectivity_enabled", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": "32768" - }, - { - "desc": "Limit for the number of regions after which no more region splitting should take place. This is not a hard limit for the number of regions but acts as a guideline for the RegionServer to stop splitting after a certain limit.", - "display_name": "HBase Region Split Limit", - "name": "hbase_regionserver_regionSplitLimit", - "value": "2147483647" - }, - { - "desc": "Enables the health test that the RegionServer's process state is consistent with the role configuration", - "display_name": "RegionServer Process Health Test", - "name": "regionserver_scm_health_enabled", - "value": "true" - }, - { - "desc": "Interval, in milliseconds, between messages from the RegionServer to the HBase Master. Use a high value such as 3000 for clusters that have more than 10 hosts.", - "display_name": "HBase RegionServer Message Interval", - "name": "hbase_regionserver_msginterval", - "value": "3000" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The period over which to compute the moving average of the compaction queue size.", - "display_name": "HBase RegionServer Compaction Queue Monitoring Period", - "name": "regionserver_compaction_queue_window", - "value": "5" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "regionserver_gc_duration_window", - "value": "5" - }, - { - "desc": "Time period in seconds to reset long-running metrics (e.g. compactions). This is an HBase specific configuration.", - "display_name": "Extended Period", - "name": "hbase_metrics_extended_period", - "value": "3600" - }, - { - "desc": "Comma-separated list of codecs that the RegionServer requires to start. Use this setting to make sure that all RegionServers joining a cluster are installed with a particular set of codecs.", - "display_name": "RegionServer Codecs", - "name": "hbase_regionserver_codecs", - "value": "" - }, - { - "desc": "Maximum size of all memstores in a RegionServer before new updates are blocked and flushes are forced.", - "display_name": "Maximum Size of All Memstores in RegionServer", - "name": "hbase_regionserver_global_memstore_upperLimit", - "value": "0.4" - }, - { - "desc": "Number of handlers for processing priority requests in a RegionServer.", - "display_name": "HBase RegionServer Meta-Handler Count", - "name": "hbase_regionserver_metahandler_count", - "value": "10" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Period, in milliseconds, at which to roll the commit log.", - "display_name": "HBase RegionServer Log Roll Period", - "name": "hbase_regionserver_logroll_period", - "value": "3600000" - }, - { - "desc": "The period over which to compute the moving average of the flush queue size.", - "display_name": "HBase RegionServer Flush Queue Monitoring Period", - "name": "regionserver_flush_queue_window", - "value": "5" - }, - { - "desc": "Maximum HStoreFile size. If any one of a column families' HStoreFiles has grown to exceed this value, the hosting HRegion is split in two.", - "display_name": "HBase Maximum File Size", - "name": "hbase_hregion_max_filesize", - "value": "10737418240" - }, - { - "desc": "The health test thresholds of the weighted average size of the HBase RegionServer compaction queue over a recent period. See HBase RegionServer Compaction Queue Monitoring Period.", - "display_name": "HBase RegionServer Compaction Queue Monitoring Thresholds", - "name": "regionserver_compaction_queue_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10.0\"}" - }, - { - "desc": "If there are more than this number of HStoreFiles in any one HStore, then updates are blocked for this HRegion until a compaction is completed, or until the value specified for 'hbase.hstore.blockingWaitTime' has been exceeded.", - "display_name": "HStore Blocking Store Files", - "name": "hbase_hstore_blockingStoreFiles", - "value": "10" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "regionserver_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "The maximum number of rolled log files to keep for RegionServer logs. Typically used by log4j.", - "display_name": "RegionServer Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "The HLog file writer implementation.", - "display_name": "HLog Writer Implementation", - "name": "hbase_regionserver_hlog_writer_impl", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The port that an HBase RegionServer binds to.", - "display_name": "HBase RegionServer Port", - "name": "hbase_regionserver_port", - "value": "60020" - }, - { - "desc": "Number of RPC Server instances spun up on RegionServers.", - "display_name": "HBase RegionServer Handler Count", - "name": "hbase_regionserver_handler_count", - "value": "30" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stackssubdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of HBase RegionServer in Bytes", - "name": "hbase_regionserver_java_heapsize", - "value": "4294967296" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "When computing the overall RegionServer health, consider the host's health.", - "display_name": "RegionServer Host Health Test", - "name": "regionserver_host_health_enabled", - "value": "true" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/hbase-service.json b/sahara/plugins/cdh/v5/resources/hbase-service.json deleted file mode 100644 index 168d6546..00000000 --- a/sahara/plugins/cdh/v5/resources/hbase-service.json +++ /dev/null @@ -1,476 +0,0 @@ -[ - { - "desc": "Comma-delimited list of hosts where you want to allow the HBase user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that does not correspond to a host name, such as '_no_host'. Note: This property is used only if HBase REST/Thrift Server Authentication is enabled.", - "display_name": "HBase Proxy User Hosts", - "name": "hbase_proxy_user_hosts_list", - "value": "*" - }, - { - "desc": "The user that this service's processes should run as.", - "display_name": "System User", - "name": "process_username", - "value": "hbase" - }, - { - "desc": "The frequency in which the log4j event publication appender will retry sending undelivered log events to the Event server, in seconds", - "display_name": "Log Event Retry Frequency", - "name": "log_event_retry_frequency", - "value": "30" - }, - { - "desc": "A general client pause time value. Used mostly as a time period to wait before retrying operations such as a failed get or region lookup.", - "display_name": "HBase Client Pause", - "name": "hbase_client_pause", - "value": "100" - }, - { - "desc": "Comma-delimited list of groups that you want to allow the HBase user to impersonate. The default '*' allows all groups. To disable entirely, use a string that does not correspond to a group name, such as '_no_group_'. Note: This property is used only if HBase REST/Thrift Server Authentication is enabled.", - "display_name": "HBase Proxy User Groups", - "name": "hbase_proxy_user_groups_list", - "value": "*" - }, - { - "desc": "Number of rows to fetch when calling next on a scanner if it is not served from memory. Higher caching values enable faster scanners but require more memory and some calls of next may take longer when the cache is empty.", - "display_name": "HBase Client Scanner Caching", - "name": "hbase_client_scanner_caching", - "value": "100" - }, - { - "desc": "\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n
\n\n\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n
\n\nEvent trackers are defined in a JSON object like the following:
\n\n\n{\n \"timeToLive\" : [integer],\n \"fields\" : [\n {\n \"type\" : [string],\n \"name\" : [string]\n }\n ]\n}\n\n\n\nWhere:\n
\n\n\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n
\n\n\nThe following is the list of fields that can used to compare HBase events:\n
\nEvent filters are defined in a JSON object like the following:
\n\n\n{\n \"defaultAction\" : (\"accept\", \"discard\"),\n \"rules\" : [\n {\n \"action\" : (\"accept\", \"discard\"),\n \"fields\" : [\n {\n \"name\" : \"fieldName\",\n \"match\" : \"regex\"\n }\n ]\n }\n ]\n}\n\n\n\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n
\n\n\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n
\n\n\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n
\n\n\nThe following is the list of fields that can be filtered for HBase events:\n
\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "An alert is published if the HBase region health canary runs slowly.", - "display_name": "HBase Region Health Canary Slow Run Alert Enabled", - "name": "hbase_region_health_canary_slow_run_alert_enabled", - "value": "true" - }, - { - "desc": "Set to true to cause the hosting server (Master or RegionServer) to abort if a coprocessor throws a Throwable object that is not IOException or a subclass of IOException. Setting it to true might be useful in development environments where one wants to terminate the server as soon as possible to simplify coprocessor failure analysis.", - "display_name": "HBase Coprocessor Abort on Error", - "name": "hbase_coprocessor_abort_on_error", - "value": "false" - }, - { - "desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.", - "display_name": "Enable Log Event Capture", - "name": "catch_events", - "value": "true" - }, - { - "desc": "Allow HBase tables to be replicated.", - "display_name": "Enable Replication", - "name": "hbase_enable_replication", - "value": "false" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Path to the keystore file containing the server certificate and private key used for encrypted web UIs", - "display_name": "SSL Server Keystore File Location", - "name": "ssl_server_keystore_location", - "value": null - }, - { - "desc": "Enable HBase authorization", - "display_name": "HBase Secure Authorization", - "name": "hbase_security_authorization", - "value": "false" - }, - { - "desc": "Name of the scheduler pool to use for MR jobs created during export/import of remote snapshots in AWS S3.", - "display_name": "Scheduler pool for remote snapshots in AWS S3.", - "name": "hbase_snapshot_s3_scheduler_pool", - "value": null - }, - { - "desc": "Period of time, in milliseconds, to pause between searches for work. Used as a sleep interval by service threads such as a META scanner and log roller.", - "display_name": "HBase Server Thread Wake Frequency", - "name": "hbase_server_thread_wakefrequency", - "value": "10000" - }, - { - "desc": "An alert is published if the HBase region health canary detects at least this percentage of total regions are unhealthy. This threshold is used if the explicit count is not set via the hbase_canary_alert_unhealthy_region_count_threshold config.", - "display_name": "HBase Canary Unhealthy Region Percentage Alert Threshold", - "name": "hbase_canary_alert_unhealthy_region_percent_threshold", - "value": "0.1" - }, - { - "desc": "The health test thresholds of the overall RegionServer health. The check returns \"Concerning\" health if the percentage of \"Healthy\" RegionServers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" RegionServers falls below the critical threshold.", - "display_name": "Healthy RegionServer Monitoring Thresholds", - "name": "hbase_regionservers_healthy_thresholds", - "value": "{\"critical\":\"90.0\",\"warning\":\"95.0\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for navigator.client.properties.", - "display_name": "HBASE Client Advanced Configuration Snippet (Safety Valve) for navigator.client.properties", - "name": "navigator_client_config_safety_valve", - "value": null - }, - { - "desc": "Duration to wait before starting up a 'hedged' read.", - "display_name": "HDFS Hedged Read Delay Threshold", - "name": "hbase_server_dfs_client_hedged_read_threshold_millis", - "value": "500" - }, - { - "desc": "Enables the HBase Hbck Poller so that Hbck reports will be available. Enabling the Hbck poller will increase the amount of memory used by the Service Monitor. Consider increasing the Service Monitor Java heap size by an additional 3KB per region. For example, for a cluster with 10,000 regions, increase the JVM heap size by approximately 30MB.", - "display_name": "HBase Hbck Poller", - "name": "hbase_hbck_poller_enabled", - "value": "false" - }, - { - "desc": "List of users or groups, who are allowed full privileges, regardless of stored ACLs, across the cluster. Only used when HBase security is enabled.", - "display_name": "HBase Superusers", - "name": "hbase_superuser", - "value": "" - }, - { - "desc": "Maximum number of hlog entries to replicate in one go. If this is large, and a consumer takes a while to process the events, the HBase RPC call will time out.", - "display_name": "Replication Batch Size", - "name": "hbase_replication_source_nb_capacity", - "value": "1000" - }, - { - "desc": "An alert is published if the HBase region health canary detects at least this many unhealthy regions. This setting takes precedence over the hbase_canary_alert_unhealthy_region_percent_threshold config.", - "display_name": "HBase Canary Unhealthy Region Count Alert Threshold", - "name": "hbase_canary_alert_unhealthy_region_count_threshold", - "value": null - }, - { - "desc": "The number of times to retry connections to ZooKeeper. Used for reading and writing root region location. Used together with ${zookeeper.pause} in an exponential backoff fashion when making queries to ZooKeeper.", - "display_name": "ZooKeeper Connection Retries", - "name": "zookeeper_retries", - "value": null - }, - { - "desc": "ZooKeeper session timeout in milliseconds. HBase passes this to the ZooKeeper quorum as the suggested maximum time for a session. See http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions The client sends a requested timeout, the server responds with the timeout that it can give the client.", - "display_name": "ZooKeeper Session Timeout", - "name": "zookeeper_session_timeout", - "value": "60000" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "Write buffer size in bytes. A larger buffer requires more memory on both the client and the server because the server instantiates the passed write buffer to process it but reduces the number of remote procedure calls (RPC). To estimate the amount of server memory used, multiply the value of 'hbase.client.write.buffer' by the value of 'hbase.regionserver.handler.count'.", - "display_name": "HBase Client Write Buffer", - "name": "hbase_client_write_buffer", - "value": "2097152" - }, - { - "desc": "The user the management services will impersonate when connecting to HBase. Defaults to 'hbase', a superuser.", - "display_name": "HBase User to Impersonate", - "name": "hbase_user_to_impersonate", - "value": "hbase" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/hdfs-balancer.json b/sahara/plugins/cdh/v5/resources/hdfs-balancer.json deleted file mode 100644 index f2f777d7..00000000 --- a/sahara/plugins/cdh/v5/resources/hdfs-balancer.json +++ /dev/null @@ -1,44 +0,0 @@ -[ - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Balancer", - "name": "balancer_java_opts", - "value": "" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "Balancer Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "balancer_config_safety_valve", - "value": null - }, - { - "desc": "The policy that should be used to rebalance HDFS storage. The default DataNode policy balances the storage at the DataNode level. This is similar to the balancing policy from prior releases. The BlockPool policy balances the storage at the block pool level as well as at the Datanode level. The BlockPool policy is relevant only to a Federated HDFS service.", - "display_name": "Rebalancing Policy", - "name": "rebalancing_policy", - "value": "DataNode" - }, - { - "desc": "The percentage deviation from average utilization, after which a node will be rebalanced. (for example, '10.0' for 10%)", - "display_name": "Rebalancing Threshold", - "name": "rebalancer_threshold", - "value": "10.0" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Maximum size for the Java Process heap memory. Passed to Java -Xmx. Measured in bytes.", - "display_name": "Java Heap Size of Balancer in Bytes", - "name": "balancer_java_heapsize", - "value": "1073741824" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/hdfs-datanode.json b/sahara/plugins/cdh/v5/resources/hdfs-datanode.json deleted file mode 100644 index 46c9fb2c..00000000 --- a/sahara/plugins/cdh/v5/resources/hdfs-datanode.json +++ /dev/null @@ -1,380 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 5, \"content\":\"Datanode registration failed\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Got a command from standby NN - ignoring command:.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "Specifies the maximum number of threads to use for transferring data in and out of the DataNode.", - "display_name": "Maximum Number of Transfer Threads", - "name": "dfs_datanode_max_xcievers", - "value": "4096" - }, - { - "desc": "Comma-separated list of DataNode plug-ins to be activated. If one plug-in cannot be loaded, all the plug-ins are ignored.", - "display_name": "DateNode Plugins", - "name": "dfs_datanode_plugins_list", - "value": "" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "In some workloads, the data read from HDFS is known to be significantly large enough that it is unlikely to be useful to cache it in the operating system buffer cache. In this case, the DataNode may be configured to automatically purge all data from the buffer cache after it is delivered to the client. This may improve performance for some workloads by freeing buffer cache spare usage for more cacheable data. This behavior will always be disabled for workloads that read only short sections of a block (e.g HBase random-IO workloads). This property is supported in CDH3u3 or later deployments.", - "display_name": "Enable purging cache after reads", - "name": "dfs_datanode_drop_cache_behind_reads", - "value": "false" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Comma-delimited list of directories on the local file system where the DataNode stores HDFS block data. Typical values are /data/N/dfs/dn for N = 1, 2, 3... These directories should be mounted using the noatime option and the disks should be configured using JBOD. RAID is not recommended.", - "display_name": "DataNode Data Directory", - "name": "dfs_data_dir_list", - "value": null - }, - { - "desc": "The number of volumes that are allowed to fail before a DataNode stops offering service. By default, any volume failure will cause a DataNode to shutdown.", - "display_name": "DataNode Failed Volumes Tolerated", - "name": "dfs_datanode_failed_volumes_tolerated", - "value": "0" - }, - { - "desc": "In some workloads, the data written to HDFS is known to be significantly large enough that it is unlikely to be useful to cache it in the operating system buffer cache. In this case, the DataNode may be configured to automatically purge all data from the buffer cache after it is written to disk. This may improve performance for some workloads by freeing buffer cache spare usage for more cacheable data. This property is supported in CDH3u3 or later deployments.", - "display_name": "Enable purging cache after writes", - "name": "dfs_datanode_drop_cache_behind_writes", - "value": "false" - }, - { - "desc": "If enabled, the DataNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind DataNode to Wildcard Address", - "name": "dfs_datanode_bind_wildcard", - "value": "false" - }, - { - "desc": "The number of server threads for the DataNode.", - "display_name": "Handler Count", - "name": "dfs_datanode_handler_count", - "value": "3" - }, - { - "desc": "When computing the overall DataNode health, consider the host's health.", - "display_name": "DataNode Host Health Test", - "name": "datanode_host_health_enabled", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "The maximum number of rolled log files to keep for DataNode logs. Typically used by log4j.", - "display_name": "DataNode Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:red\",\n \"streamThreshold\": 0}, \"enabled\": \"true\"]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Timeout in seconds for the Hue Thrift server running on each DataNode", - "display_name": "Hue Thrift Server Timeout", - "name": "dfs_thrift_timeout", - "value": "60" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "datanode_gc_duration_window", - "value": "5" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Whether DataNodes should use DataNode hostnames when connecting to DataNodes for data transfer. This property is supported in CDH3u4 or later deployments.", - "display_name": "Use DataNode Hostname", - "name": "dfs_datanode_use_datanode_hostname", - "value": "false" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for DataNode", - "name": "datanode_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "datanode_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "datanode_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Enables the health test that the DataNode's process state is consistent with the role configuration", - "display_name": "DataNode Process Health Test", - "name": "datanode_scm_health_enabled", - "value": "true" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "DataNode Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "DataNode Policy for picking which volume should get a new block. The Available Space policy is only available starting with CDH 4.3.", - "display_name": "DataNode Volume Choosing Policy", - "name": "dfs_datanode_volume_choosing_policy", - "value": "org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Port for the DataNode HTTP web UI. Combined with the DataNode's hostname to build its HTTP address.", - "display_name": "DataNode HTTP Web UI Port", - "name": "dfs_datanode_http_port", - "value": "50075" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "Directory where DataNode will place its log files.", - "display_name": "DataNode Log Directory", - "name": "datanode_log_dir", - "value": "/var/log/hadoop-hdfs" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "DataNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "datanode_config_safety_valve", - "value": null - }, - { - "desc": "While reading block files, the DataNode can use the posix_fadvise system call to explicitly page data into the operating system buffer cache ahead of the current reader's position. This can improve performance especially when disks are highly contended. This configuration specifies the number of bytes ahead of the current read position which the DataNode will attempt to read ahead. A value of 0 disables this feature. This property is supported in CDH3u3 or later deployments.", - "display_name": "Number of read ahead bytes", - "name": "dfs_datanode_readahead_bytes", - "value": "4194304" - }, - { - "desc": "The maximum size, in megabytes, per log file for DataNode logs. Typically used by log4j.", - "display_name": "DataNode Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The maximum amount of memory a DataNode may use to cache data blocks in memory. Setting it to zero will disable caching.", - "display_name": "Maximum Memory Used for Caching", - "name": "dfs_datanode_max_locked_memory", - "value": "4294967296" - }, - { - "desc": "Only used when the DataNode Volume Choosing Policy is set to Available Space. Controls how much DataNode volumes are allowed to differ in terms of bytes of free disk space before they are considered imbalanced. If the free space of all the volumes are within this range of each other, the volumes will be considered balanced and block assignments will be done on a pure round robin basis.", - "display_name": "Available Space Policy Balanced Threshold", - "name": "dfs_datanode_available_space_balanced_threshold", - "value": "10737418240" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds of free space in a DataNode. Specified as a percentage of the capacity on the DataNode.", - "display_name": "DataNode Free Space Monitoring Thresholds", - "name": "datanode_free_space_thresholds", - "value": "{\"critical\":\"10.0\",\"warning\":\"20.0\"}" - }, - { - "desc": "Port for the various DataNode Protocols. Combined with the DataNode's hostname to build its IPC port address.", - "display_name": "DataNode Protocol Port", - "name": "dfs_datanode_ipc_port", - "value": "50020" - }, - { - "desc": "Port for DataNode's XCeiver Protocol. Combined with the DataNode's hostname to build its address.", - "display_name": "DataNode Transceiver Port", - "name": "dfs_datanode_port", - "value": "50010" - }, - { - "desc": "Minimum number of running threads for the Hue Thrift server running on each DataNode", - "display_name": "Hue Thrift Server Min Threadcount", - "name": "dfs_thrift_threads_min", - "value": "10" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "datanode_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "If this configuration is enabled, the DataNode will instruct the operating system to enqueue all written data to the disk immediately after it is written. This differs from the usual OS policy which may wait for up to 30 seconds before triggering writeback. This may improve performance for some workloads by smoothing the IO profile for data written to disk. This property is supported in CDH3u3 or later deployments.", - "display_name": "Enable immediate enqueuing of data to disk after writes", - "name": "dfs_datanode_sync_behind_writes", - "value": "false" - }, - { - "desc": "The health test thresholds of the number of blocks on a DataNode", - "display_name": "DataNode Block Count Thresholds", - "name": "datanode_block_count_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"200000.0\"}" - }, - { - "desc": "Permissions for the directories on the local file system where the DataNode stores its blocks. The permissions must be octal. 755 and 700 are typical values.", - "display_name": "DataNode Data Directory Permissions", - "name": "dfs_datanode_data_dir_perm", - "value": "700" - }, - { - "desc": "Reserved space in bytes per volume for non Distributed File System (DFS) use.", - "display_name": "Reserved Space for Non DFS Use", - "name": "dfs_datanode_du_reserved", - "value": "10737418240" - }, - { - "desc": "The amount of time to wait for the DataNode to fully start up and connect to the NameNode before enforcing the connectivity check.", - "display_name": "DataNode Connectivity Tolerance at Startup", - "name": "datanode_connectivity_tolerance", - "value": "180" - }, - { - "desc": "The base port where the secure DataNode web UI listens. Combined with the DataNode's hostname to build its secure web UI address.", - "display_name": "Secure DataNode Web UI Port (SSL)", - "name": "dfs_datanode_https_port", - "value": "50475" - }, - { - "desc": "Maximum amount of bandwidth that each DataNode can use for balancing. Specified in bytes per second.", - "display_name": "DataNode Balancing Bandwidth", - "name": "dfs_balance_bandwidthPerSec", - "value": "10485760" - }, - { - "desc": "Maximum number of running threads for the Hue Thrift server running on each DataNode", - "display_name": "Hue Thrift Server Max Threadcount", - "name": "dfs_thrift_threads_max", - "value": "20" - }, - { - "desc": "Only used when the DataNode Volume Choosing Policy is set to Available Space. Controls what percentage of new block allocations will be sent to volumes with more available disk space than others. This setting should be in the range 0.0 - 1.0, though in practice 0.5 - 1.0, since there should be no reason to prefer that volumes with less available disk space receive more block allocations.", - "display_name": "Available Space Policy Balanced Preference", - "name": "dfs_datanode_available_space_balanced_preference", - "value": "0.75" - }, - { - "desc": "The health test thresholds of failed volumes in a DataNode.", - "display_name": "DataNode Volume Failures Thresholds", - "name": "datanode_volume_failures_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "The minimum log level for DataNode logs", - "display_name": "DataNode Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "datanode_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "Maximum size for the Java Process heap memory. Passed to Java -Xmx. Measured in bytes.", - "display_name": "Java Heap Size of DataNode in Bytes", - "name": "datanode_java_heapsize", - "value": "1073741824" - }, - { - "desc": "Enables the health test that verifies the DataNode is connected to the NameNode", - "display_name": "DataNode Connectivity Health Test", - "name": "datanode_connectivity_health_enabled", - "value": "true" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/hdfs-failover.json b/sahara/plugins/cdh/v5/resources/hdfs-failover.json deleted file mode 100644 index 030cc45e..00000000 --- a/sahara/plugins/cdh/v5/resources/hdfs-failover.json +++ /dev/null @@ -1,170 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Directory where Failover Controller will place its log files.", - "display_name": "Failover Controller Log Directory", - "name": "failover_controller_log_dir", - "value": "/var/log/hadoop-hdfs" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "Failover Controller Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "fc_config_safety_valve", - "value": null - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for Failover Controller logs. Typically used by log4j.", - "display_name": "Failover Controller Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:red\",\n \"streamThreshold\": 0}, \"enabled\": \"true\"]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Maximum size for the Java Process heap memory. Passed to Java -Xmx. Measured in bytes.", - "display_name": "Java Heap Size of Failover Controller in Bytes", - "name": "failover_controller_java_heapsize", - "value": "268435456" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Failover Controller Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When computing the overall Failover Controller health, consider the host's health.", - "display_name": "Failover Controller Host Health Test", - "name": "failovercontroller_host_health_enabled", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Failover Controller logs. Typically used by log4j.", - "display_name": "Failover Controller Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "failovercontroller_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Enables the health test that the Failover Controller's process state is consistent with the role configuration", - "display_name": "Failover Controller Process Health Test", - "name": "failovercontroller_scm_health_enabled", - "value": "true" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Failover Controller", - "name": "failover_controller_java_opts", - "value": "" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The minimum log level for Failover Controller logs", - "display_name": "Failover Controller Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/hdfs-gateway.json b/sahara/plugins/cdh/v5/resources/hdfs-gateway.json deleted file mode 100644 index 279015f3..00000000 --- a/sahara/plugins/cdh/v5/resources/hdfs-gateway.json +++ /dev/null @@ -1,50 +0,0 @@ -[ - { - "desc": "For advanced use only, a string to be inserted into the client configuration for hdfs-site.xml.", - "display_name": "HDFS Client Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "hdfs_client_config_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the client configuration for hadoop-env.sh.", - "display_name": "HDFS Client Environment Advanced Configuration Snippet for hadoop-env.sh (Safety Valve)", - "name": "hdfs_client_env_safety_valve", - "value": null - }, - { - "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", - "display_name": "Alternatives Priority", - "name": "client_config_priority", - "value": "90" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Maximum size for the Java process heap memory. Passed to Java -Xmx. Measured in bytes.", - "display_name": "Client Java Heap Size in Bytes", - "name": "hdfs_client_java_heapsize", - "value": "268435456" - }, - { - "desc": "Enable HDFS short circuit read. This allows a client co-located with the DataNode to read HDFS file blocks directly. This gives a performance boost to distributed clients that are aware of locality.", - "display_name": "Enable HDFS Short Circuit Read", - "name": "dfs_client_read_shortcircuit", - "value": "false" - }, - { - "desc": "These are Java command line arguments. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Client Java Configuration Options", - "name": "hbase_client_java_opts", - "value": "-Djava.net.preferIPv4Stack=true" - }, - { - "desc": "Move deleted files to the trash so that they can be recovered if necessary. This client side configuration takes effect only if the HDFS service-wide trash is disabled (NameNode Filesystem Trash Interval set to 0) and is ignored otherwise. The trash is not automatically emptied when enabled with this configuration.", - "display_name": "Use Trash", - "name": "dfs_client_use_trash", - "value": "false" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/hdfs-httpfs.json b/sahara/plugins/cdh/v5/resources/hdfs-httpfs.json deleted file mode 100644 index 17d1a291..00000000 --- a/sahara/plugins/cdh/v5/resources/hdfs-httpfs.json +++ /dev/null @@ -1,194 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "httpfs_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The user that the HttpFS server process should run as.", - "display_name": "System User", - "name": "httpfs_process_username", - "value": "httpfs" - }, - { - "desc": "The group that the HttpFS server process should run as.", - "display_name": "System Group", - "name": "httpfs_process_groupname", - "value": "httpfs" - }, - { - "desc": "When computing the overall HttpFS health, consider the host's health.", - "display_name": "HttpFS Host Health Test", - "name": "httpfs_host_health_enabled", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for HttpFS logs. Typically used by log4j.", - "display_name": "HttpFS Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:red\",\n \"streamThreshold\": 0}, \"enabled\": \"true\"]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Enables the health test that the HttpFS's process state is consistent with the role configuration", - "display_name": "HttpFS Process Health Test", - "name": "httpfs_scm_health_enabled", - "value": "true" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Directory where HttpFS will place its log files.", - "display_name": "HttpFS Log Directory", - "name": "httpfs_log_dir", - "value": "/var/log/hadoop-httpfs" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "HttpFS Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "For advanced use only, a string to be inserted into httpfs-site.xml for this role only.", - "display_name": "HttpFS Advanced Configuration Snippet (Safety Valve) for httpfs-site.xml", - "name": "httpfs_config_safety_valve", - "value": null - }, - { - "desc": "The maximum size, in megabytes, per log file for HttpFS logs. Typically used by log4j.", - "display_name": "HttpFS Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for HttpFS", - "name": "httpfs_java_opts", - "value": "" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The secret to use for signing client authentication tokens.", - "display_name": "Signature Secret", - "name": "hdfs_httpfs_signature_secret", - "value": "hadoop httpfs secret" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Maximum size for the Java Process heap memory. Passed to Java -Xmx. Measured in bytes.", - "display_name": "Java Heap Size of HttpFS in Bytes", - "name": "httpfs_java_heapsize", - "value": "268435456" - }, - { - "desc": "The port for the administration interface.", - "display_name": "Administration Port", - "name": "hdfs_httpfs_admin_port", - "value": "14001" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "The HTTP port where the REST interface to HDFS is available.", - "display_name": "HTTP Port", - "name": "hdfs_httpfs_http_port", - "value": "14000" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The minimum log level for HttpFS logs", - "display_name": "HttpFS Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/hdfs-journalnode.json b/sahara/plugins/cdh/v5/resources/hdfs-journalnode.json deleted file mode 100644 index 3e2b46ee..00000000 --- a/sahara/plugins/cdh/v5/resources/hdfs-journalnode.json +++ /dev/null @@ -1,242 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "journalnode_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Maximum size for the Java Process heap memory. Passed to Java -Xmx. Measured in bytes.", - "display_name": "Java Heap Size of JournalNode in Bytes", - "name": "journalNode_java_heapsize", - "value": "268435456" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "journalnode_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "Enables the health test that the JournalNode's process state is consistent with the role configuration", - "display_name": "JournalNode Process Health Test", - "name": "journalnode_scm_health_enabled", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "The maximum number of rolled log files to keep for JournalNode logs. Typically used by log4j.", - "display_name": "JournalNode Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The health check thresholds for monitoring of free space on the filesystem that contains the JournalNode's edits directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Edits Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Edits Directory Free Space Monitoring Percentage Thresholds", - "name": "journalnode_edits_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Directory on the local file system where the NameNode's edits are written.", - "display_name": "JournalNode Edits Directory", - "name": "dfs_journalnode_edits_dir", - "value": null - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "journalnode_gc_duration_window", - "value": "5" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "JournalNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "jn_config_safety_valve", - "value": null - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "When computing the overall JournalNode health, consider the host's health.", - "display_name": "JournalNode Host Health Test", - "name": "journalnode_host_health_enabled", - "value": "true" - }, - { - "desc": "The amount of time at JournalNode startup allowed for the active NameNode to get in sync with the JournalNode.", - "display_name": "Active NameNode Sync Status Startup Tolerance", - "name": "journalnode_sync_status_startup_tolerance", - "value": "180" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "journalnode_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "JournalNode Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Enables the health check that verifies the active NameNode's sync status to the JournalNode", - "display_name": "Active NameNode Sync Status Health Check", - "name": "journalnode_sync_status_enabled", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "journalnode_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "Port for the JournalNode's RPC. Combined with the JournalNode's hostname to build its RPC address.", - "display_name": "JournalNode RPC Port", - "name": "dfs_journalnode_rpc_port", - "value": "8485" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for JournalNode logs. Typically used by log4j.", - "display_name": "JournalNode Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "If enabled, the JournalNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind JournalNode to Wildcard Address", - "name": "journalnode_bind_wildcard", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The health check thresholds for monitoring of free space on the filesystem that contains the JournalNode's edits directory.", - "display_name": "Edits Directory Free Space Monitoring Absolute Thresholds", - "name": "journalnode_edits_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:red\",\n \"streamThreshold\": 0}, \"enabled\": \"true\"]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Directory where JournalNode will place its log files.", - "display_name": "JournalNode Log Directory", - "name": "journalnode_log_dir", - "value": "/var/log/hadoop-hdfs" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for JournalNode", - "name": "journalNode_java_opts", - "value": "" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Port for the JournalNode's HTTP web UI. Combined with the JournalNode's hostname to build its HTTP address.", - "display_name": "JournalNode HTTP Port", - "name": "dfs_journalnode_http_port", - "value": "8480" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The minimum log level for JournalNode logs", - "display_name": "JournalNode Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/hdfs-namenode.json b/sahara/plugins/cdh/v5/resources/hdfs-namenode.json deleted file mode 100644 index becac6d9..00000000 --- a/sahara/plugins/cdh/v5/resources/hdfs-namenode.json +++ /dev/null @@ -1,452 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", - "display_name": "Health Check Startup Tolerance", - "name": "namenode_startup_tolerance", - "value": "5" - }, - { - "desc": "The health test thresholds of failed status directories in a NameNode.", - "display_name": "NameNode Directory Failures Thresholds", - "name": "namenode_directory_failures_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Name of the journal located on each JournalNodes' filesystem.", - "display_name": "Quorum-based Storage Journal name", - "name": "dfs_namenode_quorum_journal_name", - "value": null - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "namenode_gc_duration_window", - "value": "5" - }, - { - "desc": "The number of server threads for the NameNode.", - "display_name": "NameNode Handler Count", - "name": "dfs_namenode_handler_count", - "value": "30" - }, - { - "desc": "Enables the health test that the NameNode is not in safemode", - "display_name": "NameNode Safemode Health Test", - "name": "namenode_safe_mode_enabled", - "value": "true" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"},\n {\"alert\": false, \"rate\": 1, \"threshold\":\"INFO\", \"content\":\"Triggering checkpoint.*\"}\n ]\n}\n" - }, - { - "desc": "The base port where the DFS NameNode web UI listens. If the port number is 0, then the server starts on a free port. Combined with the NameNode's hostname to build its HTTP address.", - "display_name": "NameNode Web UI Port", - "name": "dfs_http_port", - "value": "50070" - }, - { - "desc": "For advanced use only, a string to be inserted into dfs_hosts_exclude.txt for this role only.", - "display_name": "NameNode Advanced Configuration Snippet (Safety Valve) for dfs_hosts_exclude.txt", - "name": "namenode_hosts_exclude_safety_valve", - "value": null - }, - { - "desc": "Mountpoints that are mapped to this NameNode's Nameservice.", - "display_name": "Mountpoints", - "name": "nameservice_mountpoints", - "value": "/" - }, - { - "desc": "The health test thresholds of the age of the HDFS namespace checkpoint. Specified as a percentage of the configured checkpoint interval.", - "display_name": "Filesystem Checkpoint Age Monitoring Thresholds", - "name": "namenode_checkpoint_age_thresholds", - "value": "{\"critical\":\"400.0\",\"warning\":\"200.0\"}" - }, - { - "desc": "The base port where the secure NameNode web UI listens.", - "display_name": "Secure NameNode Web UI Port (SSL)", - "name": "dfs_https_port", - "value": "50470" - }, - { - "desc": "Maximum size for the Java Process heap memory. Passed to Java -Xmx. Measured in bytes.", - "display_name": "Java Heap Size of Namenode in Bytes", - "name": "namenode_java_heapsize", - "value": "1073741824" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "NameNode Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "This determines the total amount of block transfers to begin in parallel at a DataNode for replication, when such a command list is being sent over a DataNode heartbeat by the NameNode. The actual number is obtained by multiplying this value by the total number of live nodes in the cluster. The result number is the number of blocks to transfer immediately, per DataNode heartbeat.", - "display_name": "Replication Work Multiplier Per Iteration", - "name": "dfs_namenode_replication_work_multiplier_per_iteration", - "value": "2" - }, - { - "desc": "The health test thresholds of the number of transactions since the last HDFS namespace checkpoint. Specified as a percentage of the configured checkpointing transaction limit.", - "display_name": "Filesystem Checkpoint Transactions Monitoring Thresholds", - "name": "namenode_checkpoint_transactions_thresholds", - "value": "{\"critical\":\"400.0\",\"warning\":\"200.0\"}" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "Determines extension of safemode in milliseconds after the threshold level is reached.", - "display_name": "Safemode Extension", - "name": "dfs_safemode_extension", - "value": "30000" - }, - { - "desc": "Determines where on the local file system the NameNode should store the name table (fsimage). For redundancy, enter a comma-delimited list of directories to replicate the name table in all of the directories. Typical values are /data/N/dfs/nn where N=1..3.", - "display_name": "NameNode Data Directories", - "name": "dfs_name_dir_list", - "value": null - }, - { - "desc": "Timeout in seconds for the Hue Thrift server running on the NameNode", - "display_name": "Hue Thrift Server Timeout", - "name": "dfs_thrift_timeout", - "value": "60" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "namenode_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "The health check thresholds of the NameNode's RPC latency.", - "display_name": "NameNode RPC Latency Thresholds", - "name": "namenode_rpc_latency_thresholds", - "value": "{\"critical\":\"5000.0\",\"warning\":\"1000.0\"}" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "namenode_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "The number of server threads for the NameNode used for service calls. Only used when NameNode Service RPC Port is configured.", - "display_name": "NameNode Service Handler Count", - "name": "dfs_namenode_service_handler_count", - "value": "30" - }, - { - "desc": "Full path to a custom topology script on the host file system. The topology script is used to determine the rack location of nodes. If left blank, a topology script will be provided that uses your hosts' rack information, visible in the \"Hosts\" page.", - "display_name": "Topology Script File Name", - "name": "topology_script_file_name", - "value": null - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "namenode_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The time between two periodic file system checkpoints.", - "display_name": "Filesystem Checkpoint Period", - "name": "fs_checkpoint_period", - "value": "3600" - }, - { - "desc": "Specifies the number of DataNodes that must be live before the name node exits safemode. Enter a value less than or equal to 0 to take the number of live DataNodes into account when deciding whether to remain in safemode during startup. Values greater than the number of DataNodes in the cluster will make safemode permanent.", - "display_name": "Safemode Minimum DataNodes", - "name": "dfs_safemode_min_datanodes", - "value": "0" - }, - { - "desc": "The port where the NameNode runs the HDFS protocol. Combined with the NameNode's hostname to build its address.", - "display_name": "NameNode Port", - "name": "namenode_port", - "value": "8020" - }, - { - "desc": "Enables the health test of the upgrade status of the NameNode.", - "display_name": "HDFS Upgrade Status Health Test", - "name": "namenode_upgrade_status_enabled", - "value": "true" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for NameNode", - "name": "namenode_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "NameNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "namenode_config_safety_valve", - "value": null - }, - { - "desc": "The period to review when computing the moving average of the NameNode's RPC latency.", - "display_name": "NameNode RPC Latency Monitoring Window", - "name": "namenode_rpc_latency_window", - "value": "5" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Optional port for the service-rpc address which can be used by HDFS daemons instead of sharing the RPC address used by the clients.", - "display_name": "NameNode Service RPC Port", - "name": "dfs_namenode_servicerpc_address", - "value": null - }, - { - "desc": "When computing the overall NameNode health, consider the host's health.", - "display_name": "NameNode Host Health Test", - "name": "namenode_host_health_enabled", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for NameNode logs. Typically used by log4j.", - "display_name": "NameNode Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "Enables the health test that the NameNode's process state is consistent with the role configuration", - "display_name": "NameNode Process Health Test", - "name": "namenode_scm_health_enabled", - "value": "true" - }, - { - "desc": "The health check thresholds for the number of out-of-sync JournalNodes for this NameNode.", - "display_name": "NameNode Out-Of-Sync JournalNodes Thresholds", - "name": "namenode_out_of_sync_journal_nodes_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into dfs_hosts_allow.txt for this role only.", - "display_name": "NameNode Advanced Configuration Snippet (Safety Valve) for dfs_hosts_allow.txt", - "name": "namenode_hosts_allow_safety_valve", - "value": null - }, - { - "desc": "Number of minutes between trash checkpoints. Also controls the number of minutes after which a trash checkpoint directory is deleted. To disable the trash feature, enter 0.", - "display_name": "Filesystem Trash Interval", - "name": "fs_trash_interval", - "value": "1440" - }, - { - "desc": "The access time for HDFS file is precise upto this value. Setting the value of 0 disables access times for HDFS. When using the NFS Gateway role, make sure this property is enabled.", - "display_name": "Access Time Precision", - "name": "dfs_access_time_precision", - "value": "3600000" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:red\",\n \"streamThreshold\": 0}, \"enabled\": \"true\"]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Minimum number of running threads for the Hue Thrift server running on the NameNode", - "display_name": "Hue Thrift Server Min Threadcount", - "name": "dfs_thrift_threads_min", - "value": "10" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "If enabled, the NameNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind NameNode to Wildcard Address", - "name": "namenode_bind_wildcard", - "value": "false" - }, - { - "desc": "Comma-separated list of NameNode plug-ins to be activated. If one plug-in cannot be loaded, all the plug-ins are ignored.", - "display_name": "NameNode Plugins", - "name": "dfs_namenode_plugins_list", - "value": "" - }, - { - "desc": "Directory on a shared storage device, such as a Quorum-based Storage URI or a local directory that is an NFS mount from a NAS, to store the NameNode edits. The value of this configuration is automatically generated to be the Quourm Journal URI if there are JournalNodes and this NameNode is Highly Available.", - "display_name": "Shared Edits Directory", - "name": "dfs_namenode_shared_edits_dir", - "value": null - }, - { - "desc": "This determines the percentage amount of block invalidations (deletes) to do over a single DataNode heartbeat deletion command. The final deletion count is determined by applying this percentage to the number of live nodes in the system. The resultant number is the number of blocks from the deletion list chosen for proper invalidation over a single heartbeat of a single DataNode.", - "display_name": "Invalidate Work Percentage Per Iteration", - "name": "dfs_namenode_invalidate_work_pct_per_iteration", - "value": "0.32" - }, - { - "desc": "Enable Automatic Failover to maintain High Availability. Requires a ZooKeeper service and a High Availability NameNode partner.", - "display_name": "Enable Automatic Failover", - "name": "autofailover_enabled", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for NameNode logs. Typically used by log4j.", - "display_name": "NameNode Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "Maximum number of running threads for the Hue Thrift server running on the NameNode", - "display_name": "Hue Thrift Server Max Threadcount", - "name": "dfs_thrift_threads_max", - "value": "20" - }, - { - "desc": "Directory where NameNode will place its log files.", - "display_name": "NameNode Log Directory", - "name": "namenode_log_dir", - "value": "/var/log/hadoop-hdfs" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "namenode_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystems that contain this role's data directories. Specified as a percentage of the capacity on the filesystem. This setting is not used if a Data Directories Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Data Directories Free Space Monitoring Percentage Thresholds", - "name": "namenode_data_directories_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Nameservice of this NameNode. The Nameservice represents the interface to this NameNode and its High Availability partner. The Nameservice also represents the namespace associated with a federated NameNode.", - "display_name": "NameNode Nameservice", - "name": "dfs_federation_namenode_nameservice", - "value": null - }, - { - "desc": "The number of transactions after which the NameNode or SecondaryNameNode will create a checkpoint of the namespace, regardless of whether the checkpoint period has expired.", - "display_name": "Filesystem Checkpoint Transaction Threshold", - "name": "fs_checkpoint_txns", - "value": "1000000" - }, - { - "desc": "The minimum log level for NameNode logs", - "display_name": "NameNode Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Specifies the percentage of blocks that should satisfy the minimal replication requirement defined by dfs.replication.min. Enter a value less than or equal to 0 to wait for any particular percentage of blocks before exiting safemode. Values greater than 1 will make safemode permanent.", - "display_name": "Safemode Threshold Percentage", - "name": "dfs_safemode_threshold_pct", - "value": "0.999" - }, - { - "desc": "Directories on the local file system to store the NameNode edits. If not set, the edits are stored in the NameNode's Data Directories. The value of this configuration is automatically generated to be the Quorum-based Storage URI if there are JournalNodes and this NameNode is not Highly Available.", - "display_name": "NameNode Edits Directories", - "name": "dfs_namenode_edits_dir", - "value": null - }, - { - "desc": "If set to false and if one of the replicas of the NameNode storage fails, such as temporarily failure of NFS, this directory is not used until the NameNode restarts. If enabled, failed storage is re-checked on every checkpoint and, if it becomes valid, the NameNode will try to restore the edits and fsimage.", - "display_name": "Restore NameNode Directories at Checkpoint Time", - "name": "dfs_name_dir_restore", - "value": "false" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystems that contain this role's data directories.", - "display_name": "Data Directories Free Space Monitoring Absolute Thresholds", - "name": "namenode_data_directories_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/hdfs-nfs.json b/sahara/plugins/cdh/v5/resources/hdfs-nfs.json deleted file mode 100644 index 8622afdd..00000000 --- a/sahara/plugins/cdh/v5/resources/hdfs-nfs.json +++ /dev/null @@ -1,200 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "nfsgateway_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "NFS clients often reorder writes. As a result, sequential writes can arrive at the NFS Gateway in random order. This directory is used to temporarily save out-of-order writes before writing to HDFS. For each file, the out-of-order writes are dumped after they are accumulated to exceed certain threshold (e.g., 1MB) in memory. Please make sure this directory has enough space. For example, if the application uploads 10 files with each having 100MB, it is recommended that this directory have roughly 1GB of space in case write reorder happens (in the worst case) to every file.", - "display_name": "Temporary Dump Directory", - "name": "dfs_nfs3_dump_dir", - "value": "/tmp/.hdfs-nfs" - }, - { - "desc": "Enables the health test that the NFS Gateway's process state is consistent with the role configuration", - "display_name": "NFS Gateway Process Health Test", - "name": "nfsgateway_scm_health_enabled", - "value": "true" - }, - { - "desc": "Maximum size for the Java Process heap memory. Passed to Java -Xmx. Measured in bytes.", - "display_name": "Java Heap Size of NFS Gateway in Bytes", - "name": "nfsgateway_java_heapsize", - "value": "268435456" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "NFS Gateway Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "nfsgateway_config_safety_valve", - "value": null - }, - { - "desc": "The port number of the system portmap or rpcbind service. This configuration is used by Cloudera Manager to verify if the system portmap or rpcbind service is running before starting NFS Gateway role. Cloudera Manager does not manage the system portmap or rpcbind service.", - "display_name": "Portmap (or Rpcbind) Port", - "name": "nfs3_portmap_port", - "value": "111" - }, - { - "desc": "The NFS Gateway server port.", - "display_name": "NFS Gateway Server Port", - "name": "nfs3_server_port", - "value": "2049" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for NFS Gateway", - "name": "nfsgateway_java_opts", - "value": "" - }, - { - "desc": "The maximum number of rolled log files to keep for NFS Gateway logs. Typically used by log4j.", - "display_name": "NFS Gateway Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:red\",\n \"streamThreshold\": 0}, \"enabled\": \"true\"]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "When computing the overall NFS Gateway health, consider the host's health.", - "display_name": "NFS Gateway Host Health Test", - "name": "nfsgateway_host_health_enabled", - "value": "true" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "By default, NFS Gateway exported directories can be mounted by any client. For better access control, update this property with a list of host names and access privileges separated by whitespace characters. Host name format can be a single host, a Java regular expression, or an IPv4 address. The access privilege uses rw to specify readwrite and ro to specify readonly access. If the access privilege is not provided, the default is read-only. Examples of host name format and access privilege: \"192.168.0.0/22 rw\", \"host.*.example.com\", \"host1.test.org ro\".", - "display_name": "Allowed Hosts and Privileges", - "name": "dfs_nfs_exports_allowed_hosts", - "value": "* rw" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The port number of the mount daemon implemented inside the NFS Gateway server role.", - "display_name": "NFS Gateway MountD Port", - "name": "nfs3_mountd_port", - "value": "4242" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "NFS Gateway Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for NFS Gateway logs. Typically used by log4j.", - "display_name": "NFS Gateway Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The minimum log level for NFS Gateway logs", - "display_name": "NFS Gateway Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Directory where NFS Gateway will place its log files.", - "display_name": "NFS Gateway Log Directory", - "name": "nfsgateway_log_dir", - "value": "/var/log/hadoop-hdfs" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/hdfs-secondarynamenode.json b/sahara/plugins/cdh/v5/resources/hdfs-secondarynamenode.json deleted file mode 100644 index 8c637d20..00000000 --- a/sahara/plugins/cdh/v5/resources/hdfs-secondarynamenode.json +++ /dev/null @@ -1,254 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"},\n {\"alert\": false, \"rate\": 1, \"threshold\":\"INFO\", \"content\":\"Triggering checkpoint.*\"}\n ]\n}\n" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Secondary NameNode", - "name": "secondarynamenode_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystems that contain this role's checkpoint directories.", - "display_name": "Checkpoint Directories Free Space Monitoring Absolute Thresholds", - "name": "secondarynamenode_checkpoint_directories_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Maximum size for the Java Process heap memory. Passed to Java -Xmx. Measured in bytes.", - "display_name": "Java Heap Size of Secondary namenode in Bytes", - "name": "secondary_namenode_java_heapsize", - "value": "1073741824" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for SecondaryNameNode logs. Typically used by log4j.", - "display_name": "SecondaryNameNode Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:red\",\n \"streamThreshold\": 0}, \"enabled\": \"true\"]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "SecondaryNameNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "secondarynamenode_config_safety_valve", - "value": null - }, - { - "desc": "Enables the health test that the SecondaryNameNode's process state is consistent with the role configuration", - "display_name": "SecondaryNameNode Process Health Test", - "name": "secondarynamenode_scm_health_enabled", - "value": "true" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The time between two periodic file system checkpoints.", - "display_name": "Filesystem Checkpoint Period", - "name": "fs_checkpoint_period", - "value": "3600" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "If enabled, the SecondaryNameNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind SecondaryNameNode to Wildcard Address", - "name": "secondary_namenode_bind_wildcard", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "SecondaryNameNode Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The number of transactions after which the NameNode or SecondaryNameNode will create a checkpoint of the namespace, regardless of whether the checkpoint period has expired.", - "display_name": "Filesystem Checkpoint Transaction Threshold", - "name": "fs_checkpoint_txns", - "value": "1000000" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "secondarynamenode_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "secondarynamenode_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystems that contain this role's checkpoint directories. Specified as a percentage of the capacity on the filesystem. This setting is not used if a Checkpoint Directories Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Checkpoint Directories Free Space Monitoring Percentage Thresholds", - "name": "secondarynamenode_checkpoint_directories_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "When computing the overall SecondaryNameNode health, consider the host's health.", - "display_name": "SecondaryNameNode Host Health Test", - "name": "secondarynamenode_host_health_enabled", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for SecondaryNameNode logs. Typically used by log4j.", - "display_name": "SecondaryNameNode Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "secondarynamenode_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "secondarynamenode_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "secondarynamenode_gc_duration_window", - "value": "5" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Determines where on the local file system the DFS SecondaryNameNode should store the temporary images to merge. For redundancy, enter a comma-delimited list of directories to replicate the image in all of the directories. Typical values are /data/N/dfs/snn for N = 1, 2, 3...", - "display_name": "HDFS Checkpoint Directory", - "name": "fs_checkpoint_dir_list", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "Nameservice of this SecondaryNameNode", - "display_name": "SecondaryNameNode Nameservice", - "name": "dfs_secondarynamenode_nameservice", - "value": null - }, - { - "desc": "The minimum log level for SecondaryNameNode logs", - "display_name": "SecondaryNameNode Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "The base port where the secure SecondaryNameNode web UI listens.", - "display_name": "Secure SecondaryNameNode Web UI Port (SSL)", - "name": "dfs_secondary_https_port", - "value": "50495" - }, - { - "desc": "Directory where SecondaryNameNode will place its log files.", - "display_name": "SecondaryNameNode Log Directory", - "name": "secondarynamenode_log_dir", - "value": "/var/log/hadoop-hdfs" - }, - { - "desc": "The SecondaryNameNode HTTP port. If the port is 0, then the server starts on a free port. Combined with the SecondaryNameNode's hostname to build its HTTP address.", - "display_name": "SecondaryNameNode Web UI Port", - "name": "dfs_secondary_http_port", - "value": "50090" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/hdfs-service.json b/sahara/plugins/cdh/v5/resources/hdfs-service.json deleted file mode 100644 index a3c07a2f..00000000 --- a/sahara/plugins/cdh/v5/resources/hdfs-service.json +++ /dev/null @@ -1,608 +0,0 @@ -[ - { - "desc": "Timeout in milliseconds for the parallel RPCs made in DistributedFileSystem#getFileBlockStorageLocations(). This value is only emitted for Impala.", - "display_name": "HDFS File Block Storage Location Timeout", - "name": "dfs_client_file_block_storage_locations_timeout", - "value": "10000" - }, - { - "desc": "The domain to use for the HTTP cookie that stores the authentication token. In order for authentiation to work correctly across all Hadoop nodes' web-consoles the domain must be correctly set. Important: when using IP addresses, browsers ignore cookies with domain settings. For this setting to work properly all nodes in the cluster must be configured to generate URLs with hostname.domain names on it.", - "display_name": "Hadoop HTTP Authentication Cookie Domain", - "name": "hadoop_http_auth_cookie_domain", - "value": "" - }, - { - "desc": "The user that this service's processes should run as (except the HttpFS server, which has its own user)", - "display_name": "System User", - "name": "process_username", - "value": "hdfs" - }, - { - "desc": "Event filters are defined in a JSON object like the following:
\n\n\n{\n \"defaultAction\" : (\"accept\", \"discard\"),\n \"rules\" : [\n {\n \"action\" : (\"accept\", \"discard\"),\n \"fields\" : [\n {\n \"name\" : \"fieldName\",\n \"match\" : \"regex\"\n }\n ]\n }\n ]\n}\n\n\n\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n
\n\n\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n
\n\n\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n
\n\n\nThe following is the list of fields that can be filtered for HDFS events:\n
\n\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:red\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "Comma-separated list of Kerberos realms that Hadoop services should trust. If empty, defaults to the configured default_realm in the krb5.conf file. After changing this value and restarting the service, any services depending on this one must be restarted as well. The hadoop.security.auth_to_local property is configured using this information.", - "display_name": "Trusted Kerberos Realms", - "name": "trusted_realms", - "value": "" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "Enables the health check that verifies that the failover controllers associated with this service are healthy and running.", - "display_name": "Failover Controllers Healthy", - "name": "failover_controllers_healthy_enabled", - "value": "true" - }, - { - "desc": "The attribute of the group object that identifies the users that are members of the group. The default will usually be appropriate for any LDAP installation.", - "display_name": "Hadoop User Group Mapping LDAP Group Membership Attribute", - "name": "hadoop_group_mapping_ldap_member_attr", - "value": "member" - }, - { - "desc": "Comma separated list of users allowed to do short circuit read. A short circuit read allows a client co-located with the data to read HDFS file blocks directly from HDFS. If empty, will default to the DataNode process' user.", - "display_name": "DataNode Local Path Access Users", - "name": "dfs_block_local_path_access_user", - "value": null - }, - { - "desc": "The timeout, in milliseconds, to use with the Cloudera Manager agent-based fencer.", - "display_name": "Timeout for Cloudera Manager Fencing Strategy", - "name": "dfs_ha_fencing_cloudera_manager_timeout_millis", - "value": "10000" - }, - { - "desc": "Enable collection of audit events from the service's roles.", - "display_name": "Enable Collection", - "name": "navigator_audit_enabled", - "value": "true" - }, - { - "desc": "Maximum bandwidth used for image transfer in bytes per second. This can help keep normal namenode operations responsive during checkpointing. A default value of 0 indicates that throttling is disabled.", - "display_name": "FsImage Transfer Bandwidth", - "name": "dfs_image_transfer_bandwidthPerSec", - "value": "0" - }, - { - "desc": "The user the management services will impersonate as when connecting to HDFS. Defaults to 'hdfs', a superuser.", - "display_name": "HDFS User to Impersonate", - "name": "hdfs_user_to_impersonate", - "value": "hdfs" - }, - { - "desc": "File path to the SSL keystore containing the SSL certificate required by the LDAP server.", - "display_name": "Hadoop User Group Mapping LDAP SSL Keystore", - "name": "hadoop_group_mapping_ldap_keystore", - "value": "" - }, - { - "desc": "The short name of Hue's Kerberos principal", - "display_name": "Hue's Kerberos Principal Short Name", - "name": "hue_kerberos_principal_shortname", - "value": "hue" - }, - { - "desc": "The minimal block replication.", - "display_name": "Minimal Block Replication", - "name": "dfs_replication_min", - "value": "1" - }, - { - "desc": "The maximal block replication.", - "display_name": "Maximal Block Replication", - "name": "dfs_replication_max", - "value": "512" - }, - { - "desc": "The service monitor will use these permissions to create the directory and files to test if the hdfs service is healthy. Permissions are specified using the 10-character unix-symbolic format e.g. '-rwxr-xr-x'.", - "display_name": "HDFS Health Canary Directory Permissions", - "name": "firehose_hdfs_canary_directory_permissions", - "value": "-rwxrwxrwx" - }, - { - "desc": "Enable authorization", - "display_name": "Hadoop Secure Authorization", - "name": "hadoop_security_authorization", - "value": "false" - }, - { - "desc": "The attribute of the group object that identifies the group name. The default will usually be appropriate for all LDAP systems.", - "display_name": "Hadoop User Group Mapping LDAP Group Name Attribute", - "name": "hadoop_group_mapping_ldap_group_name_attr", - "value": "cn" - }, - { - "desc": "Enables DataNode support for the experimental DistributedFileSystem.getFileVBlockStorageLocations API. Applicable to CDH 4.1 and onwards.", - "display_name": "Enable HDFS Block Metadata API", - "name": "dfs_datanode_hdfs_blocks_metadata_enabled", - "value": "true" - }, - { - "desc": "The tolerance window that will be used in HDFS service tests that depend on detection of the active NameNode.", - "display_name": "Active NameNode Detection Window", - "name": "hdfs_active_namenode_detecton_window", - "value": "3" - }, - { - "desc": "Default block replication. The number of replications to make when the file is created. The default value is used if a replication number is not specified.", - "display_name": "Replication Factor", - "name": "dfs_replication", - "value": "3" - }, - { - "desc": "Comma-delimited list of groups that you want to allow the HTTP user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'. This is used by WebHCat.", - "display_name": "HTTP Proxy User Groups", - "name": "HTTP_proxy_user_groups_list", - "value": "*" - }, - { - "desc": "The name of the system group shared by all the core Hadoop services.", - "display_name": "Shared Hadoop Group Name", - "name": "hdfs_hadoop_group_name", - "value": "hadoop" - }, - { - "desc": "The amount of time to wait for HDFS filesystem image transfer from NameNode to complete.", - "display_name": "FsImage Transfer Timeout", - "name": "dfs_image_transfer_timeout", - "value": "60000" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the Hive user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", - "display_name": "Hive Proxy User Hosts", - "name": "hive_proxy_user_hosts_list", - "value": "*" - }, - { - "desc": "An additional filter to use when searching for LDAP users. The default will usually be appropriate for Active Directory installations. If connecting to a generic LDAP server, ''sAMAccountName'' will likely be replaced with ''uid''. {0} is a special string used to denote where the username fits into the filter.", - "display_name": "Hadoop User Group Mapping LDAP User Search Filter", - "name": "hadoop_group_mapping_ldap_user_filter", - "value": "(&(objectClass=user)(sAMAccountName={0}))" - }, - { - "desc": "List of fencing methods to use for service fencing. shell(./cloudera_manager_agent_fencer.py) is a fencing mechanism designed to take advantage of the CM agent. The sshfence method uses SSH. If using custom fencers (that may talk to shared store, power units, or network switches), use the shell mechanism to invoke them.", - "display_name": "HDFS High Availability Fencing Methods", - "name": "dfs_ha_fencing_methods", - "value": "shell(./cloudera_manager_agent_fencer.py)" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the environment of HDFS replication jobs.", - "display_name": "HDFS Replication Advanced Configuration Snippet (Safety Valve)", - "name": "hdfs_replication_env_safety_valve", - "value": null - }, - { - "desc": "Enables the health check that a client can create, read, write, and delete files", - "display_name": "HDFS Canary Health Check", - "name": "hdfs_canary_health_enabled", - "value": "true" - }, - { - "desc": "Path on the DataNode's local file system to a UNIX domain socket used for communication between the DataNode and local HDFS clients. This socket is used for Short Circuit Reads. Only the HDFS System User and \"root\" should have write access to the parent directory and all of its ancestors. This property is supported in CDH 4.2 or later deployments.", - "display_name": "UNIX Domain Socket path", - "name": "dfs_domain_socket_path", - "value": "/var/run/hdfs-sockets/dn" - }, - { - "desc": "Algorithm to encrypt data transfer between DataNodes and clients, and among DataNodes. 3des is more cryptographically secure, but rc4 is substantially faster.", - "display_name": "Data Transfer Encryption Algorithm", - "name": "dfs_encrypt_data_transfer_algorithm", - "value": "rc4" - }, - { - "desc": "The health check thresholds of the number of under-replicated blocks. Specified as a percentage of the total number of blocks.", - "display_name": "Under-replicated Block Monitoring Thresholds", - "name": "hdfs_under_replicated_blocks_thresholds", - "value": "{\"critical\":\"40.0\",\"warning\":\"10.0\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml. Applies to configurations of all roles in this service except client configuration.", - "display_name": "HDFS Service Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "hdfs_service_config_safety_valve", - "value": null - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the HTTP user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'. This is used by WebHCat.", - "display_name": "HTTP Proxy User Hosts", - "name": "HTTP_proxy_user_hosts_list", - "value": "*" - }, - { - "desc": "\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n
\n\n\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n
\n\nEvent trackers are defined in a JSON object like the following:
\n\n\n{\n \"timeToLive\" : [integer],\n \"fields\" : [\n {\n \"type\" : [string],\n \"name\" : [string]\n }\n ]\n}\n\n\n\nWhere:\n
\n\n\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n
\n\n\nThe following is the list of fields that can be used to compare HDFS events:\n
\n\nThe configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned will cause the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", - "display_name": "HiveServer2 Advanced Configuration Snippet (Safety Valve) for hive-site.xml", - "name": "hive_hs2_config_safety_valve", - "value": null - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "HiveServer2 Environment Advanced Configuration Snippet (Safety Valve)", - "name": "hive_hs2_env_safety_valve", - "value": null - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "HiveServer2 Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "When computing the overall HiveServer2 health, consider the host's health.", - "display_name": "HiveServer2 Host Health Test", - "name": "hiveserver2_host_health_enabled", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Enables the health test that the HiveServer2's process state is consistent with the role configuration", - "display_name": "HiveServer2 Process Health Test", - "name": "hiveserver2_scm_health_enabled", - "value": "true" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of HiveServer2 in Bytes", - "name": "hiveserver2_java_heapsize", - "value": "268435456" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "Directory where HiveServer2 will place its log files.", - "display_name": "HiveServer2 Log Directory", - "name": "hive_log_dir", - "value": "/var/log/hive" - }, - { - "desc": "Maximum number of worker threads in HiveServer2's thread pool", - "display_name": "Max HiveServer2 Threads", - "name": "hiveserver2_max_threads", - "value": "100" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "hiveserver2_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The minimum log level for HiveServer2 logs", - "display_name": "HiveServer2 Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for HiveServer2", - "name": "hiveserver2_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/hive-metastore.json b/sahara/plugins/cdh/v5/resources/hive-metastore.json deleted file mode 100644 index d520595e..00000000 --- a/sahara/plugins/cdh/v5/resources/hive-metastore.json +++ /dev/null @@ -1,194 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Hive Metastore Server", - "name": "hive_metastore_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "Maximum number of worker threads in the Hive Metastore Server's thread pool", - "display_name": "Max Hive Metastore Server Threads", - "name": "hive_metastore_max_threads", - "value": "100000" - }, - { - "desc": "Enables the health test that checks that basic Hive Metastore operations succeed", - "display_name": "Hive Metastore Canary Health Test", - "name": "metastore_canary_health_enabled", - "value": "true" - }, - { - "desc": "Enables the health test that the Hive Metastore Server's process state is consistent with the role configuration", - "display_name": "Hive Metastore Server Process Health Test", - "name": "hivemetastore_scm_health_enabled", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for Hive Metastore Server logs. Typically used by log4j.", - "display_name": "Hive Metastore Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned will cause the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "Minimum number of worker threads in the Hive Metastore Server's thread pool", - "display_name": "Min Hive Metastore Server Threads", - "name": "hive_metastore_min_threads", - "value": "200" - }, - { - "desc": "When computing the overall Hive Metastore Server health, consider the host's health.", - "display_name": "Hive Metastore Server Host Health Test", - "name": "hivemetastore_host_health_enabled", - "value": "true" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Hive Metastore Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Hive Metastore Server in Bytes", - "name": "hive_metastore_java_heapsize", - "value": "1073741824" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "hivemetastore_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Port on which Hive Metastore Server will listen for connections.", - "display_name": "Hive Metastore Server Port", - "name": "hive_metastore_port", - "value": "9083" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", - "display_name": "Hive Metastore Server Advanced Configuration Snippet (Safety Valve) for hive-site.xml", - "name": "hive_metastore_config_safety_valve", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The maximum size, in megabytes, per log file for Hive Metastore Server logs. Typically used by log4j.", - "display_name": "Hive Metastore Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "Directory where Hive Metastore Server will place its log files.", - "display_name": "Hive Metastore Server Log Directory", - "name": "hive_log_dir", - "value": "/var/log/hive" - }, - { - "desc": "The minimum log level for Hive Metastore Server logs", - "display_name": "Hive Metastore Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Hive Metastore Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "hive_metastore_env_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/hive-service.json b/sahara/plugins/cdh/v5/resources/hive-service.json deleted file mode 100644 index 0fa23992..00000000 --- a/sahara/plugins/cdh/v5/resources/hive-service.json +++ /dev/null @@ -1,314 +0,0 @@ -[ - { - "desc": "For advanced use only, a string to be inserted into hive-site.xml. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Hive Service Advanced Configuration Snippet (Safety Valve) for hive-site.xml", - "name": "hive_service_config_safety_valve", - "value": null - }, - { - "desc": "Directory containing auxiliary JARs used by Hive. This should be a directory location and not a classpath containing one or more JARs. This directory must be created and managed manually on Hive CLI or HiveServer2 host.", - "display_name": "Hive Auxiliary JARs Directory", - "name": "hive_aux_jars_path_dir", - "value": null - }, - { - "desc": "Comma-delimited list of groups that you want to allow the Oozie user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", - "display_name": "Oozie Proxy User Groups", - "name": "oozie_proxy_user_groups_list", - "value": "*" - }, - { - "desc": "Password for Hive Metastore database", - "display_name": "Hive Metastore Database Password", - "name": "hive_metastore_database_password", - "value": "" - }, - { - "desc": "Directory name where Hive Metastore's database is stored (only for Derby)", - "display_name": "Hive Metastore Derby Path", - "name": "hive_metastore_derby_path", - "value": "/var/lib/hive/cloudera_manager/derby/metastore_db" - }, - { - "desc": "Hive warehouse directory is the location in HDFS where Hive's tables are stored. Note that Hive's default value for its warehouse directory is '/user/hive/warehouse'.", - "display_name": "Hive Warehouse Directory", - "name": "hive_warehouse_directory", - "value": "/user/hive/warehouse" - }, - { - "desc": "SSL keystore password.", - "display_name": "Keystore Password", - "name": "hiveserver2_keystore_password", - "value": null - }, - { - "desc": "\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n
\n\n\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n
\n\nEvent trackers are defined in a JSON object like the following:
\n\n\n{\n \"timeToLive\" : [integer],\n \"fields\" : [\n {\n \"type\" : [string],\n \"name\" : [string]\n }\n ]\n}\n\n\n\nWhere:\n
\n\n\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n
\n\n\nThe following is the list of fields that can be used to compare Hive events:\n
\n\nEvent filters are defined in a JSON object like the following:
\n\n\n{\n \"defaultAction\" : (\"accept\", \"discard\"),\n \"rules\" : [\n {\n \"action\" : (\"accept\", \"discard\"),\n \"fields\" : [\n {\n \"name\" : \"fieldName\",\n \"match\" : \"regex\"\n }\n ]\n }\n ]\n}\n\n\n\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n
\n\n\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n
\n\n\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n
\n\n\nThe following is the list of fields that can be filtered for Hive events:\n
\n\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned will cause the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "Maximum number of rolled over audit logs to retain. The logs will not be deleted if they contain audit events that have not yet been propagated to Audit Server.", - "display_name": "Number of Audit Logs to Retain", - "name": "navigator_audit_log_max_backup_index", - "value": "10" - }, - { - "desc": "Port number of Hive Metastore database", - "display_name": "Hive Metastore Database Port", - "name": "hive_metastore_database_port", - "value": "3306" - }, - { - "desc": "Name of the ZooKeeper service that this Hive service instance depends on.", - "display_name": "ZooKeeper Service", - "name": "zookeeper_service", - "value": null - }, - { - "desc": "Host name of Hive Metastore database", - "display_name": "Hive Metastore Database Host", - "name": "hive_metastore_database_host", - "value": "localhost" - }, - { - "desc": "Maximum size of audit log file in MB before it is rolled over.", - "display_name": "Maximum Audit Log File Size", - "name": "navigator_audit_log_max_file_size", - "value": "100" - }, - { - "desc": "Prevent Metastore operations in the event of schema version incompatibility. Consider setting this to true to reduce probability of schema corruption during Metastore operations. Note that setting this property to true will also set datanucleus.autoCreateSchema property to false and datanucleus.fixedDatastore property to true. Any values set in Cloudera Manager for these properties will be overridden.", - "display_name": "Strict Hive Metastore Schema Validation", - "name": "hive_metastore_schema_verification", - "value": "true" - }, - { - "desc": "Path to the SSL keystore.", - "display_name": "Keystore File Path", - "name": "hiveserver2_keystore_path", - "value": null - }, - { - "desc": "HDFS path to the global policy file for Sentry authorization. This should be a relative path (and not a full HDFS URL). The global policy file must be in Sentry policy file format.", - "display_name": "Sentry Global Policy File", - "name": "hive_sentry_provider_resource", - "value": "/user/hive/sentry/sentry-provider.ini" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Hive Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "hive_service_env_safety_valve", - "value": null - }, - { - "desc": "Path to the directory where audit logs will be written. The directory will be created if it doesn't exist.", - "display_name": "Audit Log Directory", - "name": "audit_event_log_dir", - "value": "/var/log/hive/audit" - }, - { - "desc": "Name of Hive Metastore database", - "display_name": "Hive Metastore Database Name", - "name": "hive_metastore_database_name", - "value": "metastore" - }, - { - "desc": "The class to use in Sentry authorization for user to group mapping. Sentry authorization may be configured to use either Hadoop user to group mapping or local groups defined in the policy file. Hadoop user to group mapping may be configured in the Cloudera Manager HDFS service configuration page under the Security section.", - "display_name": "Sentry User to Group Mapping Class", - "name": "hive_sentry_provider", - "value": "org.apache.sentry.provider.file.HadoopGroupResourceAuthorizationProvider" - }, - { - "desc": "Use Sentry to enable role-based, fine-grained authorization. This configuration enables Sentry using policy files. To enable Sentry using Sentry service instead, add Sentry service as a dependency to Hive service. Sentry service provides concurrent and secure access to authorization policy metadata and is the recommended option for enabling Sentry. Sentry is supported only on CDH 4.4 or later deployments. Before enabling Sentry, read the requirements and configuration steps outlined in Setting Up Hive Authorization with Sentry.", - "display_name": "Enable Sentry Authorization using Policy Files", - "name": "sentry_enabled", - "value": "false" - }, - { - "desc": "This configuration overrides the value set for Hive Proxy User Groups configuration in HDFS service for use by Hive Metastore Server. Specify a comma-delimited list of groups that you want to allow access to Hive Metastore metadata and allow the Hive user to impersonate. A value of '*' allows all groups. Default value of empty inherits the value set for Hive Proxy User Groups configuration in HDFS service.", - "display_name": "Hive Metastore Access Control and Proxy User Groups Override", - "name": "hive_proxy_user_groups_list", - "value": null - }, - { - "desc": "Default number of reduce tasks per job. Usually set to a prime number close to the number of available hosts. Ignored when mapred.job.tracker is \"local\". Hadoop sets this to 1 by default, while Hive uses -1 as the default. When set to -1, Hive will automatically determine an appropriate number of reducers for each job.", - "display_name": "Hive Reduce Tasks", - "name": "hive_reduce_tasks", - "value": "-1" - }, - { - "desc": "Let the table directories inherit the permission of the Warehouse or Database directory instead of being created with the permissions derived from dfs umask. This allows Impala to insert into tables created via Hive.", - "display_name": "Hive Warehouse Subdirectories Inherit Permissions", - "name": "hive_warehouse_subdir_inherit_perms", - "value": "true" - }, - { - "desc": "The group that this service's processes should run as.", - "display_name": "System Group", - "name": "process_groupname", - "value": "hive" - }, - { - "desc": "The health test thresholds of the overall Hive Metastore Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Hive Metastore Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Hive Metastore Servers falls below the critical threshold.", - "display_name": "Healthy Hive Metastore Server Monitoring Thresholds", - "name": "hive_hivemetastores_healthy_thresholds", - "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The server name used when defining privilege rules in Sentry authorization. Sentry uses this name as an alias for the Hive service. It has nothing to do with any physical server name.", - "display_name": "Server Name for Sentry Authorization", - "name": "hive_sentry_server", - "value": "server1" - }, - { - "desc": "The health test thresholds of the overall HiveServer2 health. The check returns \"Concerning\" health if the percentage of \"Healthy\" HiveServer2s falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" HiveServer2s falls below the critical threshold.", - "display_name": "Healthy HiveServer2 Monitoring Thresholds", - "name": "hive_hiveserver2s_healthy_thresholds", - "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" - }, - { - "desc": "For advanced use only, a list of configuration properties that will be used by the Service Monitor instead of the current client configuration for the service.", - "display_name": "Service Monitor Client Config Overrides", - "name": "smon_client_config_overrides", - "value": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned will cause the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of WebHCat Server in Bytes", - "name": "hive_webhcat_java_heapsize", - "value": "268435456" - }, - { - "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", - "display_name": "WebHCat Server Advanced Configuration Snippet (Safety Valve) for hive-site.xml", - "name": "hive_webhcat_hive_config_safety_valve", - "value": null - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "webhcat_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "WebHCat Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "WebHCat Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "hive_webhcat_env_safety_valve", - "value": null - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for WebHCat Server", - "name": "hive_webhcat_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The maximum size, in megabytes, per log file for WebHCat Server logs. Typically used by log4j.", - "display_name": "WebHCat Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "Enables the health test that the WebHCat Server's process state is consistent with the role configuration", - "display_name": "WebHCat Server Process Health Test", - "name": "webhcat_scm_health_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into webhcat-site.xml for this role only.", - "display_name": "WebHCat Server Advanced Configuration Snippet (Safety Valve) for webhcat-site.xml", - "name": "hive_webhcat_config_safety_valve", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "When computing the overall WebHCat Server health, consider the host's health.", - "display_name": "WebHCat Server Host Health Test", - "name": "webhcat_host_health_enabled", - "value": "true" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The minimum log level for WebHCat Server logs", - "display_name": "WebHCat Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Directory where WebHCat Server will place its log files.", - "display_name": "WebHCat Server Log Directory", - "name": "hcatalog_log_dir", - "value": "/var/log/hcatalog" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/hue-hue.json b/sahara/plugins/cdh/v5/resources/hue-hue.json deleted file mode 100644 index bd21c71d..00000000 --- a/sahara/plugins/cdh/v5/resources/hue-hue.json +++ /dev/null @@ -1,158 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "hue_server_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Enable HTTPS for the Hue web server.", - "display_name": "Enable HTTPS", - "name": "ssl_enable", - "value": "false" - }, - { - "desc": "Path to the SSL private key on the host running the Hue web server. This file must be readable by the Hue system user. Hue only supports a key without a passphrase.", - "display_name": "Local Path to SSL Private Key", - "name": "ssl_private_key", - "value": null - }, - { - "desc": "Directory where Hue Server will place its log files.", - "display_name": "Hue Server Log Directory", - "name": "hue_server_log_dir", - "value": "/var/log/hue" - }, - { - "desc": "Timeout in seconds for Thrift calls to HiveServer2.", - "display_name": "HiveServer2 Connection Timeout", - "name": "hs2_conn_timeout", - "value": "120" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Location on HDFS where the jobsub examples and templates are stored.", - "display_name": "Jobsub Examples and Templates Directory", - "name": "hue_server_remote_data_dir", - "value": "/user/hue/jobsub" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "When computing the overall Hue Server health, consider the host's health.", - "display_name": "Hue Server Host Health Test", - "name": "hue_server_host_health_enabled", - "value": "true" - }, - { - "desc": "Port to use to connect to the Hue server.", - "display_name": "Hue HTTP Port", - "name": "hue_http_port", - "value": "8888" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into hue_safety_valve_server.ini for this role only.", - "display_name": "Hue Server Advanced Configuration Snippet (Safety Valve) for hue_safety_valve_server.ini", - "name": "hue_server_hue_safety_valve", - "value": null - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Path to the SSL certificate on the host running the Hue web server. This file must be readable by the Hue system user.", - "display_name": "Local Path to SSL Certificate", - "name": "ssl_certificate", - "value": null - }, - { - "desc": "If enabled, the Hue server binds to the wildcard address (\"0.0.0.0\") for its ports.", - "display_name": "Bind Hue Server to Wildcard Address", - "name": "hue_server_bind_wildcard", - "value": "false" - }, - { - "desc": "Enables the health test that the Hue Server's process state is consistent with the role configuration", - "display_name": "Hue Server Process Health Test", - "name": "hue_server_scm_health_enabled", - "value": "true" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Random string used for secure hashing in the session store.", - "display_name": "Secret Key", - "name": "secret_key", - "value": null - } -] diff --git a/sahara/plugins/cdh/v5/resources/hue-service.json b/sahara/plugins/cdh/v5/resources/hue-service.json deleted file mode 100644 index 6e8ba875..00000000 --- a/sahara/plugins/cdh/v5/resources/hue-service.json +++ /dev/null @@ -1,332 +0,0 @@ -[ - { - "desc": "Create users in Hue when they try to login with their LDAP credentials. For use when using LdapBackend for Hue authentication.", - "display_name": "Create LDAP users on login", - "name": "create_users_on_login", - "value": "true" - }, - { - "desc": "Number of threads used by the Hue web server.", - "display_name": "Hue Web Server Threads", - "name": "cherrypy_server_threads", - "value": "10" - }, - { - "desc": "Thrift server to use for HBase app.", - "display_name": "HBase Thrift Server", - "name": "hue_hbase_thrift", - "value": null - }, - { - "desc": "Name of the Oozie service that this Hue service instance depends on", - "display_name": "Oozie Service", - "name": "oozie_service", - "value": null - }, - { - "desc": "The distinguished name to use as a search base for finding users and groups.", - "display_name": "LDAP Search Base", - "name": "base_dn", - "value": null - }, - { - "desc": "Comma-separated list of regular expressions, which match 'host:port' of requested proxy target.", - "display_name": "Whitelist", - "name": "whitelist", - "value": "(localhost|127\\.0\\.0\\.1):(50030|50070|50060|50075)" - }, - { - "desc": "Name of the Sentry service that this Hue service instance depends on", - "display_name": "Sentry Service", - "name": "sentry_service", - "value": null - }, - { - "desc": "Name of the HBase service that this Hue service instance depends on", - "display_name": "HBase Service", - "name": "hbase_service", - "value": null - }, - { - "desc": "Base filter for searching for groups", - "display_name": "LDAP Group Filter", - "name": "group_filter", - "value": null - }, - { - "desc": "Time zone name.", - "display_name": "Time Zone", - "name": "time_zone", - "value": "America/Los_Angeles" - }, - { - "desc": "The health test thresholds of the overall Kerberos Ticket Renewer health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Kerberos Ticket Renewers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Kerberos Ticket Renewers falls below the critical threshold.", - "display_name": "Healthy Kerberos Ticket Renewer Monitoring Thresholds", - "name": "hue_kt_renewers_healthy_thresholds", - "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" - }, - { - "desc": "The user that this service's processes should run as.", - "display_name": "System User", - "name": "process_username", - "value": "hue" - }, - { - "desc": "The health test thresholds of the overall Hue Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Hue Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Hue Servers falls below the critical threshold.", - "display_name": "Healthy Hue Server Monitoring Thresholds", - "name": "hue_hue_servers_healthy_thresholds", - "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The group name attribute in the LDAP schema.", - "display_name": "LDAP Group Name Attribute", - "name": "group_name_attr", - "value": null - }, - { - "desc": "HDFS directory used for storing temporary files.", - "display_name": "HDFS Temporary Directory", - "name": "hdfs_tmp_dir", - "value": "/tmp" - }, - { - "desc": "Name of the Solr service that this Hue service instance depends on", - "display_name": "Solr Service", - "name": "solr_service", - "value": null - }, - { - "desc": "If the database is SQLite3, this is the filename of the database to use, and the directory of this file must be writable by the 'hue' user.", - "display_name": "Hue Database Directory", - "name": "database_dir", - "value": "/var/lib/hue/desktop.db" - }, - { - "desc": "The name of a default group that users will be added to at creation time.", - "display_name": "Default User Group", - "name": "default_user_group", - "value": null - }, - { - "desc": "When you enable anonymous usage data collection Hue tracks anonymised pages and application versions in order to gather information about each application's usage levels. The data collected does not include any hostnames or IDs. Data collection option is available on CDH 4.4 and later deployments.", - "display_name": "Enable Usage Data Collection", - "name": "usage_data_collection_enable", - "value": "true" - }, - { - "desc": "Name of the ZooKeeper service that this Hue service instance depends on", - "display_name": "ZooKeeper Service", - "name": "zookeeper_service", - "value": null - }, - { - "desc": "Distinguished name of the user to bind as -- not necessary if the LDAP server supports anonymous searches.", - "display_name": "LDAP Bind User", - "name": "bind_dn", - "value": null - }, - { - "desc": "URL of LDAP Server", - "display_name": "LDAP URL", - "name": "ldap_url", - "value": null - }, - { - "desc": "Whether to use StartTLS (as opposed to ldaps) to communicate securely with the LDAP server. This is only effective when the LDAP certificate is specified.", - "display_name": "Use StartTLS", - "name": "use_start_tls", - "value": "true" - }, - { - "desc": "Name of the Sqoop service that this Hue service instance depends on", - "display_name": "Sqoop Service", - "name": "sqoop_service", - "value": null - }, - { - "desc": "Active Directory Domain", - "display_name": "NT Domain", - "name": "nt_domain", - "value": null - }, - { - "desc": "Class that defines extra accessor methods for user objects.", - "display_name": "User Augmentor", - "name": "user_augmentor", - "value": "desktop.auth.backend.DefaultUserAugmentor" - }, - { - "desc": "File where the database gets dumped to or loaded from.", - "display_name": "Database Dump File", - "name": "database_dump_file", - "value": "/tmp/hue_database_dump.json" - }, - { - "desc": "LDAP Username Pattern for use with non-Active Directory LDAP implementations. Must contain the special '<username>' string for replacement during authentication.", - "display_name": "LDAP Username Pattern", - "name": "ldap_username_pattern", - "value": null - }, - { - "desc": "The password of the bind user.", - "display_name": "LDAP Bind Password", - "name": "bind_password", - "value": null - }, - { - "desc": "The group that this service's processes should run as.", - "display_name": "System Group", - "name": "process_groupname", - "value": "hue" - }, - { - "desc": "LDAP certificate for authentication over TLS", - "display_name": "LDAP Certificate", - "name": "ldap_cert", - "value": null - }, - { - "desc": "Type of database used for Hue", - "display_name": "Hue Database Type", - "name": "database_type", - "value": "sqlite3" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "When computing the overall HUE health, consider Beeswax Server's health", - "display_name": "Beeswax Server Role Health Test", - "name": "hue_beeswax_server_health_enabled", - "value": "true" - }, - { - "desc": "HTTPFS role or Namenode (if webhdfs is enabled) that hue can use to communicate with HDFS.", - "display_name": "HDFS Web Interface Role", - "name": "hue_webhdfs", - "value": null - }, - { - "desc": "Mode of authenticating login credentials. Select desktop.auth.backend.LdapBackend to use LDAP to authenticate login credentials. LDAP requires you to also set the LDAP URL, NT Domain, and optionally LDAP certificate if you are using secure LDAP. Select desktop.auth.backend.PamBackend to use PAM to authenticate login credentials.", - "display_name": "Authentication Backend", - "name": "auth_backend", - "value": "desktop.auth.backend.AllowFirstUserDjangoBackend" - }, - { - "desc": "The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "The username to use to log into the Hue database. Not necessary for SQLite3.", - "display_name": "Hue Database Username", - "name": "database_user", - "value": "hue" - }, - { - "desc": "Comma-separated list of regular expressions, which match any prefix of 'host:port/path' of requested proxy target. This does not support matching GET parameters.", - "display_name": "Blacklist", - "name": "blacklist", - "value": "()" - }, - { - "desc": "For advanced use only, a string to be inserted into sentry-site.xml. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Hue Service Advanced Configuration Snippet (Safety Valve) for sentry-site.xml", - "name": "hue_sentry_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Hue Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "hue_service_env_safety_valve", - "value": null - }, - { - "desc": "Name of the Impala service that this Hue service instance depends on", - "display_name": "Impala Service", - "name": "impala_service", - "value": null - }, - { - "desc": "Name of Hue database.", - "display_name": "Hue Database Name", - "name": "database_name", - "value": "hue" - }, - { - "desc": "The attribute of the group object which identifies the members of the group.", - "display_name": "LDAP Group Membership Attribute", - "name": "group_member_attr", - "value": null - }, - { - "desc": "Name of the Hive service that this Hue service instance depends on", - "display_name": "Hive Service", - "name": "hive_service", - "value": null - }, - { - "desc": "The username attribute in the LDAP schema.", - "display_name": "LDAP Username Attribute", - "name": "user_name_attr", - "value": null - }, - { - "desc": "Default encoding for site data.", - "display_name": "Default Site Encoding", - "name": "default_site_encoding", - "value": "utf" - }, - { - "desc": "Search Bind Authentication connects to the LDAP server using credentials provided in the 'bind_dn' and 'bind_password' configurations. If these configurations are not set, then an anonymous search is performed.", - "display_name": "Use Search Bind Authentication", - "name": "search_bind_authentication", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into hue_safety_valve.ini. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Hue Service Advanced Configuration Snippet (Safety Valve) for hue_safety_valve.ini", - "name": "hue_service_safety_valve", - "value": null - }, - { - "desc": "Port on host where the Hue database is running. Not necessary for SQLite3.", - "display_name": "Hue Database Port", - "name": "database_port", - "value": "3306" - }, - { - "desc": "Password for Hue database. Not necessary for SQLite3.", - "display_name": "Hue Database Password", - "name": "database_password", - "value": "" - }, - { - "desc": "Base filter for searching for users.", - "display_name": "LDAP User Filter", - "name": "user_filter", - "value": null - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "Name of host where the Hue database is running. Not necessary for SQLite3.", - "display_name": "Hue Database Hostname", - "name": "database_host", - "value": "localhost" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/mapred-site.xml b/sahara/plugins/cdh/v5/resources/mapred-site.xml deleted file mode 100644 index 5228ed17..00000000 --- a/sahara/plugins/cdh/v5/resources/mapred-site.xml +++ /dev/null @@ -1,157 +0,0 @@ - - - -The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:red\",\n \"streamThreshold\": 0}, \"enabled\": \"true\"]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the Hue user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", - "display_name": "Hue Proxy User Hosts", - "name": "hue_proxy_user_hosts_list", - "value": "*" - }, - { - "desc": "The admin port Oozie server runs.", - "display_name": "Oozie Admin Port", - "name": "oozie_admin_port", - "value": "11001" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "If true, enables the Oozie Server web console. ExtJS 2.2 zip archive must be extracted to /var/lib/oozie on the same host as the Oozie Server.", - "display_name": "Enable Oozie Server Web Console", - "name": "oozie_web_console", - "value": "false" - }, - { - "desc": "Username for connecting to the database used by Oozie Server. Does not apply if you are using Derby as the database type.", - "display_name": "Oozie Server Database User", - "name": "oozie_database_user", - "value": "sa" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "Comma-separated list of Oozie plug-ins to be activated. If one plugin cannot be loaded, all the plugins are ignored.", - "display_name": "Oozie Server Plugins", - "name": "oozie_plugins_list", - "value": "" - }, - { - "desc": "Directory where the Oozie Server will place its data. Only applicable when using Derby as the database type.", - "display_name": "Oozie Server Data Directory", - "name": "oozie_data_dir", - "value": "/var/lib/oozie/data" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "oozie_server_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Oozie Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Location of the keystore file on the local file system.", - "display_name": "Oozie SSL Keystore File", - "name": "oozie_https_keystore_file", - "value": "/var/lib/oozie/.keystore" - }, - { - "desc": "The from address to be used for mailing all emails for Oozie email action", - "display_name": "Oozie Email Action From Address", - "name": "oozie_email_from_address", - "value": "oozie@localhost" - }, - { - "desc": "Comma-separated list of SchemaService workflow extension schemas for additional action types.", - "display_name": "Oozie SchemaService Workflow Extension Schemas", - "name": "oozie_workflow_extension_schemas", - "value": "hive-action-0.2.xsd,sqoop-action-0.2.xsd,email-action-0.1.xsd,distcp-action-0.1.xsd,shell-action-0.1.xsd,ssh-action-0.1.xsd,distcp-action-0.2.xsd,hive-action-0.3.xsd,hive-action-0.4.xsd,hive-action-0.5.xsd,oozie-sla-0.1.xsd,oozie-sla-0.2.xsd,sqoop-action-0.3.xsd,sqoop-action-0.4.xsd,shell-action-0.2.xsd,shell-action-0.3.xsd,ssh-action-0.2.xsd" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The SMTP server port to use for Oozie email action", - "display_name": "Oozie Email Action SMTP Port", - "name": "oozie_email_smtp_prt", - "value": "25" - }, - { - "desc": "Port of the Oozie Server while using SSL.", - "display_name": "Oozie HTTPS Port", - "name": "oozie_https_port", - "value": "11443" - }, - { - "desc": "The maximum number of rolled log files to keep for Oozie Server logs. Typically used by log4j.", - "display_name": "Oozie Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "720" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Name of the database used by Oozie Server.", - "display_name": "Oozie Server Database Name", - "name": "oozie_database_name", - "value": "oozie" - }, - { - "desc": "Directory where Oozie Server will place its log files.", - "display_name": "Oozie Server Log Directory", - "name": "oozie_log_dir", - "value": "/var/log/oozie" - }, - { - "desc": "Maximum size for the Java Process heap memory. Passed to Java -Xmx. Measured in bytes.", - "display_name": "Java Heap Size of Oozie Server in Bytes", - "name": "oozie_java_heapsize", - "value": "1073741824" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Oozie Server", - "name": "oozie_java_opts", - "value": "" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Comma-separated list of ActionService executor extension classes. Only action types with associated executors can be used in workflows.", - "display_name": "Oozie ActionService Executor Extension Classes", - "name": "oozie_executor_extension_classes", - "value": "org.apache.oozie.action.hadoop.HiveActionExecutor,org.apache.oozie.action.hadoop.SqoopActionExecutor,org.apache.oozie.action.email.EmailActionExecutor,org.apache.oozie.action.hadoop.ShellActionExecutor,org.apache.oozie.action.hadoop.DistcpActionExecutor" - }, - { - "desc": "Type of the database used by Oozie Server.", - "display_name": "Oozie Server Database Type", - "name": "oozie_database_type", - "value": "derby" - }, - { - "desc": "Maximum callable queue size", - "display_name": "Maximum Callable Queue Size", - "name": "oozie_service_callablequeueservice_queue_size", - "value": "10000" - }, - { - "desc": "Port of Oozie Server", - "display_name": "Oozie HTTP Port", - "name": "oozie_http_port", - "value": "11000" - }, - { - "desc": "For advanced use only, a string to be inserted into oozie-site.xml for this role only.", - "display_name": "Oozie Server Advanced Configuration Snippet (Safety Valve) for oozie-site.xml", - "name": "oozie_config_safety_valve", - "value": null - }, - { - "desc": "Password for the keystore.", - "display_name": "Oozie SSL Keystore Password", - "name": "oozie_https_keystore_password", - "value": null - }, - { - "desc": "The SMTP server host to use for Oozie email action", - "display_name": "Oozie Email Action SMTP Host", - "name": "oozie_email_smtp_host", - "value": "localhost" - }, - { - "desc": "The minimum log level for Oozie Server logs", - "display_name": "Oozie Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Hostname of the database used by Oozie Server. If the port is non-default for your database type, use host:port notation. Does not apply if you are using Derby as the database type.", - "display_name": "Oozie Server Database Host", - "name": "oozie_database_host", - "value": "localhost" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/oozie-service.json b/sahara/plugins/cdh/v5/resources/oozie-service.json deleted file mode 100644 index 456041f7..00000000 --- a/sahara/plugins/cdh/v5/resources/oozie-service.json +++ /dev/null @@ -1,98 +0,0 @@ -[ - { - "desc": "Namespace used by this Oozie service in ZooKeeper when High Availability is enabled.", - "display_name": "ZooKeeper Namespace", - "name": "oozie_zookeeper_namespace", - "value": "oozie" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Enable SSL for Oozie. Note:This is supported only from CDH 4.3 onwards.", - "display_name": "Use SSL", - "name": "oozie_use_ssl", - "value": "false" - }, - { - "desc": "The group that this service's processes should run as.", - "display_name": "System Group", - "name": "process_groupname", - "value": "oozie" - }, - { - "desc": "Use ACLs on Znode while a secure ZooKeeper is used for Oozie High Availability. Note: This config is not emitted if ZooKeeper is not secure.", - "display_name": "Use ACLs on Znode", - "name": "oozie_zk_secure", - "value": "true" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Oozie Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "oozie_env_safety_valve", - "value": null - }, - { - "desc": "Name of the ZooKeeper service that this Oozie service instance depends on", - "display_name": "ZooKeeper Service", - "name": "zookeeper_service", - "value": null - }, - { - "desc": "A list of credential class mappings for CredentialsProvider.", - "display_name": "Oozie Credential Classes", - "name": "oozie_credential_classes", - "value": "hcat=org.apache.oozie.action.hadoop.HCatCredentials,hbase=org.apache.oozie.action.hadoop.HbaseCredentials,hive2=org.apache.oozie.action.hadoop.Hive2Credentials" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Address of the load balancer used if Oozie HA is enabled. Should be specified in host:port format.", - "display_name": "Oozie Load Balancer", - "name": "oozie_load_balancer", - "value": null - }, - { - "desc": "Coordinator Job Lookup trigger command is scheduled at this interval (in seconds).", - "display_name": "Coordinator Job Lookup Interval", - "name": "oozie_service_coord_lookup_interval", - "value": "300" - }, - { - "desc": "The user that this service's processes should run as.", - "display_name": "System User", - "name": "process_username", - "value": "oozie" - }, - { - "desc": "The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:red\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "The health test thresholds of the overall Oozie Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Oozie Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Oozie Servers falls below the critical threshold.", - "display_name": "Healthy Oozie Server Monitoring Thresholds", - "name": "oozie_servers_healthy_thresholds", - "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" - }, - { - "desc": "Service to run MapReduce jobs against", - "display_name": "MapReduce Service", - "name": "mapreduce_yarn_service", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/priority-one-confs.json b/sahara/plugins/cdh/v5/resources/priority-one-confs.json deleted file mode 100644 index f7c77953..00000000 --- a/sahara/plugins/cdh/v5/resources/priority-one-confs.json +++ /dev/null @@ -1,45 +0,0 @@ -[ - "dfs_block_size", - "dfs_umaskmode", - "dfs_webhdfs_enabled", - "dfs_permissions", - "dfs_replication", - "io_compression_codecs", - "io_sort_mb", - "dfs_datanode_du_reserved", - "dfs_datanode_failed_volumes_tolerated", - "dfs_name_dir_restore", - "fs_trash_interval", - "dfs_safemode_min_datanodes", - "dfs_safemode_extension", - "dfs_access_time_precision", - "yarn_acl_enable", - "yarn_admin_acl", - "yarn_log_aggregation_enable", - "yarn_log_aggregation_retain_seconds", - "mapreduce_jobhistory_max_age_ms", - "mapreduce_jobhistory_cleaner_interval", - "mapreduce_reduce_memory_mb", - "mapreduce_reduce_java_opts", - "mapreduce_map_memory_mb", - "mapreduce_map_java_opts", - "yarn_nodemanager_container_manager_thread_count", - "yarn_nodemanager_delete_thread_count", - "yarn_nodemanager_heartbeat_interval_ms", - "yarn_nodemanager_localizer_cache_cleanup_interval_ms", - "yarn_nodemanager_localizer_client_thread_count", - "yarn_nodemanager_localizer_cache_target_size_mb", - "yarn_nodemanager_localizer_fetch_thread_count", - "yarn_nodemanager_log_retain_seconds", - "yarn_nodemanager_resource_memory_mb", - "yarn_resourcemanager_client_thread_count", - "yarn_resourcemanager_scheduler_client_thread_count", - "yarn_resourcemanager_admin_client_thread_count", - "yarn_resourcemanager_amliveliness_monitor_interval_ms", - "yarn_am_liveness_monitor_expiry_interval_ms", - "yarn_resourcemanager_am_max_retries", - "yarn_scheduler_minimum_allocation_mb", - "yarn_scheduler_maximum_allocation_mb", - "yarn_app_mapreduce_am_command_opts", - "yarn_app_mapreduce_am_resource_mb" -] diff --git a/sahara/plugins/cdh/v5/resources/spark-history.json b/sahara/plugins/cdh/v5/resources/spark-history.json deleted file mode 100644 index 73711a75..00000000 --- a/sahara/plugins/cdh/v5/resources/spark-history.json +++ /dev/null @@ -1,146 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "History Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "SPARK_YARN_HISTORY_SERVER_role_env_safety_valve", - "value": null - }, - { - "desc": "The port of the history server WebUI", - "display_name": "History Server WebUI Port", - "name": "history_server_web_port", - "value": "18088" - }, - { - "desc": "The maximum size, in megabytes, per log file for History Server logs. Typically used by log4j.", - "display_name": "History Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "spark_yarn_history_server_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The log directory for log files of the role History Server.", - "display_name": "History Server Log Directory", - "name": "log_dir", - "value": "/var/log/spark" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "History Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "Maximum size for the Java process heap memory. Passed to Java -Xmx. Measured in bytes.", - "display_name": "Java Heap Size of History Server in Bytes", - "name": "history_server_max_heapsize", - "value": "268435456" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "When computing the overall History Server health, consider the host's health.", - "display_name": "History Server Host Health Test", - "name": "spark_yarn_history_server_host_health_enabled", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum number of rolled log files to keep for History Server logs. Typically used by log4j.", - "display_name": "History Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Enables the health test that the History Server's process state is consistent with the role configuration", - "display_name": "History Server Process Health Test", - "name": "spark_yarn_history_server_scm_health_enabled", - "value": "true" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "The minimum log level for History Server logs", - "display_name": "History Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/spark-service.json b/sahara/plugins/cdh/v5/resources/spark-service.json deleted file mode 100644 index 526b6201..00000000 --- a/sahara/plugins/cdh/v5/resources/spark-service.json +++ /dev/null @@ -1,62 +0,0 @@ -[ - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The location of the Spark jar in HDFS", - "display_name": "Spark Jar Location (HDFS)", - "name": "spark_jar_hdfs_path", - "value": "/user/spark/share/lib/spark-assembly.jar" - }, - { - "desc": "The group that this service's processes should run as.", - "display_name": "System Group", - "name": "process_groupname", - "value": "spark" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Spark Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "SPARK_ON_YARN_service_env_safety_valve", - "value": null - }, - { - "desc": "The location of Spark application history logs in HDFS. Changing this value will not move existing logs to the new location.", - "display_name": "Spark History Location (HDFS)", - "name": "spark_history_log_dir", - "value": "/user/spark/applicationHistory" - }, - { - "desc": "Name of the YARN (MR2 Included) service that this Spark service instance depends on", - "display_name": "YARN (MR2 Included) Service", - "name": "yarn_service", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The user that this service's processes should run as.", - "display_name": "System User", - "name": "process_username", - "value": "spark" - }, - { - "desc": "The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/yarn-gateway.json b/sahara/plugins/cdh/v5/resources/yarn-gateway.json deleted file mode 100644 index 0f36fdbd..00000000 --- a/sahara/plugins/cdh/v5/resources/yarn-gateway.json +++ /dev/null @@ -1,308 +0,0 @@ -[ - { - "desc": "For MapReduce job outputs that are compressed, specify the compression codec to use. Will be part of generated client configuration.", - "display_name": "Compression Codec of MapReduce Job Output", - "name": "mapred_output_compression_codec", - "value": "org.apache.hadoop.io.compress.DefaultCodec" - }, - { - "desc": "The default number of parallel transfers run by reduce during the copy (shuffle) phase. This number should be between sqrt(nodes*number_of_map_slots_per_node) and nodes*number_of_map_slots_per_node/2. Will be part of generated client configuration.", - "display_name": "Default Number of Parallel Transfers During Shuffle", - "name": "mapred_reduce_parallel_copies", - "value": "10" - }, - { - "desc": "The number of streams to merge at the same time while sorting files. That is, the number of sort heads to use during the merge sort on the reducer side. This determines the number of open file handles. Merging more files in parallel reduces merge sort iterations and improves run time by eliminating disk I/O. Note that merging more files in parallel uses more memory. If 'io.sort.factor' is set too high or the maximum JVM heap is set too low, excessive garbage collection will occur. The Hadoop default is 10, but Cloudera recommends a higher value. Will be part of generated client configuration.", - "display_name": "I/O Sort Factor", - "name": "io_sort_factor", - "value": "64" - }, - { - "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", - "display_name": "Alternatives Priority", - "name": "client_config_priority", - "value": "92" - }, - { - "desc": "Limit on the number of counters allowed per job.", - "display_name": "Job Counters Limit", - "name": "mapreduce_job_counters_limit", - "value": "120" - }, - { - "desc": "The number of virtual CPU cores allocated for each map task of a job. This parameter has no effect prior to CDH 4.4.", - "display_name": "Map Task CPU Virtual Cores", - "name": "mapreduce_map_cpu_vcores", - "value": "1" - }, - { - "desc": "If enabled, multiple instances of some reduce tasks may be executed in parallel.", - "display_name": "Reduce Tasks Speculative Execution", - "name": "mapred_reduce_tasks_speculative_execution", - "value": "false" - }, - { - "desc": "Threshold for number of reduces, beyond which a job is considered too big for the ubertask optimization. Note: as of CDH5.0.0, MR2 cannot support more than one reduce in an ubertask. (Zero is a valid max.)", - "display_name": "Ubertask Maximum Reduces", - "name": "mapreduce_job_ubertask_maxreduces", - "value": "1" - }, - { - "desc": "The application framework to run jobs with. If not set, jobs will be run with the local job runner.", - "display_name": "Application Framework", - "name": "mapreduce_framework_name", - "value": "yarn" - }, - { - "desc": "Base sleep time between failover attempts. Used only if RM HA is enabled.", - "display_name": "Client Failover Sleep Base Time", - "name": "client_failover_sleep_base", - "value": "100" - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for yarn-site.xml.", - "display_name": "YARN Client Advanced Configuration Snippet (Safety Valve) for yarn-site.xml", - "name": "yarn_client_config_safety_valve", - "value": null - }, - { - "desc": "Compression level for the codec used to compress MapReduce outputs. Default compression is a balance between speed and compression ratio.", - "display_name": "Compression Level of Codecs", - "name": "zlib_compress_level", - "value": "DEFAULT_COMPRESSION" - }, - { - "desc": "Size of buffer for read and write operations of SequenceFiles.", - "display_name": "SequenceFile I/O Buffer Size", - "name": "io_file_buffer_size", - "value": "65536" - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for mapred-site.xml.", - "display_name": "MapReduce Client Advanced Configuration Snippet (Safety Valve) for mapred-site.xml", - "name": "mapreduce_client_config_safety_valve", - "value": null - }, - { - "desc": "The maximum heap size, in bytes, of the Java MapReduce ApplicationMaster. This number will be formatted and concatenated with 'ApplicationMaster Java Opts Base' to pass to Hadoop.", - "display_name": "ApplicationMaster Java Maximum Heap Size", - "name": "yarn_app_mapreduce_am_max_heap", - "value": "825955249" - }, - { - "desc": "The replication level for submitted job files.", - "display_name": "Mapreduce Submit Replication", - "name": "mapred_submit_replication", - "value": "10" - }, - { - "desc": "The total amount of memory buffer, in megabytes, to use while sorting files. Note that this memory comes out of the user JVM heap size (meaning total user JVM heap - this amount of memory = total user usable heap space. Note that Cloudera's default differs from Hadoop's default; Cloudera uses a bigger buffer by default because modern machines often have more RAM. The smallest value across all TaskTrackers will be part of generated client configuration.", - "display_name": "I/O Sort Memory Buffer (MiB)", - "name": "io_sort_mb", - "value": "256" - }, - { - "desc": "Whether to enable the small-jobs \"ubertask\" optimization, which runs \"sufficiently small\" jobs sequentially within a single JVM. \"Small\" is defined by the mapreduce.job.ubertask.maxmaps, mapreduce.job.ubertask.maxreduces, and mapreduce.job.ubertask.maxbytes settings.", - "display_name": "Enable Ubertask Optimization", - "name": "mapreduce_job_ubertask_enabled", - "value": "false" - }, - { - "desc": "Java command line arguments passed to the MapReduce ApplicationMaster.", - "display_name": "ApplicationMaster Java Opts Base", - "name": "yarn_app_mapreduce_am_command_opts", - "value": "-Djava.net.preferIPv4Stack=true" - }, - { - "desc": "The amount of physical memory, in MiB, allocated for each reduce task of a job. This parameter has no effect prior to CDH 4.4.", - "display_name": "Reduce Task Memory", - "name": "mapreduce_reduce_memory_mb", - "value": "1024" - }, - { - "desc": "The maximum permissible size of the split metainfo file. The JobTracker won't attempt to read split metainfo files bigger than the configured value. No limits if set to -1.", - "display_name": "JobTracker MetaInfo Maxsize", - "name": "mapreduce_jobtracker_split_metainfo_maxsize", - "value": "10000000" - }, - { - "desc": "Fraction of the number of map tasks in the job which should be completed before reduce tasks are scheduled for the job.", - "display_name": "Number of Map Tasks to Complete Before Reduce Tasks", - "name": "mapred_reduce_slowstart_completed_maps", - "value": "0.8" - }, - { - "desc": "These are Java command line arguments. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Client Java Configuration Options", - "name": "mapreduce_client_java_opts", - "value": "-Djava.net.preferIPv4Stack=true" - }, - { - "desc": "The physical memory requirement, in MiB, for the ApplicationMaster.", - "display_name": "ApplicationMaster Memory", - "name": "yarn_app_mapreduce_am_resource_mb", - "value": "1024" - }, - { - "desc": "The maximum Java heap size, in bytes, of the reduce processes. This number will be formatted and concatenated with 'Reduce Task Java Opts Base' to pass to Hadoop.", - "display_name": "Reduce Task Maximum Heap Size", - "name": "mapreduce_reduce_java_opts_max_heap", - "value": "825955249" - }, - { - "desc": "Maximum size for the Java process heap memory. Passed to Java -Xmx. Measured in bytes.", - "display_name": "Client Java Heap Size in Bytes", - "name": "mapreduce_client_java_heapsize", - "value": "825955249" - }, - { - "desc": "The number of milliseconds before a task will be terminated if it neither reads an input, writes an output, nor updates its status string.", - "display_name": "Mapreduce Task Timeout", - "name": "mapred_task_timeout", - "value": "600000" - }, - { - "desc": "The virtual CPU cores requirement, for the ApplicationMaster. This parameter has no effect prior to CDH 4.4.", - "display_name": "ApplicationMaster Virtual CPU Cores", - "name": "yarn_app_mapreduce_am_resource_cpu_vcores", - "value": "1" - }, - { - "desc": "The amount of physical memory, in MiB, allocated for each map task of a job.", - "display_name": "Map Task Memory", - "name": "mapreduce_map_memory_mb", - "value": "1024" - }, - { - "desc": "The number of virtual CPU cores for each reduce task of a job.", - "display_name": "Reduce Task CPU Virtual Cores", - "name": "mapreduce_reduce_cpu_vcores", - "value": "1" - }, - { - "desc": "For MapReduce map outputs that are compressed, specify the compression codec to use. Will be part of generated client configuration.", - "display_name": "Compression Codec of MapReduce Map Output", - "name": "mapred_map_output_compression_codec", - "value": "org.apache.hadoop.io.compress.SnappyCodec" - }, - { - "desc": "Classpaths to include for MapReduce applications.", - "display_name": "MR Application Classpath", - "name": "mapreduce_application_classpath", - "value": "$HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$MR2_CLASSPATH" - }, - { - "desc": "When set, each role will identify important log events and forward them to Cloudera Manager.", - "display_name": "Enable Log Event Capture", - "name": "catch_events", - "value": "true" - }, - { - "desc": "Maximum sleep time between failover attempts. Used only if RM HA is enabled.", - "display_name": "Client Failover Sleep Max Time", - "name": "client_failover_sleep_max", - "value": "2000" - }, - { - "desc": "Location to store the job history files of running jobs. This is a path on the host where the JobTracker is running.", - "display_name": "Running Job History Location", - "name": "hadoop_job_history_dir", - "value": "/var/log/hadoop-mapreduce/history" - }, - { - "desc": "Compress the output of MapReduce jobs. Will be part of generated client configuration.", - "display_name": "Compress MapReduce Job Output", - "name": "mapred_output_compress", - "value": "false" - }, - { - "desc": "For MapReduce job outputs that are compressed as SequenceFiles, you can select one of these compression type options: NONE, RECORD or BLOCK. Cloudera recommends BLOCK. Will be part of generated client configuration.", - "display_name": "Compression Type of MapReduce Job Output", - "name": "mapred_output_compression_type", - "value": "BLOCK" - }, - { - "desc": "Java opts for the map processes. The following symbol, if present, will be interpolated: @taskid@ is replaced by current TaskID. Any other occurrences of '@' will go unchanged. For example, to enable verbose gc logging to a file named for the taskid in /tmp pass a value of: \"-verbose:gc -Xloggc:/tmp/@taskid@.gc\". The configuration variable 'Map Task Memory' can be used to control the maximum memory of the map processes.", - "display_name": "Map Task Java Opts Base", - "name": "mapreduce_map_java_opts", - "value": "-Djava.net.preferIPv4Stack=true" - }, - { - "desc": "Additional execution environment entries for map and reduce task processes.", - "display_name": "MR Application Environment", - "name": "mapreduce_admin_user_env", - "value": "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH" - }, - { - "desc": "The soft limit in either the buffer or record collection buffers. When this limit is reached, a thread will begin to spill the contents to disk in the background. Note that this does not imply any chunking of data to the spill. A value less than 0.5 is not recommended. The syntax is in decimal units; the default is 80% and is formatted 0.8. Will be part of generated client configuration.", - "display_name": "I/O Sort Spill Percent", - "name": "io_sort_spill_percent", - "value": "0.8" - }, - { - "desc": "The default number of reduce tasks per job. Will be part of generated client configuration.", - "display_name": "Default Number of Reduce Tasks per Job", - "name": "mapred_reduce_tasks", - "value": "1" - }, - { - "desc": "Maximum allowed connections for the shuffle. Set to 0 (zero) to indicate no limit on the number of connections.", - "display_name": "Max Shuffle Connections", - "name": "mapreduce_shuffle_max_connections", - "value": "80" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the client configuration for hadoop-env.sh.", - "display_name": "Gateway Client Environment Advanced Configuration Snippet for hadoop-env.sh (Safety Valve)", - "name": "mapreduce_client_env_safety_valve", - "value": null - }, - { - "desc": "The maximum Java heap size, in bytes, of the map processes. This number will be formatted and concatenated with 'Map Task Java Opts Base' to pass to Hadoop.", - "display_name": "Map Task Maximum Heap Size", - "name": "mapreduce_map_java_opts_max_heap", - "value": "825955249" - }, - { - "desc": "A shared directory for temporary files.", - "display_name": "Shared Temp Directories", - "name": "mapreduce_cluster_temp_dir", - "value": null - }, - { - "desc": "Java opts for the reduce processes. The following symbol, if present, will be interpolated: @taskid@ is replaced by current TaskID. Any other occurrences of '@' will go unchanged. For example, to enable verbose gc logging to a file named for the taskid in /tmp pass a value of: \"-verbose:gc -Xloggc:/tmp/@taskid@.gc\". The configuration variable 'Reduce Task Memory' can be used to control the maximum memory of the reduce processes.", - "display_name": "Reduce Task Java Opts Base", - "name": "mapreduce_reduce_java_opts", - "value": "-Djava.net.preferIPv4Stack=true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "If enabled, uses compression on the map outputs before they are sent across the network. Will be part of generated client configuration.", - "display_name": "Use Compression on Map Outputs", - "name": "mapred_compress_map_output", - "value": "true" - }, - { - "desc": "Threshold for number of maps, beyond which a job is considered too big for the ubertask optimization.", - "display_name": "Ubertask Maximum Maps", - "name": "mapreduce_job_ubertask_maxmaps", - "value": "9" - }, - { - "desc": "If enabled, multiple instances of some map tasks may be executed in parallel.", - "display_name": "Map Tasks Speculative Execution", - "name": "mapred_map_tasks_speculative_execution", - "value": "false" - }, - { - "desc": "Threshold for number of input bytes, beyond which a job is considered too big for the ubertask optimization. If no value is specified, dfs.block.size is used as a default.", - "display_name": "Ubertask Maximum Job Size", - "name": "mapreduce_job_ubertask_maxbytes", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/yarn-jobhistory.json b/sahara/plugins/cdh/v5/resources/yarn-jobhistory.json deleted file mode 100644 index f2db9cd4..00000000 --- a/sahara/plugins/cdh/v5/resources/yarn-jobhistory.json +++ /dev/null @@ -1,248 +0,0 @@ -[ - { - "desc": "Maximum size for the Java Process heap memory. Passed to Java -Xmx. Measured in bytes.", - "display_name": "Java Heap Size of JobHistory Server in Bytes", - "name": "mr2_jobhistory_java_heapsize", - "value": "1073741824" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "jobhistory_gc_duration_window", - "value": "5" - }, - { - "desc": "The root HDFS directory of the staging area for users' MR2 jobs; for example /user. The staging directories are always named after the user.", - "display_name": "MapReduce ApplicationMaster Staging Root Directory", - "name": "yarn_app_mapreduce_am_staging_dir", - "value": "/user" - }, - { - "desc": "Enables the health test that the JobHistory Server's process state is consistent with the role configuration", - "display_name": "JobHistory Server Process Health Test", - "name": "jobhistory_scm_health_enabled", - "value": "true" - }, - { - "desc": "When computing the overall JobHistory Server health, consider the host's health.", - "display_name": "JobHistory Server Host Health Test", - "name": "jobhistory_host_health_enabled", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for JobHistory Server logs. Typically used by log4j.", - "display_name": "JobHistory Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:red\",\n \"streamThreshold\": 0}, \"enabled\": \"true\"]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Directory where JobHistory Server will place its log files.", - "display_name": "JobHistory Server Log Directory", - "name": "mr2_jobhistory_log_dir", - "value": "/var/log/hadoop-mapreduce" - }, - { - "desc": "Time interval for history cleaner to check for files to delete. Files are only deleted if they are older than mapreduce.jobhistory.max-age-ms.", - "display_name": "Job History Files Cleaner Interval", - "name": "mapreduce_jobhistory_cleaner_interval", - "value": "86400000" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "jobhistory_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for JobHistory Server", - "name": "mr2_jobhistory_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "For advanced use only, a string to be inserted into yarn-site.xml for this role only.", - "display_name": "JobHistory Server Advanced Configuration Snippet (Safety Valve) for yarn-site.xml", - "name": "jobhistory_config_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "JobHistory Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The port of the MapReduce JobHistory Server. Together with the host name of the JobHistory role forms the address. ", - "display_name": "MapReduce JobHistory Server Port", - "name": "mapreduce_jobhistory_address", - "value": "10020" - }, - { - "desc": "The group that the JobHistory Server process should run as.", - "display_name": "System Group", - "name": "history_process_groupname", - "value": "hadoop" - }, - { - "desc": "Job history files older than this time duration will deleted when the history cleaner runs.", - "display_name": "Job History Files Maximum Age", - "name": "mapreduce_jobhistory_max_age_ms", - "value": "604800000" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for JobHistory Server logs. Typically used by log4j.", - "display_name": "JobHistory Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "jobhistory_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The port of the MapReduce JobHistory Server web application. Together with the host name of the JobHistory role forms the address.", - "display_name": "MapReduce JobHistory Webapp Port", - "name": "mapreduce_jobhistory_webapp_address", - "value": "19888" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "jobhistory_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The minimum log level for JobHistory Server logs", - "display_name": "JobHistory Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "For advanced use only, a string to be inserted into mapred-site.xml for this role only.", - "display_name": "JobHistory Server Advanced Configuration Snippet (Safety Valve) for mapred-site.xml", - "name": "jobhistory_mapred_safety_valve", - "value": null - }, - { - "desc": "If enabled, the JobHistory Server binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind JobHistory Server to Wildcard Address", - "name": "yarn_jobhistory_bind_wildcard", - "value": "false" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The user that the JobHistory Server process should run as.", - "display_name": "System User", - "name": "history_process_username", - "value": "mapred" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "jobhistory_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/yarn-nodemanager.json b/sahara/plugins/cdh/v5/resources/yarn-nodemanager.json deleted file mode 100644 index 4479a92a..00000000 --- a/sahara/plugins/cdh/v5/resources/yarn-nodemanager.json +++ /dev/null @@ -1,380 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "For advanced use only, a string to be inserted into yarn-site.xml for this role only.", - "display_name": "NodeManager Advanced Configuration Snippet (Safety Valve) for yarn-site.xml", - "name": "nodemanager_config_safety_valve", - "value": null - }, - { - "desc": "The address of the NodeManager web application.", - "display_name": "NodeManager Web Application Address", - "name": "nodemanager_webserver_port", - "value": "8042" - }, - { - "desc": "If enabled, adds 'org.apache.hadoop.mapred.ShuffleHandler' to the NodeManager auxiliary services. This is required for MapReduce applications.", - "display_name": "Enable Shuffle Auxiliary Service", - "name": "mapreduce_aux_service", - "value": "true" - }, - { - "desc": "Heartbeat interval to ResourceManager", - "display_name": "Heartbeat Interval", - "name": "yarn_nodemanager_heartbeat_interval_ms", - "value": "1000" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Enables the health test that the NodeManager's process state is consistent with the role configuration", - "display_name": "NodeManager Process Health Test", - "name": "nodemanager_scm_health_enabled", - "value": "true" - }, - { - "desc": "The amount of time to wait for the NodeManager to fully start up and connect to the ResourceManager before enforcing the connectivity check.", - "display_name": "NodeManager Connectivity Tolerance at Startup", - "name": "nodemanager_connectivity_tolerance_seconds", - "value": "180" - }, - { - "desc": "List of directories on the local filesystem where a NodeManager stores intermediate data files.", - "display_name": "NodeManager Local Directory List", - "name": "yarn_nodemanager_local_dirs", - "value": null - }, - { - "desc": "Enables the health check that verifies the NodeManager is connected to the ResourceManager", - "display_name": "NodeManager Connectivity Health Check", - "name": "nodemanager_connectivity_health_enabled", - "value": "true" - }, - { - "desc": "Number of seconds after an application finishes before the NodeManager's DeletionService will delete the application's localized file and log directory. To diagnose YARN application problems, set this property's value large enough (for example, to 600 = 10 minutes) to permit examination of these directories.", - "display_name": "Localized Dir Deletion Delay", - "name": "yarn_nodemanager_delete_debug_delay_sec", - "value": "0" - }, - { - "desc": "The minimum Linux user ID allowed. Used to prevent other super users.", - "display_name": "Minimum User ID", - "name": "container_executor_min_user_id", - "value": "1000" - }, - { - "desc": "Number of threads to use for localization fetching.", - "display_name": "Localizer Fetch Thread Count", - "name": "yarn_nodemanager_localizer_fetch_thread_count", - "value": "4" - }, - { - "desc": "Target size of localizer cache in MB, per local directory.", - "display_name": "Localizer Cache Target Size", - "name": "yarn_nodemanager_localizer_cache_target_size_mb", - "value": "10240" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for NodeManager", - "name": "node_manager_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:red\",\n \"streamThreshold\": 0}, \"enabled\": \"true\"]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "List of users explicitly whitelisted to be allowed to run containers. Users with IDs lower than the \"Minimum User Id\" setting may be whitelisted by using this setting.", - "display_name": "Allowed System Users", - "name": "container_executor_allowed_system_users", - "value": "nobody,impala,hive,llama" - }, - { - "desc": "Directory where NodeManager will place its log files.", - "display_name": "NodeManager Log Directory", - "name": "node_manager_log_dir", - "value": "/var/log/hadoop-yarn" - }, - { - "desc": "The system group that owns the container-executor binary. This does not need to be changed unless the ownership of the binary is explicitly changed.", - "display_name": "Container Executor Group", - "name": "container_executor_group", - "value": "yarn" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "nodemanager_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "nodemanager_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "Environment variables that containers may override rather than use NodeManager's default.", - "display_name": "Containers Environment Variables Whitelist ", - "name": "yarn_nodemanager_env_whitelist", - "value": "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME" - }, - { - "desc": "The address of the NodeManager IPC.", - "display_name": "NodeManager IPC Address", - "name": "yarn_nodemanager_address", - "value": "8041" - }, - { - "desc": "Environment variables that should be forwarded from the NodeManager's environment to the container's.", - "display_name": "Containers Environment Variable", - "name": "yarn_nodemanager_admin_env", - "value": "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "nodemanager_gc_duration_window", - "value": "5" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "NodeManager Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "HDFS directory where application logs are stored when an application completes.", - "display_name": "Remote App Log Directory", - "name": "yarn_nodemanager_remote_app_log_dir", - "value": "/tmp/logs" - }, - { - "desc": "Time in seconds to retain user logs. Only applicable if log aggregation is disabled.", - "display_name": "Log Retain Duration", - "name": "yarn_nodemanager_log_retain_seconds", - "value": "10800" - }, - { - "desc": "Number of threads used in cleanup.", - "display_name": "Cleanup Thread Count", - "name": "yarn_nodemanager_delete_thread_count", - "value": "4" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "Number of threads container manager uses.", - "display_name": "Container Manager Thread Count", - "name": "yarn_nodemanager_container_manager_thread_count", - "value": "20" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Address where the localizer IPC is.", - "display_name": "Localizer Port", - "name": "yarn_nodemanager_localizer_address", - "value": "8040" - }, - { - "desc": "Address where the localizer IPC is.", - "display_name": "Localizer Cache Cleanup Interval", - "name": "yarn_nodemanager_localizer_cache_cleanup_interval_ms", - "value": "600000" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "Absolute path to the script which is periodically run by the node health monitoring service to determine if the node is healthy or not. If the value of this key is empty or the file does not exist in the location configured here, the node health monitoring service is not started.", - "display_name": "Healthchecker Script Path", - "name": "mapred_healthchecker_script_path", - "value": "" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When computing the overall NodeManager health, consider the host's health.", - "display_name": "NodeManager Host Health Test", - "name": "nodemanager_host_health_enabled", - "value": "true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Path (rooted in the cgroups hierarchy on the machine) where to place YARN-managed cgroups.", - "display_name": "CGroups Hierarchy", - "name": "linux_container_executor_cgroups_hierarchy", - "value": "/hadoop-yarn" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "nodemanager_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "For advanced use only, a string to be inserted into mapred-site.xml for this role only.", - "display_name": "NodeManager Advanced Configuration Snippet (Safety Valve) for mapred-site.xml", - "name": "nodemanager_mapred_safety_valve", - "value": null - }, - { - "desc": "The maximum size, in megabytes, per log file for NodeManager logs. Typically used by log4j.", - "display_name": "NodeManager Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The remote log dir will be created at {yarn.nodemanager.remote-app-log-dir}/${user}/{thisParam}", - "display_name": "Remote App Log Directory Suffix", - "name": "yarn_nodemanager_remote_app_log_dir_suffix", - "value": "logs" - }, - { - "desc": "Number of threads to handle localization requests.", - "display_name": "Localizer Client Thread Count", - "name": "yarn_nodemanager_localizer_client_thread_count", - "value": "5" - }, - { - "desc": "The maximum number of rolled log files to keep for NodeManager logs. Typically used by log4j.", - "display_name": "NodeManager Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "Amount of physical memory, in MiB, that can be allocated for containers.", - "display_name": "Container Memory", - "name": "yarn_nodemanager_resource_memory_mb", - "value": "8192" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "nodemanager_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Number of virtual CPU cores that can be allocated for containers. This parameter has no effect prior to CDH 4.4.", - "display_name": "Container Virtual CPU Cores", - "name": "yarn_nodemanager_resource_cpu_vcores", - "value": "8" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "Comma-separated list of arguments which are to be passed to node health script when it is being launched.", - "display_name": "Healthchecker Script Arguments", - "name": "mapred_healthchecker_script_args", - "value": "" - }, - { - "desc": "List of directories on the local filesystem where a NodeManager stores container log files.", - "display_name": "NodeManager Container Log Directories", - "name": "yarn_nodemanager_log_dirs", - "value": "/var/log/hadoop-yarn/container" - }, - { - "desc": "The minimum log level for NodeManager logs", - "display_name": "NodeManager Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Maximum size for the Java Process heap memory. Passed to Java -Xmx. Measured in bytes.", - "display_name": "Java Heap Size of NodeManager in Bytes", - "name": "node_manager_java_heapsize", - "value": "1073741824" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/yarn-resourcemanager.json b/sahara/plugins/cdh/v5/resources/yarn-resourcemanager.json deleted file mode 100644 index 9513a623..00000000 --- a/sahara/plugins/cdh/v5/resources/yarn-resourcemanager.json +++ /dev/null @@ -1,440 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "resourcemanager_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "Enter an XML string that will be inserted verbatim into the Fair Scheduler allocations file. For CDH5, overrides the configuration set using the Pools configuration UI. For CDH4, this is the only way to configure the Fair Scheduler for YARN.", - "display_name": "Fair Scheduler XML Advanced Configuration Snippet (Safety Valve)", - "name": "resourcemanager_fair_scheduler_configuration", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into nodes_allow.txt for this role only.", - "display_name": "ResourceManager Advanced Configuration Snippet (Safety Valve) for nodes_allow.txt", - "name": "rm_hosts_allow_safety_valve", - "value": null - }, - { - "desc": "The maximum number that an ApplicationMaster will retry.", - "display_name": "ApplicationMaster Max Retries", - "name": "yarn_resourcemanager_am_max_retries", - "value": "1" - }, - { - "desc": "For advanced use only, a string to be inserted into nodes_exclude.txt for this role only.", - "display_name": "ResourceManager Advanced Configuration Snippet (Safety Valve) for nodes_exclude.txt", - "name": "rm_hosts_exclude_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "resourcemanager_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The maximum number of completed applications that the ResourceManager keeps.", - "display_name": "Max Completed Applications", - "name": "yarn_resourcemanager_max_completed_applications", - "value": "10000" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "For advanced use only, a string to be inserted into yarn-site.xml for this role only.", - "display_name": "ResourceManager Advanced Configuration Snippet (Safety Valve) for yarn-site.xml", - "name": "resourcemanager_config_safety_valve", - "value": null - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for ResourceManager", - "name": "resource_manager_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "Enables the health test that the ResourceManager's process state is consistent with the role configuration", - "display_name": "ResourceManager Process Health Test", - "name": "resourcemanager_scm_health_enabled", - "value": "true" - }, - { - "desc": "Directory where ResourceManager will place its log files.", - "display_name": "ResourceManager Log Directory", - "name": "resource_manager_log_dir", - "value": "/var/log/hadoop-yarn" - }, - { - "desc": "The smallest amount of physical memory, in MiB, that can be requested for a container. If using the Capacity or FIFO scheduler (or any scheduler, prior to CDH 5), memory requests will be rounded up to the nearest multiple of this number.", - "display_name": "Container Memory Minimum", - "name": "yarn_scheduler_minimum_allocation_mb", - "value": "1024" - }, - { - "desc": "The number of threads used to handle applications manager requests.", - "display_name": "Client Thread Count", - "name": "yarn_resourcemanager_client_thread_count", - "value": "50" - }, - { - "desc": "The periodic interval that the ResourceManager will check whether containers are still alive.", - "display_name": "Container Monitor Interval", - "name": "yarn_resourcemanager_container_liveness_monitor_interval_ms", - "value": "600000" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The expiry interval to wait until a NodeManager is considered dead.", - "display_name": "NodeManager Monitor Expiry", - "name": "yarn_nm_liveness_monitor_expiry_interval_ms", - "value": "600000" - }, - { - "desc": "The largest amount of physical memory, in MiB, that can be requested for a container.", - "display_name": "Container Memory Maximum", - "name": "yarn_scheduler_maximum_allocation_mb", - "value": "65536" - }, - { - "desc": "The largest number of virtual CPU cores that can be requested for a container. This parameter has no effect prior to CDH 4.4.", - "display_name": "Container Virtual CPU Cores Maximum", - "name": "yarn_scheduler_maximum_allocation_vcores", - "value": "32" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "resourcemanager_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "Number of threads to handle resource tracker calls.", - "display_name": "Resource Tracker Thread Count", - "name": "yarn_resourcemanager_resource_tracker_client_thread_count", - "value": "50" - }, - { - "desc": "If using the Fair Scheduler, virtual core requests will be rounded up to the nearest multiple of this number. This parameter has no effect prior to CDH 5.", - "display_name": "Container Virtual CPU Cores Increment", - "name": "yarn_scheduler_increment_allocation_vcores", - "value": "1" - }, - { - "desc": "The address of the scheduler interface in the ResourceManager.", - "display_name": "Scheduler Address", - "name": "yarn_resourcemanager_scheduler_address", - "value": "8030" - }, - { - "desc": "For advanced use only, a string to be inserted into mapred-site.xml for this role only.", - "display_name": "ResourceManager Advanced Configuration Snippet (Safety Valve) for mapred-site.xml", - "name": "resourcemanager_mapred_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The address of the ResourceManager web application.", - "display_name": "Web Application Address", - "name": "resourcemanager_webserver_port", - "value": "8088" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "resourcemanager_gc_duration_window", - "value": "5" - }, - { - "desc": "For applications that request containers on particular racks, the number of scheduling opportunities since the last container assignment to wait before accepting a placement on another rack. Expressed as a float between 0 and 1, which, as a fraction of the cluster size, is the number of scheduling opportunities to pass up. If not set, this means don't pass up any scheduling opportunities. Requires Fair Scheduler continuous scheduling to be disabled. If continuous scheduling is enabled, yarn.scheduler.fair.locality-delay-rack-ms should be used instead.", - "display_name": "Fair Scheduler Rack Locality Threshold", - "name": "resourcemanager_fair_scheduler_locality_threshold_rack", - "value": null - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "The address of the applications manager interface in the ResourceManager.", - "display_name": "ResourceManager Address", - "name": "yarn_resourcemanager_address", - "value": "8032" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "ResourceManager Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "Enables multiple Fair Scheduler container assignments in one heartbeat, which improves cluster throughput when there are many small tasks to run.", - "display_name": "Fair Scheduler Assign Multiple Tasks", - "name": "resourcemanager_fair_scheduler_assign_multiple", - "value": "false" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "resourcemanager_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "When enabled, any applications that were running on the cluster when the ResourceManager died will be recovered when the ResourceManager next starts. Note: If RM-HA is enabled, then this configuration is always enabled.", - "display_name": "Enable ResourceManager Recovery", - "name": "yarn_resourcemanager_recovery_enabled", - "value": "false" - }, - { - "desc": "The periodic interval that the ResourceManager will check whether ApplicationMasters is still alive.", - "display_name": "ApplicationMaster Monitor Interval", - "name": "yarn_resourcemanager_amliveliness_monitor_interval_ms", - "value": "1000" - }, - { - "desc": "For applications that request containers on particular nodes, the minimum time in milliseconds the Fair Scheduler waits before accepting a placement on another node. Requires Fair Scheduler continuous scheduling to be enabled. If continuous scheduling is disabled, yarn.scheduler.fair.locality.threshold.node should be used instead.", - "display_name": "Fair Scheduler Node Locality Delay", - "name": "yarn_scheduler_fair_locality_delay_node_ms", - "value": "2000" - }, - { - "desc": "Number of threads used to handle the ResourceManager admin interface.", - "display_name": "Admin Client Thread Count", - "name": "yarn_resourcemanager_admin_client_thread_count", - "value": "1" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Maximum size for the Java Process heap memory. Passed to Java -Xmx. Measured in bytes.", - "display_name": "Java Heap Size of ResourceManager in Bytes", - "name": "resource_manager_java_heapsize", - "value": "1073741824" - }, - { - "desc": "The maximum size, in megabytes, per log file for ResourceManager logs. Typically used by log4j.", - "display_name": "ResourceManager Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When enabled, if a pool's minimum share is not met for some period of time, the Fair Scheduler preempts applications in other pools. Preemption guarantees that production applications are not starved while also allowing the cluster to be used for experimental and research applications. To minimize wasted computation, the Fair Scheduler preempts the most recently launched applications.", - "display_name": "Fair Scheduler Preemption", - "name": "resourcemanager_fair_scheduler_preemption", - "value": "false" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The smallest number of virtual CPU cores that can be requested for a container. If using the Capacity or FIFO scheduler (or any scheduler, prior to CDH 5), virtual core requests will be rounded up to the nearest multiple of this number. This parameter has no effect prior to CDH 4.4.", - "display_name": "Container Virtual CPU Cores Minimum", - "name": "yarn_scheduler_minimum_allocation_vcores", - "value": "1" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "If using the Fair Scheduler, memory requests will be rounded up to the nearest multiple of this number. This parameter has no effect prior to CDH 5.", - "display_name": "Container Memory Increment", - "name": "yarn_scheduler_increment_allocation_mb", - "value": "512" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:red\",\n \"streamThreshold\": 0}, \"enabled\": \"true\"]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "If enabled, the ResourceManager binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind ResourceManager to Wildcard Address", - "name": "yarn_rm_bind_wildcard", - "value": "false" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Enable continuous scheduling in the Fair Scheduler. When enabled, scheduling decisions are decoupled from NodeManager heartbeats, leading to faster resource allocations.", - "display_name": "Enable Fair Scheduler Continuous Scheduling", - "name": "yarn_scheduler_fair_continuous_scheduling_enabled", - "value": "true" - }, - { - "desc": "When computing the overall ResourceManager health, consider the host's health.", - "display_name": "ResourceManager Host Health Test", - "name": "resourcemanager_host_health_enabled", - "value": "true" - }, - { - "desc": "The number of threads used to handle requests through the scheduler interface.", - "display_name": "Scheduler Thread Count", - "name": "yarn_resourcemanager_scheduler_client_thread_count", - "value": "50" - }, - { - "desc": "The maximum number of rolled log files to keep for ResourceManager logs. Typically used by log4j.", - "display_name": "ResourceManager Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "When enabled, the Fair Scheduler will assign shares to individual apps based on their size, rather than providing an equal share to all apps regardless of size.", - "display_name": "Fair Scheduler Size-Based Weight", - "name": "resourcemanager_fair_scheduler_size_based_weight", - "value": "false" - }, - { - "desc": "For applications that request containers on particular nodes, the number of scheduling opportunities since the last container assignment to wait before accepting a placement on another node. Expressed as a float between 0 and 1, which, as a fraction of the cluster size, is the number of scheduling opportunities to pass up. If not set, this means don't pass up any scheduling opportunities. Requires Fair Scheduler continuous scheduling to be disabled. If continuous scheduling is enabled, yarn.scheduler.fair.locality-delay-node-ms should be used instead.", - "display_name": "Fair Scheduler Node Locality Threshold", - "name": "resourcemanager_fair_scheduler_locality_threshold_node", - "value": null - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "For applications that request containers on particular racks, the minimum time in milliseconds the Fair Scheduler waits before accepting a placement on another rack. Requires Fair Scheduler continuous scheduling to be enabled. If continuous scheduling is disabled, yarn.scheduler.fair.locality.threshold.rack should be used instead.", - "display_name": "Fair Scheduler Rack Locality Delay", - "name": "yarn_scheduler_fair_locality_delay_rack_ms", - "value": "4000" - }, - { - "desc": "When set to true, the Fair Scheduler uses the username as the default pool name, in the event that a pool name is not specified. When set to false, all applications are run in a shared pool, called default.", - "display_name": "Fair Scheduler User As Default Queue", - "name": "resourcemanager_fair_scheduler_user_as_default_queue", - "value": "true" - }, - { - "desc": "The address of the resource tracker interface in the ResourceManager.", - "display_name": "Resource Tracker Address", - "name": "yarn_resourcemanager_resource_tracker_address", - "value": "8031" - }, - { - "desc": "The expiry interval to wait until an ApplicationMaster is considered dead.", - "display_name": "ApplicationMaster Monitor Expiry", - "name": "yarn_am_liveness_monitor_expiry_interval_ms", - "value": "600000" - }, - { - "desc": "The class to use as the resource scheduler. FairScheduler is only supported in CDH 4.2.1 and later.", - "display_name": "Scheduler Class", - "name": "yarn_resourcemanager_scheduler_class", - "value": "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler" - }, - { - "desc": "The minimum log level for ResourceManager logs", - "display_name": "ResourceManager Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", - "display_name": "Health Check Startup Tolerance", - "name": "resourcemanager_startup_tolerance_minutes", - "value": "5" - }, - { - "desc": "The address of the admin interface in the ResourceManager.", - "display_name": "Administration Address", - "name": "yarn_resourcemanager_admin_address", - "value": "8033" - }, - { - "desc": "The periodic interval that the ResourceManager will check whether NodeManagers are still alive.", - "display_name": "NodeManager Monitor Interval", - "name": "yarn_resourcemanager_nm_liveness_monitor_interval_ms", - "value": "1000" - }, - { - "desc": "Enter an XML string that represents the Capacity Scheduler configuration.", - "display_name": "Capacity Scheduler Configuration", - "name": "resourcemanager_capacity_scheduler_configuration", - "value": "\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:red\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "Controls which applications an admin user can see in the applications list view", - "display_name": "Admin Users Applications List Visibility Settings", - "name": "admin_application_list_settings", - "value": "ALL" - }, - { - "desc": "Whether YARN uses the Linux Container Executor both in secure (Kerberos) and insecure (not Kerberos) environments. Cgroups enforcement only works when the Linux Container Executor is used.", - "display_name": "Always Use Linux Container Executor", - "name": "yarn_service_lce_always", - "value": "false" - }, - { - "desc": "When set, each role will identify important log events and forward them to Cloudera Manager.", - "display_name": "Enable Log Event Capture", - "name": "catch_events", - "value": "true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Comma-delimited list of groups that you want to allow the Llama (AM for Impala) user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", - "display_name": "Llama Proxy User Groups", - "name": "llama_proxy_user_groups_list", - "value": "*" - }, - { - "desc": "Controls the aggregate metrics generated for YARN applications. The structure is a JSON list of the attributes to aggregate and the entities to aggregate to. For example, if the attributeName is 'maps_completed' and the aggregationTargets is ['USER'] then the Service Monitor will create the metric 'yarn_application_maps_completed_rate' and, every ten minutes, will record the total maps completed for each user across all their YARN applications. By default it will also record the number of applications submitted ('apps_submitted_rate') for both users and pool. For a full list of the supported attributes see the YARN search page. Note that the valid aggregation targets are USER, YARN_POOL, and YARN (the service), and that these aggregate metrics can be viewed on both the reports and charts search pages.", - "display_name": "YARN Application Aggregates", - "name": "yarn_application_aggregates", - "value": "[\n {\n \"attributeName\": \"maps_total\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"reduces_total\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"cpu_milliseconds\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"mb_millis_maps\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"mb_millis_reduces\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"vcores_millis_maps\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"vcores_millis_reduces\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"file_bytes_read\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"file_bytes_written\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"hdfs_bytes_read\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"hdfs_bytes_written\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n }\n]\n" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the Llama (AM for Impala) user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", - "display_name": "Llama Proxy User Hosts", - "name": "llama_proxy_user_hosts_list", - "value": "*" - }, - { - "desc": "UNIX user that containers run as when Linux-container-executor is used in nonsecure mode.", - "display_name": "UNIX User for Nonsecure Mode with Linux Container Executor", - "name": "yarn_nodemanager_linux_container_executor_nonsecure_mode_local_user", - "value": "nobody" - }, - { - "desc": "For advanced use only, a string to be inserted into mapred-site.xml. Applies to configurations of all roles in this service except client configuration.", - "display_name": "YARN Service MapReduce Advanced Configuration Snippet (Safety Valve)", - "name": "yarn_service_mapred_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "YARN Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "yarn_service_env_safety_valve", - "value": null - }, - { - "desc": "Whether to enable log aggregation", - "display_name": "Enable Log Aggregation", - "name": "yarn_log_aggregation_enable", - "value": "true" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/zookeeper-server.json b/sahara/plugins/cdh/v5/resources/zookeeper-server.json deleted file mode 100644 index 9c5959b2..00000000 --- a/sahara/plugins/cdh/v5/resources/zookeeper-server.json +++ /dev/null @@ -1,350 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "The port to monitor for inter-server communication", - "display_name": "Quorum Port", - "name": "quorumPort", - "value": "3181" - }, - { - "desc": "The period to review when computing the moving average of the outstanding requests queue size. Specified in minutes.", - "display_name": "ZooKeeper Server Outstanding Requests Monitoring Period", - "name": "zookeeper_server_outstanding_requests_window", - "value": "3" - }, - { - "desc": "The port to monitor for leadership election", - "display_name": "Election Port", - "name": "electionPort", - "value": "4181" - }, - { - "desc": "Specifies the name of the user that has read-only privileges when using password file based authentication for JMX access. JMX authentication must be enabled for this setting to take effect.", - "display_name": "Name of User with Read-Only access to the JMX Agent", - "name": "jmx_passwd_file_readonly_user", - "value": "monitorRole" - }, - { - "desc": "The health check thresholds of the weighted average size of the ZooKeeper Server connection count over a recent period. See ZooKeeper Server Connection Count Monitoring Period.", - "display_name": "ZooKeeper Server Connection Count Thresholds", - "name": "zookeeper_server_connection_count_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The disk location that ZooKeeper will use to store its transaction logs.", - "display_name": "Transaction Log Directory", - "name": "dataLogDir", - "value": "/var/lib/zookeeper" - }, - { - "desc": "The port used by the JMX agent to service requests", - "display_name": "JMX Agent Port", - "name": "server_jmx_agent_port", - "value": "9010" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "zookeeper_server_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "The tolerance window that will be used in the detection of a ZooKeeper server's membership in a quorum. Specified in minutes.", - "display_name": "Quorum Membership Detection Window", - "name": "zookeeper_server_quorum_membership_detection_window", - "value": "3" - }, - { - "desc": "Enables the health test that the Server's process state is consistent with the role configuration", - "display_name": "Server Process Health Test", - "name": "zookeeper_server_scm_health_enabled", - "value": "true" - }, - { - "desc": "The address (IPv4, IPv6, or hostname) to monitor for client connections. This is the address that clients attempt to connect to. This setting is optional, because by default, ZooKeeper binds in such a way that any connection to the client port for any address/interface/NIC on the server will be accepted.", - "display_name": "Client Port Address", - "name": "clientPortAddress", - "value": null - }, - { - "desc": "When computing the overall Server health, consider the host's health.", - "display_name": "Server Host Health Test", - "name": "zookeeper_server_host_health_enabled", - "value": "true" - }, - { - "desc": "The disk location that ZooKeeper will use to store its database snapshots.", - "display_name": "Data Directory", - "name": "dataDir", - "value": "/var/lib/zookeeper" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "zookeeper_server_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The health check thresholds of the weighted average size of the ZooKeeper Server outstanding requests queue over a recent period. See ZooKeeper Server Outstanding Requests Monitoring Period.", - "display_name": "ZooKeeper Server Outstanding Requests Thresholds", - "name": "zookeeper_server_outstanding_requests_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of ZooKeeper Server in Bytes", - "name": "zookeeper_server_java_heapsize", - "value": "1073741824" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned will cause the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "The minimum session timeout, in milliseconds, that the ZooKeeper Server will allow the client to negotiate", - "display_name": "Minimum Session Timeout", - "name": "minSessionTimeout", - "value": "4000" - }, - { - "desc": "Specifies the name of the user that has read-write privileges when using password file based authentication for JMX access. JMX authentication must be enabled for this setting to take effect.", - "display_name": "Name of User with Read-Write Access to the JMX Agent", - "name": "jmx_passwd_file_readwrite_user", - "value": "controlRole" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health check thresholds for monitoring of free space on the filesystem that contains the ZooKeeper server's data directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Data Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Data Directory Free Space Monitoring Percentage Thresholds", - "name": "zookeeper_server_data_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Enables authentication when interacting with the JMX agent on the ZooKeeper server.", - "display_name": "Enable Authenticated Communication with the JMX Agent", - "name": "enable_jmx_authentication", - "value": "false" - }, - { - "desc": "The period to review when computing the moving average of the connection count. Specified in minutes.", - "display_name": "ZooKeeper Server Connection Count Monitoring Period", - "name": "zookeeper_server_connection_count_window", - "value": "3" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into zoo.cfg for this role only.", - "display_name": "Server Advanced Configuration Snippet (Safety Valve) for zoo.cfg", - "name": "zookeeper_config_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The maximum number of concurrent connections (at the socket level) that a single client, identified by the IP address, may make to a single member of the ZooKeeper ensemble. This setting is used to prevent certain classes of DoS attacks, including file descriptor exhaustion. To remove the limit on concurrent connections, set this value to 0.", - "display_name": "Maximum Client Connections", - "name": "maxClientCnxns", - "value": "60" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Zookeeper Server", - "name": "zk_server_java_opts", - "value": "" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Server logs. Typically used by log4j.", - "display_name": "Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "The maximum session timeout, in milliseconds, that the ZooKeeper Server will allow the client to negotiate", - "display_name": "Maximum Session Timeout", - "name": "maxSessionTimeout", - "value": "40000" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The health check thresholds for monitoring of free space on the filesystem that contains the ZooKeeper server's data log directory.", - "display_name": "Data Log Directory Free Space Monitoring Absolute Thresholds", - "name": "zookeeper_server_data_log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The port to monitor for client connections. This is the port that clients attempt to connect to.", - "display_name": "Client Port", - "name": "clientPort", - "value": "2181" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "zookeeper_server_gc_duration_window", - "value": "5" - }, - { - "desc": "Specifies the password of the user that has read-write privileges when using password file based authentication for JMX access. JMX authentication must be enabled for this setting to take effect.", - "display_name": "Password of user with read-write access to the JMX agent", - "name": "jmx_passwd_file_readwrite_user_password", - "value": "CONTROL" - }, - { - "desc": "Directory where ZooKeeper will place its log files.", - "display_name": "ZooKeeper Log Directory", - "name": "zk_server_log_dir", - "value": "/var/log/zookeeper" - }, - { - "desc": "The maximum number of rolled log files to keep for Server logs. Typically used by log4j.", - "display_name": "Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "Enables the JMX agent on the ZooKeeper server. Turning this off on any of the ZooKeeper servers that are part of a service will prevent Cloudera Manager from being able to monitor that server and may affect the monitoring provided on the entire service.", - "display_name": "Enable JMX Agent", - "name": "enable_jmx_agent", - "value": "true" - }, - { - "desc": "The percentage thresholds of the ratio of the maximum request latency to the maximum client-negotiable session timeout since the server was started.", - "display_name": "Maximum Latency Monitoring Thresholds", - "name": "zookeeper_server_max_latency_thresholds", - "value": "{\"critical\":\"100.0\",\"warning\":\"75.0\"}" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The health check thresholds for monitoring of free space on the filesystem that contains the ZooKeeper Server's data directory.", - "display_name": "Data Directory Free Space Monitoring Absolute Thresholds", - "name": "zookeeper_server_data_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Enables the quorum membership check for this ZooKeeper Server.", - "display_name": "Enable the Quorum Membership Check", - "name": "zookeeper_server_quorum_membership_enabled", - "value": "true" - }, - { - "desc": "The health check thresholds for monitoring of free space on the filesystem that contains the ZooKeeper server's data log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Data Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Data Log Directory Free Space Monitoring Percentage Thresholds", - "name": "zookeeper_server_data_log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Unique identifier for each ZooKeeper server, typically starts at 1", - "display_name": "ZooKeeper Server ID", - "name": "serverId", - "value": null - }, - { - "desc": "The minimum log level for Server logs", - "display_name": "Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Specifies the password of the user that has read-only privileges when using password file based authentication for JMX access. JMX authentication must be enabled for this setting to take effect.", - "display_name": "Password of User with Read-Only Access to the JMX agent", - "name": "jmx_passwd_file_readonly_user_password", - "value": "MONITOR" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/resources/zookeeper-service.json b/sahara/plugins/cdh/v5/resources/zookeeper-service.json deleted file mode 100644 index c377ad38..00000000 --- a/sahara/plugins/cdh/v5/resources/zookeeper-service.json +++ /dev/null @@ -1,146 +0,0 @@ -[ - { - "desc": "Configures the path of the root znode under which all canary updates are performed", - "display_name": "ZooKeeper Canary Root Znode Path", - "name": "zookeeper_canary_root_path", - "value": "/cloudera_manager_zookeeper_canary" - }, - { - "desc": "The user that this service's processes should run as.", - "display_name": "System User", - "name": "process_username", - "value": "zookeeper" - }, - { - "desc": "The frequency in which the log4j event publication appender will retry sending undelivered log events to the Event server, in seconds", - "display_name": "Log Event Retry Frequency", - "name": "log_event_retry_frequency", - "value": "30" - }, - { - "desc": "Amount of time, in ticks, to allow followers to sync with ZooKeeper. If followers fall too far behind a leader, they are dropped.", - "display_name": "Synchronization Limit", - "name": "syncLimit", - "value": "5" - }, - { - "desc": "When enabled, ZooKeeper auto purge feature retains this many most recent snapshots and the corresponding transaction logs in the dataDir and dataLogDir respectively and deletes the rest. Defaults to 5. Minimum value is 3.", - "display_name": "Auto Purge Snapshots Retain Count", - "name": "autopurgeSnapRetainCount", - "value": "5" - }, - { - "desc": "The time interval in hours for which the purge task has to be triggered. Set to a positive integer (1 and above) to enable the auto purging. Defaults to 24.", - "display_name": "Auto Purge Time Interval", - "name": "autopurgeInterval", - "value": "24" - }, - { - "desc": "Enable Kerberos authentication for ZooKeeper.", - "display_name": "Enable Kerberos Authentication", - "name": "enableSecurity", - "value": "false" - }, - { - "desc": "The number of snapshot files and corresponding transaction logs to keep when running the Cleanup command.", - "display_name": "Cleanup Retain Count", - "name": "cleanupRetainCount", - "value": "5" - }, - { - "desc": "Enables the health check that a client can connect to ZooKeeper and perform basic operations", - "display_name": "ZooKeeper Canary Health Check", - "name": "zookeeper_canary_health_enabled", - "value": "true" - }, - { - "desc": "Configures the timeout used by the canary sessions with ZooKeeper servers", - "display_name": "ZooKeeper Canary Session Timeout", - "name": "zookeeper_canary_session_timeout", - "value": "30000" - }, - { - "desc": "The group that this service's processes should run as.", - "display_name": "System Group", - "name": "process_groupname", - "value": "zookeeper" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Amount of time, in ticks, to allow followers to connect and sync to a leader. Increase this value as needed, if the amount of data managed by ZooKeeper is large.", - "display_name": "Initialization Limit", - "name": "initLimit", - "value": "10" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "ZooKeeper Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "zookeeper_env_safety_valve", - "value": null - }, - { - "desc": "Configures the timeout used by the canary for connection establishment with ZooKeeper servers", - "display_name": "ZooKeeper Canary Connection Timeout", - "name": "zookeeper_canary_connection_timeout", - "value": "10000" - }, - { - "desc": "Configures the timeout used by the canary for ZooKeeper operations", - "display_name": "ZooKeeper Canary Operation Timeout", - "name": "zookeeper_canary_operation_timeout", - "value": "30000" - }, - { - "desc": "The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned will cause the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "The length of time, in milliseconds, of a single tick, which is the basic time unit used by ZooKeeper. A tick is used to regulate heartbeats and timeouts.", - "display_name": "Tick Time", - "name": "tickTime", - "value": "2000" - }, - { - "desc": "When set, each role will identify important log events and forward them to Cloudera Manager.", - "display_name": "Enable Log Event Capture", - "name": "catch_events", - "value": "true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The health test thresholds of the overall Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Servers falls below the critical threshold.", - "display_name": "Healthy Server Monitoring Thresholds", - "name": "zookeeper_servers_healthy_thresholds", - "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "Whether the leader accepts client connections.", - "display_name": "Leader Serves", - "name": "leaderServes", - "value": "yes" - }, - { - "desc": "Automatically create data directories at startup, if they do not exist. Enabling this configuration should be used with care as it will suppress any errors in setup of data directories.", - "display_name": "Enable auto-creation of data directories", - "name": "zookeeper_datadir_autocreate", - "value": "false" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5/validation.py b/sahara/plugins/cdh/v5/validation.py deleted file mode 100644 index 37db63c2..00000000 --- a/sahara/plugins/cdh/v5/validation.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2014 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.plugins.cdh.v5 import plugin_utils as pu -from sahara.plugins.cdh import validation - - -class ValidatorV5(validation.Validator): - PU = pu.PluginUtilsV5() diff --git a/sahara/plugins/cdh/v5/versionhandler.py b/sahara/plugins/cdh/v5/versionhandler.py deleted file mode 100644 index 41aba39d..00000000 --- a/sahara/plugins/cdh/v5/versionhandler.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) 2014 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.plugins.cdh import abstractversionhandler as avm -from sahara.plugins.cdh.v5 import cloudera_utils -from sahara.plugins.cdh.v5 import config_helper -from sahara.plugins.cdh.v5 import deploy -from sahara.plugins.cdh.v5 import edp_engine -from sahara.plugins.cdh.v5 import plugin_utils -from sahara.plugins.cdh.v5 import validation - - -class VersionHandler(avm.BaseVersionHandler): - - def __init__(self): - super(VersionHandler, self).__init__() - self.config_helper = config_helper.ConfigHelperV5() - self.cloudera_utils = cloudera_utils.ClouderaUtilsV5() - self.plugin_utils = plugin_utils.PluginUtilsV5() - self.deploy = deploy - self.edp_engine = edp_engine - self.validation = validation.ValidatorV5() - - def get_node_processes(self): - return { - "CLOUDERA": ['CLOUDERA_MANAGER'], - "HDFS": ['HDFS_NAMENODE', 'HDFS_DATANODE', - 'HDFS_SECONDARYNAMENODE'], - "YARN": ['YARN_RESOURCEMANAGER', 'YARN_NODEMANAGER', - 'YARN_JOBHISTORY'], - "OOZIE": ['OOZIE_SERVER'], - "HIVE": ['HIVE_SERVER2', 'HIVE_METASTORE', 'HIVE_WEBHCAT'], - "HUE": ['HUE_SERVER'], - "SPARK_ON_YARN": ['SPARK_YARN_HISTORY_SERVER'], - "ZOOKEEPER": ['ZOOKEEPER_SERVER'], - "HBASE": ['HBASE_MASTER', 'HBASE_REGIONSERVER'], - "YARN_GATEWAY": [], - "HDFS_GATEWAY": [] - } diff --git a/sahara/plugins/cdh/v5_3_0/__init__.py b/sahara/plugins/cdh/v5_3_0/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/sahara/plugins/cdh/v5_3_0/cloudera_utils.py b/sahara/plugins/cdh/v5_3_0/cloudera_utils.py deleted file mode 100644 index ae9056c8..00000000 --- a/sahara/plugins/cdh/v5_3_0/cloudera_utils.py +++ /dev/null @@ -1,395 +0,0 @@ -# Copyright (c) 2014 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.i18n import _ -from sahara.plugins.cdh import cloudera_utils as cu -from sahara.plugins.cdh.v5_3_0 import config_helper -from sahara.plugins.cdh.v5_3_0 import plugin_utils as pu -from sahara.plugins.cdh.v5_3_0 import validation -from sahara.swift import swift_helper -from sahara.utils import cluster_progress_ops as cpo -from sahara.utils import configs as s_cfg -from sahara.utils import xmlutils - - -HDFS_SERVICE_TYPE = 'HDFS' -YARN_SERVICE_TYPE = 'YARN' -OOZIE_SERVICE_TYPE = 'OOZIE' -HIVE_SERVICE_TYPE = 'HIVE' -HUE_SERVICE_TYPE = 'HUE' -SPARK_SERVICE_TYPE = 'SPARK_ON_YARN' -ZOOKEEPER_SERVICE_TYPE = 'ZOOKEEPER' -HBASE_SERVICE_TYPE = 'HBASE' -FLUME_SERVICE_TYPE = 'FLUME' -SENTRY_SERVICE_TYPE = 'SENTRY' -SOLR_SERVICE_TYPE = 'SOLR' -SQOOP_SERVICE_TYPE = 'SQOOP' -KS_INDEXER_SERVICE_TYPE = 'KS_INDEXER' -IMPALA_SERVICE_TYPE = 'IMPALA' - -c_helper = config_helper.ConfigHelperV530() - - -class ClouderaUtilsV530(cu.ClouderaUtils): - FLUME_SERVICE_NAME = 'flume01' - SOLR_SERVICE_NAME = 'solr01' - SQOOP_SERVICE_NAME = 'sqoop01' - KS_INDEXER_SERVICE_NAME = 'ks_indexer01' - IMPALA_SERVICE_NAME = 'impala01' - SENTRY_SERVICE_NAME = 'sentry01' - CM_API_VERSION = 8 - - def __init__(self): - cu.ClouderaUtils.__init__(self) - self.pu = pu.PluginUtilsV530() - self.validator = validation.ValidatorV530 - - def get_service_by_role(self, role, cluster=None, instance=None): - cm_cluster = None - if cluster: - cm_cluster = self.get_cloudera_cluster(cluster) - elif instance: - cm_cluster = self.get_cloudera_cluster(instance.cluster) - else: - raise ValueError(_("'cluster' or 'instance' argument missed")) - - if role in ['AGENT']: - return cm_cluster.get_service(self.FLUME_SERVICE_NAME) - elif role in ['SENTRY_SERVER']: - return cm_cluster.get_service(self.SENTRY_SERVICE_NAME) - elif role in ['SQOOP_SERVER']: - return cm_cluster.get_service(self.SQOOP_SERVICE_NAME) - elif role in ['SOLR_SERVER']: - return cm_cluster.get_service(self.SOLR_SERVICE_NAME) - elif role in ['HBASE_INDEXER']: - return cm_cluster.get_service(self.KS_INDEXER_SERVICE_NAME) - elif role in ['CATALOGSERVER', 'STATESTORE', 'IMPALAD', 'LLAMA']: - return cm_cluster.get_service(self.IMPALA_SERVICE_NAME) - else: - return super(ClouderaUtilsV530, self).get_service_by_role( - role, cluster, instance) - - @cpo.event_wrapper( - True, step=_("First run cluster"), param=('cluster', 1)) - @cu.cloudera_cmd - def first_run(self, cluster): - cm_cluster = self.get_cloudera_cluster(cluster) - yield cm_cluster.first_run() - - @cpo.event_wrapper(True, step=_("Create services"), param=('cluster', 1)) - def create_services(self, cluster): - api = self.get_api_client(cluster) - cm_cluster = api.create_cluster(cluster.name, - fullVersion=cluster.hadoop_version) - - if len(self.pu.get_zookeepers(cluster)) > 0: - cm_cluster.create_service(self.ZOOKEEPER_SERVICE_NAME, - ZOOKEEPER_SERVICE_TYPE) - cm_cluster.create_service(self.HDFS_SERVICE_NAME, HDFS_SERVICE_TYPE) - cm_cluster.create_service(self.YARN_SERVICE_NAME, YARN_SERVICE_TYPE) - cm_cluster.create_service(self.OOZIE_SERVICE_NAME, OOZIE_SERVICE_TYPE) - if self.pu.get_hive_metastore(cluster): - cm_cluster.create_service(self.HIVE_SERVICE_NAME, - HIVE_SERVICE_TYPE) - if self.pu.get_hue(cluster): - cm_cluster.create_service(self.HUE_SERVICE_NAME, HUE_SERVICE_TYPE) - if self.pu.get_spark_historyserver(cluster): - cm_cluster.create_service(self.SPARK_SERVICE_NAME, - SPARK_SERVICE_TYPE) - if self.pu.get_hbase_master(cluster): - cm_cluster.create_service(self.HBASE_SERVICE_NAME, - HBASE_SERVICE_TYPE) - if len(self.pu.get_flumes(cluster)) > 0: - cm_cluster.create_service(self.FLUME_SERVICE_NAME, - FLUME_SERVICE_TYPE) - if self.pu.get_sentry(cluster): - cm_cluster.create_service(self.SENTRY_SERVICE_NAME, - SENTRY_SERVICE_TYPE) - if len(self.pu.get_solrs(cluster)) > 0: - cm_cluster.create_service(self.SOLR_SERVICE_NAME, - SOLR_SERVICE_TYPE) - if self.pu.get_sqoop(cluster): - cm_cluster.create_service(self.SQOOP_SERVICE_NAME, - SQOOP_SERVICE_TYPE) - if len(self.pu.get_hbase_indexers(cluster)) > 0: - cm_cluster.create_service(self.KS_INDEXER_SERVICE_NAME, - KS_INDEXER_SERVICE_TYPE) - if self.pu.get_catalogserver(cluster): - cm_cluster.create_service(self.IMPALA_SERVICE_NAME, - IMPALA_SERVICE_TYPE) - - def await_agents(self, cluster, instances): - self._await_agents(cluster, instances, c_helper.AWAIT_AGENTS_TIMEOUT) - - @cpo.event_wrapper( - True, step=_("Configure services"), param=('cluster', 1)) - def configure_services(self, cluster): - cm_cluster = self.get_cloudera_cluster(cluster) - - if len(self.pu.get_zookeepers(cluster)) > 0: - zookeeper = cm_cluster.get_service(self.ZOOKEEPER_SERVICE_NAME) - zookeeper.update_config(self._get_configs(ZOOKEEPER_SERVICE_TYPE, - cluster=cluster)) - - hdfs = cm_cluster.get_service(self.HDFS_SERVICE_NAME) - hdfs.update_config(self._get_configs(HDFS_SERVICE_TYPE, - cluster=cluster)) - - yarn = cm_cluster.get_service(self.YARN_SERVICE_NAME) - yarn.update_config(self._get_configs(YARN_SERVICE_TYPE, - cluster=cluster)) - - oozie = cm_cluster.get_service(self.OOZIE_SERVICE_NAME) - oozie.update_config(self._get_configs(OOZIE_SERVICE_TYPE, - cluster=cluster)) - - if self.pu.get_hive_metastore(cluster): - hive = cm_cluster.get_service(self.HIVE_SERVICE_NAME) - hive.update_config(self._get_configs(HIVE_SERVICE_TYPE, - cluster=cluster)) - - if self.pu.get_hue(cluster): - hue = cm_cluster.get_service(self.HUE_SERVICE_NAME) - hue.update_config(self._get_configs(HUE_SERVICE_TYPE, - cluster=cluster)) - - if self.pu.get_spark_historyserver(cluster): - spark = cm_cluster.get_service(self.SPARK_SERVICE_NAME) - spark.update_config(self._get_configs(SPARK_SERVICE_TYPE, - cluster=cluster)) - - if self.pu.get_hbase_master(cluster): - hbase = cm_cluster.get_service(self.HBASE_SERVICE_NAME) - hbase.update_config(self._get_configs(HBASE_SERVICE_TYPE, - cluster=cluster)) - - if len(self.pu.get_flumes(cluster)) > 0: - flume = cm_cluster.get_service(self.FLUME_SERVICE_NAME) - flume.update_config(self._get_configs(FLUME_SERVICE_TYPE, - cluster=cluster)) - - if self.pu.get_sentry(cluster): - sentry = cm_cluster.get_service(self.SENTRY_SERVICE_NAME) - sentry.update_config(self._get_configs(SENTRY_SERVICE_TYPE, - cluster=cluster)) - - if len(self.pu.get_solrs(cluster)) > 0: - solr = cm_cluster.get_service(self.SOLR_SERVICE_NAME) - solr.update_config(self._get_configs(SOLR_SERVICE_TYPE, - cluster=cluster)) - - if self.pu.get_sqoop(cluster): - sqoop = cm_cluster.get_service(self.SQOOP_SERVICE_NAME) - sqoop.update_config(self._get_configs(SQOOP_SERVICE_TYPE, - cluster=cluster)) - - if len(self.pu.get_hbase_indexers(cluster)) > 0: - ks_indexer = cm_cluster.get_service(self.KS_INDEXER_SERVICE_NAME) - ks_indexer.update_config( - self._get_configs(KS_INDEXER_SERVICE_TYPE, cluster=cluster)) - - if self.pu.get_catalogserver(cluster): - impala = cm_cluster.get_service(self.IMPALA_SERVICE_NAME) - impala.update_config(self._get_configs(IMPALA_SERVICE_TYPE, - cluster=cluster)) - - def _get_configs(self, service, cluster=None, instance=None): - def get_hadoop_dirs(mount_points, suffix): - return ','.join([x + suffix for x in mount_points]) - - all_confs = {} - if cluster: - zk_count = self.validator._get_inst_count(cluster, - 'ZOOKEEPER_SERVER') - hbm_count = self.validator._get_inst_count(cluster, 'HBASE_MASTER') - snt_count = self.validator._get_inst_count(cluster, - 'SENTRY_SERVER') - ks_count =\ - self.validator._get_inst_count(cluster, - 'KEY_VALUE_STORE_INDEXER') - imp_count = self.validator._get_inst_count(cluster, - 'IMPALA_CATALOGSERVER') - hive_count = self.validator._get_inst_count(cluster, - 'HIVE_METASTORE') - slr_count = self.validator._get_inst_count(cluster, 'SOLR_SERVER') - sqp_count = self.validator._get_inst_count(cluster, 'SQOOP_SERVER') - core_site_safety_valve = '' - if self.pu.c_helper.is_swift_enabled(cluster): - configs = swift_helper.get_swift_configs() - confs = {c['name']: c['value'] for c in configs} - core_site_safety_valve = xmlutils.create_elements_xml(confs) - all_confs = { - 'HDFS': { - 'zookeeper_service': - self.ZOOKEEPER_SERVICE_NAME if zk_count else '', - 'dfs_block_local_path_access_user': - 'impala' if imp_count else '', - 'core_site_safety_valve': core_site_safety_valve - }, - 'HIVE': { - 'mapreduce_yarn_service': self.YARN_SERVICE_NAME, - 'sentry_service': - self.SENTRY_SERVICE_NAME if snt_count else '', - 'zookeeper_service': - self.ZOOKEEPER_SERVICE_NAME if zk_count else '' - }, - 'OOZIE': { - 'mapreduce_yarn_service': self.YARN_SERVICE_NAME, - 'hive_service': - self.HIVE_SERVICE_NAME if hive_count else '', - 'zookeeper_service': - self.ZOOKEEPER_SERVICE_NAME if zk_count else '' - }, - 'YARN': { - 'hdfs_service': self.HDFS_SERVICE_NAME, - 'zookeeper_service': - self.ZOOKEEPER_SERVICE_NAME if zk_count else '' - }, - 'HUE': { - 'hive_service': self.HIVE_SERVICE_NAME, - 'oozie_service': self.OOZIE_SERVICE_NAME, - 'sentry_service': - self.SENTRY_SERVICE_NAME if snt_count else '', - 'solr_service': - self.SOLR_SERVICE_NAME if slr_count else '', - 'zookeeper_service': - self.ZOOKEEPER_SERVICE_NAME if zk_count else '', - 'hbase_service': - self.HBASE_SERVICE_NAME if hbm_count else '', - 'impala_service': - self.IMPALA_SERVICE_NAME if imp_count else '', - 'sqoop_service': - self.SQOOP_SERVICE_NAME if sqp_count else '' - }, - 'SPARK_ON_YARN': { - 'yarn_service': self.YARN_SERVICE_NAME - }, - 'HBASE': { - 'hdfs_service': self.HDFS_SERVICE_NAME, - 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME, - 'hbase_enable_indexing': 'true' if ks_count else 'false', - 'hbase_enable_replication': - 'true' if ks_count else 'false' - }, - 'FLUME': { - 'hdfs_service': self.HDFS_SERVICE_NAME, - 'solr_service': - self.SOLR_SERVICE_NAME if slr_count else '', - 'hbase_service': - self.HBASE_SERVICE_NAME if hbm_count else '' - }, - 'SENTRY': { - 'hdfs_service': self.HDFS_SERVICE_NAME, - 'sentry_server_config_safety_valve': ( - c_helper.SENTRY_IMPALA_CLIENT_SAFETY_VALVE - if imp_count else '') - }, - 'SOLR': { - 'hdfs_service': self.HDFS_SERVICE_NAME, - 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME - }, - 'SQOOP': { - 'mapreduce_yarn_service': self.YARN_SERVICE_NAME - }, - 'KS_INDEXER': { - 'hbase_service': self.HBASE_SERVICE_NAME, - 'solr_service': self.SOLR_SERVICE_NAME - }, - 'IMPALA': { - 'hdfs_service': self.HDFS_SERVICE_NAME, - 'hbase_service': - self.HBASE_SERVICE_NAME if hbm_count else '', - 'hive_service': self.HIVE_SERVICE_NAME, - 'sentry_service': - self.SENTRY_SERVICE_NAME if snt_count else '', - 'zookeeper_service': - self.ZOOKEEPER_SERVICE_NAME if zk_count else '' - } - } - hive_confs = { - 'HIVE': { - 'hive_metastore_database_type': 'postgresql', - 'hive_metastore_database_host': - self.pu.get_manager(cluster).internal_ip, - 'hive_metastore_database_port': '7432', - 'hive_metastore_database_password': - self.pu.db_helper.get_hive_db_password(cluster) - } - } - hue_confs = { - 'HUE': { - 'hue_webhdfs': self.pu.get_role_name( - self.pu.get_namenode(cluster), 'NAMENODE') - } - } - sentry_confs = { - 'SENTRY': { - 'sentry_server_database_type': 'postgresql', - 'sentry_server_database_host': - self.pu.get_manager(cluster).internal_ip, - 'sentry_server_database_port': '7432', - 'sentry_server_database_password': - self.pu.db_helper.get_sentry_db_password(cluster) - } - } - - all_confs = s_cfg.merge_configs(all_confs, hue_confs) - all_confs = s_cfg.merge_configs(all_confs, hive_confs) - all_confs = s_cfg.merge_configs(all_confs, sentry_confs) - all_confs = s_cfg.merge_configs(all_confs, cluster.cluster_configs) - - if instance: - paths = instance.storage_paths() - - instance_default_confs = { - 'NAMENODE': { - 'dfs_name_dir_list': get_hadoop_dirs(paths, '/fs/nn') - }, - 'SECONDARYNAMENODE': { - 'fs_checkpoint_dir_list': - get_hadoop_dirs(paths, '/fs/snn') - }, - 'DATANODE': { - 'dfs_data_dir_list': get_hadoop_dirs(paths, '/fs/dn'), - 'dfs_datanode_data_dir_perm': 755, - 'dfs_datanode_handler_count': 30 - }, - 'NODEMANAGER': { - 'yarn_nodemanager_local_dirs': - get_hadoop_dirs(paths, '/yarn/local') - }, - 'SERVER': { - 'maxSessionTimeout': 60000 - }, - 'HIVESERVER2': { - 'hiveserver2_enable_impersonation': - 'false' if snt_count else 'true', - 'hive_hs2_config_safety_valve': ( - c_helper.HIVE_SERVER2_SENTRY_SAFETY_VALVE - if snt_count else '') - }, - 'HIVEMETASTORE': { - 'hive_metastore_config_safety_valve': ( - c_helper.HIVE_METASTORE_SENTRY_SAFETY_VALVE - if snt_count else '') - } - } - - ng_user_confs = self.pu.convert_process_configs( - instance.node_group.node_configs) - all_confs = s_cfg.merge_configs(all_confs, ng_user_confs) - all_confs = s_cfg.merge_configs(all_confs, instance_default_confs) - - return all_confs.get(service, {}) diff --git a/sahara/plugins/cdh/v5_3_0/config_helper.py b/sahara/plugins/cdh/v5_3_0/config_helper.py deleted file mode 100644 index c496c759..00000000 --- a/sahara/plugins/cdh/v5_3_0/config_helper.py +++ /dev/null @@ -1,171 +0,0 @@ -# Copyright (c) 2014 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.plugins.cdh import config_helper as c_h -from sahara.plugins import provisioning as p -from sahara.utils import files as f - - -class ConfigHelperV530(c_h.ConfigHelper): - path_to_config = 'plugins/cdh/v5_3_0/resources/' - - CDH5_UBUNTU_REPO = ( - 'deb [arch=amd64] http://archive.cloudera.com/cdh5' - '/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib' - '\ndeb-src http://archive.cloudera.com/cdh5/ubuntu' - '/precise/amd64/cdh precise-cdh5.3.0 contrib') - - DEFAULT_CDH5_UBUNTU_REPO_KEY_URL = ( - 'http://archive.cloudera.com/cdh5/ubuntu' - '/precise/amd64/cdh/archive.key') - - CM5_UBUNTU_REPO = ( - 'deb [arch=amd64] http://archive.cloudera.com/cm5' - '/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib' - '\ndeb-src http://archive.cloudera.com/cm5/ubuntu' - '/precise/amd64/cm precise-cm5.3.0 contrib') - - DEFAULT_CM5_UBUNTU_REPO_KEY_URL = ( - 'http://archive.cloudera.com/cm5/ubuntu' - '/precise/amd64/cm/archive.key') - - CDH5_CENTOS_REPO = ( - '[cloudera-cdh5]' - '\nname=Cloudera\'s Distribution for Hadoop, Version 5' - '\nbaseurl=http://archive.cloudera.com/cdh5/redhat/6' - '/x86_64/cdh/5.3.0/' - '\ngpgkey = http://archive.cloudera.com/cdh5/redhat/6' - '/x86_64/cdh/RPM-GPG-KEY-cloudera' - '\ngpgcheck = 1') - - CM5_CENTOS_REPO = ( - '[cloudera-manager]' - '\nname=Cloudera Manager' - '\nbaseurl=http://archive.cloudera.com/cm5/redhat/6' - '/x86_64/cm/5.3.0/' - '\ngpgkey = http://archive.cloudera.com/cm5/redhat/6' - '/x86_64/cm/RPM-GPG-KEY-cloudera' - '\ngpgcheck = 1') - - HIVE_SERVER2_SENTRY_SAFETY_VALVE = f.get_file_text( - path_to_config + 'hive-server2-sentry-safety.xml') - - HIVE_METASTORE_SENTRY_SAFETY_VALVE = f.get_file_text( - path_to_config + 'hive-metastore-sentry-safety.xml') - - SENTRY_IMPALA_CLIENT_SAFETY_VALVE = f.get_file_text( - path_to_config + 'sentry-impala-client-safety.xml') - - _default_executor_classpath = ":".join( - ['/usr/lib/hadoop/lib/jackson-core-asl-1.8.8.jar', - '/usr/lib/hadoop-mapreduce/hadoop-openstack.jar']) - - EXECUTOR_EXTRA_CLASSPATH = p.Config( - 'Executor extra classpath', 'Spark', 'cluster', priority=2, - default_value=_default_executor_classpath, - description='Value for spark.executor.extraClassPath in ' - 'spark-defaults.conf (default: %s)' - % _default_executor_classpath) - - def __init__(self): - super(ConfigHelperV530, self).__init__() - self.priority_one_confs = self._load_json( - self.path_to_config + 'priority-one-confs.json') - self._load_ng_plugin_configs() - - def _get_cluster_plugin_configs(self): - confs = super(ConfigHelperV530, self)._get_ng_plugin_configs() - confs.append(self.EXECUTOR_EXTRA_CLASSPATH) - return confs - - def _load_ng_plugin_configs(self): - self.hdfs_confs = self._load_and_init_configs( - 'hdfs-service.json', 'HDFS', 'cluster') - self.namenode_confs = self._load_and_init_configs( - 'hdfs-namenode.json', 'NAMENODE', 'node') - self.datanode_confs = self._load_and_init_configs( - 'hdfs-datanode.json', 'DATANODE', 'node') - self.secnamenode_confs = self._load_and_init_configs( - 'hdfs-secondarynamenode.json', 'SECONDARYNAMENODE', 'node') - self.hdfs_gateway_confs = self._load_and_init_configs( - 'hdfs-gateway.json', 'HDFS_GATEWAY', 'node') - self.yarn_confs = self._load_and_init_configs( - 'yarn-service.json', 'YARN', 'cluster') - self.resourcemanager_confs = self._load_and_init_configs( - 'yarn-resourcemanager.json', 'YARN_GATEWAY', 'node') - self.nodemanager_confs = self._load_and_init_configs( - 'yarn-nodemanager.json', 'RESOURCEMANAGER', 'node') - self.jobhistory_confs = self._load_and_init_configs( - 'yarn-jobhistory.json', 'NODEMANAGER', 'node') - self.yarn_gateway = self._load_and_init_configs( - 'yarn-gateway.json', 'JOBHISTORY', 'node') - self.oozie_service_confs = self._load_and_init_configs( - 'oozie-service.json', 'OOZIE', 'cluster') - self.oozie_role_confs = self._load_and_init_configs( - 'oozie-oozie_server.json', 'OOZIE', 'node') - self.hive_service_confs = self._load_and_init_configs( - 'hive-service.json', 'HIVE', 'cluster') - self.hive_metastore_confs = self._load_and_init_configs( - 'hive-hivemetastore.json', 'HIVEMETASTORE', 'node') - self.hive_hiveserver_confs = self._load_and_init_configs( - 'hive-hiveserver2.json', 'HIVESERVER', 'node') - self.hive_webhcat_confs = self._load_and_init_configs( - 'hive-webhcat.json', 'WEBHCAT', 'node') - self.hue_service_confs = self._load_and_init_configs( - 'hue-service.json', 'HUE', 'cluster') - self.hue_role_confs = self._load_and_init_configs( - 'hue-hue_server.json', 'HUE', 'node') - self.spark_service_confs = self._load_and_init_configs( - 'spark-service.json', 'SPARK_ON_YARN', 'cluster') - self.spark_role_confs = self._load_and_init_configs( - 'spark-spark_yarn_history_server.json', 'SPARK_ON_YARN', 'node') - self.zookeeper_server_confs = self._load_and_init_configs( - 'zookeeper-server.json', 'ZOOKEEPER', 'cluster') - self.zookeeper_service_confs = self._load_and_init_configs( - 'zookeeper-service.json', 'ZOOKEEPER', 'node') - self.hbase_confs = self._load_and_init_configs( - 'hbase-service.json', 'HBASE', 'cluster') - self.master_confs = self._load_and_init_configs( - 'hbase-master.json', 'MASTER', 'node') - self.regionserver_confs = self._load_and_init_configs( - 'hbase-regionserver.json', 'REGIONSERVER', 'node') - self.flume_service_confs = self._load_and_init_configs( - 'flume-service.json', 'HDFS', 'cluster') - self.flume_agent_confs = self._load_and_init_configs( - 'flume-agent.json', 'FLUME', 'node') - self.sentry_service_confs = self._load_and_init_configs( - 'sentry-service.json', 'SENTRY', 'cluster') - self.sentry_server_confs = self._load_and_init_configs( - 'sentry-sentry_server.json', 'SENTRY', 'node') - self.solr_service_confs = self._load_and_init_configs( - 'solr-service.json', 'SOLR', 'cluster') - self.solr_server_confs = self._load_and_init_configs( - 'solr-solr_server.json', 'SOLR', 'node') - self.sqoop_service_confs = self._load_and_init_configs( - 'sqoop-service.json', 'SQOOP', 'cluster') - self.sqoop_server_confs = self._load_and_init_configs( - 'sqoop-sqoop_server.json', 'SQOOP', 'node') - self.ks_indexer_service_confs = self._load_and_init_configs( - 'ks_indexer-service.json', 'KS_INDEXER', 'cluster') - self.ks_indexer_role_confs = self._load_and_init_configs( - 'ks_indexer-hbase_indexer.json', 'KS_INDEXER', 'node') - self.impala_service_confs = self._load_and_init_configs( - 'impala-service.json', 'IMPALA', 'cluster') - self.impala_catalogserver_confs = self._load_and_init_configs( - 'impala-catalogserver.json', 'CATALOGSERVER', 'node') - self.impala_impalad_confs = self._load_and_init_configs( - 'impala-impalad.json', 'IMPALAD', 'node') - self.impala_statestore_confs = self._load_and_init_configs( - 'impala-statestore.json', 'STATESTORE', 'node') diff --git a/sahara/plugins/cdh/v5_3_0/db_helper.py b/sahara/plugins/cdh/v5_3_0/db_helper.py deleted file mode 100644 index e867dc22..00000000 --- a/sahara/plugins/cdh/v5_3_0/db_helper.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2014 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.plugins.cdh import db_helper as dh -from sahara.utils import files - - -def get_hive_db_password(cluster): - return dh.get_password_from_db(cluster, dh.HIVE_DB_PASSWORD) - - -def get_sentry_db_password(cluster): - return dh.get_password_from_db(cluster, dh.SENTRY_DB_PASSWORD) - - -def create_hive_database(cluster, remote): - db_password = get_hive_db_password(cluster) - create_db_script = files.get_file_text( - 'plugins/cdh/v5_3_0/resources/create_hive_db.sql') - create_db_script = create_db_script % db_password - dh.remote_execute_db_script(remote, create_db_script) - - -def create_sentry_database(cluster, remote): - db_password = get_sentry_db_password(cluster) - create_db_script = files.get_file_text( - 'plugins/cdh/v5_3_0/resources/create_sentry_db.sql') - create_db_script = create_db_script % db_password - dh.remote_execute_db_script(remote, create_db_script) diff --git a/sahara/plugins/cdh/v5_3_0/deploy.py b/sahara/plugins/cdh/v5_3_0/deploy.py deleted file mode 100644 index a1b04806..00000000 --- a/sahara/plugins/cdh/v5_3_0/deploy.py +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright (c) 2014 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.i18n import _ -from sahara.plugins.cdh import commands as cmd -from sahara.plugins.cdh.v5_3_0 import cloudera_utils as cu -from sahara.plugins import utils as gu -from sahara.service.edp import hdfs_helper as h -from sahara.utils import cluster_progress_ops as cpo - -PACKAGES = [ - 'cloudera-manager-agent', - 'cloudera-manager-daemons', - 'cloudera-manager-server', - 'cloudera-manager-server-db-2', - 'flume-ng', - 'hadoop-hdfs-datanode', - 'hadoop-hdfs-namenode', - 'hadoop-hdfs-secondarynamenode', - 'hadoop-mapreduce', - 'hadoop-mapreduce-historyserver', - 'hadoop-yarn-nodemanager', - 'hadoop-yarn-resourcemanager', - 'hbase', - 'hbase-solr', - 'hive-hcatalog', - 'hive-metastore', - 'hive-server2', - 'hive-webhcat-server', - 'hue', - 'impala', - 'impala-server', - 'impala-state-store', - 'impala-catalog', - 'impala-shell', - 'oozie', - 'oracle-j2sdk1.7', - 'sentry', - 'solr-server', - 'solr-doc', - 'search', - 'spark-history-server', - 'sqoop2', - 'unzip', - 'zookeeper' -] - -CU = cu.ClouderaUtilsV530() - - -def configure_cluster(cluster): - instances = gu.get_instances(cluster) - - if not cmd.is_pre_installed_cdh(CU.pu.get_manager(cluster).remote()): - CU.pu.configure_os(instances) - CU.pu.install_packages(instances, PACKAGES) - - CU.pu.start_cloudera_agents(instances) - CU.pu.start_cloudera_manager(cluster) - CU.update_cloudera_password(cluster) - CU.await_agents(cluster, instances) - CU.create_mgmt_service(cluster) - CU.create_services(cluster) - CU.configure_services(cluster) - CU.configure_instances(instances, cluster) - CU.deploy_configs(cluster) - - -@cpo.event_wrapper( - True, step=_("Start roles: NODEMANAGER, DATANODE"), param=('cluster', 0)) -def _start_roles(cluster, instances): - for instance in instances: - if 'HDFS_DATANODE' in instance.node_group.node_processes: - hdfs = CU.get_service_by_role('DATANODE', instance=instance) - CU.start_roles(hdfs, CU.pu.get_role_name(instance, 'DATANODE')) - - if 'YARN_NODEMANAGER' in instance.node_group.node_processes: - yarn = CU.get_service_by_role('NODEMANAGER', instance=instance) - CU.start_roles(yarn, CU.pu.get_role_name(instance, 'NODEMANAGER')) - - -def scale_cluster(cluster, instances): - if not instances: - return - - if not cmd.is_pre_installed_cdh(instances[0].remote()): - CU.pu.configure_os(instances) - CU.pu.install_packages(instances, PACKAGES) - - CU.pu.start_cloudera_agents(instances) - CU.await_agents(cluster, instances) - CU.configure_instances(instances, cluster) - CU.update_configs(instances) - CU.pu.configure_swift(cluster, instances) - _start_roles(cluster, instances) - CU.refresh_datanodes(cluster) - CU.refresh_yarn_nodes(cluster) - CU.restart_stale_services(cluster) - - -def decommission_cluster(cluster, instances): - dns = [] - dns_to_delete = [] - nms = [] - nms_to_delete = [] - for i in instances: - if 'HDFS_DATANODE' in i.node_group.node_processes: - dns.append(CU.pu.get_role_name(i, 'DATANODE')) - dns_to_delete.append( - CU.pu.get_role_name(i, 'HDFS_GATEWAY')) - - if 'YARN_NODEMANAGER' in i.node_group.node_processes: - nms.append(CU.pu.get_role_name(i, 'NODEMANAGER')) - nms_to_delete.append( - CU.pu.get_role_name(i, 'YARN_GATEWAY')) - - if dns: - CU.decommission_nodes( - cluster, 'DATANODE', dns, dns_to_delete) - - if nms: - CU.decommission_nodes( - cluster, 'NODEMANAGER', nms, nms_to_delete) - - CU.delete_instances(cluster, instances) - - CU.refresh_datanodes(cluster) - CU.refresh_yarn_nodes(cluster) - CU.restart_stale_services(cluster) - - -@cpo.event_wrapper(True, step=_("Prepare cluster"), param=('cluster', 0)) -def _prepare_cluster(cluster): - if CU.pu.get_oozie(cluster): - CU.pu.install_extjs(cluster) - - if CU.pu.get_hive_metastore(cluster): - CU.pu.configure_hive(cluster) - - if CU.pu.get_sentry(cluster): - CU.pu.configure_sentry(cluster) - - -@cpo.event_wrapper( - True, step=_("Finish cluster starting"), param=('cluster', 0)) -def _finish_cluster_starting(cluster): - if CU.pu.get_hive_metastore(cluster): - CU.pu.put_hive_hdfs_xml(cluster) - - server = CU.pu.get_hbase_master(cluster) - if CU.pu.c_helper.is_hbase_common_lib_enabled(cluster) and server: - with server.remote() as r: - h.create_hbase_common_lib(r) - - if CU.pu.get_flumes(cluster): - flume = CU.get_service_by_role('AGENT', cluster) - CU.start_service(flume) - - -def start_cluster(cluster): - _prepare_cluster(cluster) - - CU.first_run(cluster) - - CU.pu.configure_swift(cluster) - - _finish_cluster_starting(cluster) - - -def get_open_ports(node_group): - ports = [9000] # for CM agent - - ports_map = { - 'CLOUDERA_MANAGER': [7180, 7182, 7183, 7432, 7184, 8084, 8086, 10101, - 9997, 9996, 8087, 9998, 9999, 8085, 9995, 9994], - 'HDFS_NAMENODE': [8020, 8022, 50070, 50470], - 'HDFS_SECONDARYNAMENODE': [50090, 50495], - 'HDFS_DATANODE': [50010, 1004, 50075, 1006, 50020], - 'YARN_RESOURCEMANAGER': [8030, 8031, 8032, 8033, 8088], - 'YARN_NODEMANAGER': [8040, 8041, 8042], - 'YARN_JOBHISTORY': [10020, 19888], - 'HIVE_METASTORE': [9083], - 'HIVE_SERVER2': [10000], - 'HUE_SERVER': [8888], - 'OOZIE_SERVER': [11000, 11001], - 'SPARK_YARN_HISTORY_SERVER': [18088], - 'ZOOKEEPER_SERVER': [2181, 3181, 4181, 9010], - 'HBASE_MASTER': [60000], - 'HBASE_REGIONSERVER': [60020], - 'FLUME_AGENT': [41414], - 'SENTRY_SERVER': [8038], - 'SOLR_SERVER': [8983, 8984], - 'SQOOP_SERVER': [8005, 12000], - 'KEY_VALUE_STORE_INDEXER': [], - 'IMPALA_CATALOGSERVER': [25020, 26000], - 'IMPALA_STATESTORE': [25010, 24000], - 'IMPALAD': [21050, 21000, 23000, 25000, 28000, 22000] - } - - for process in node_group.node_processes: - if process in ports_map: - ports.extend(ports_map[process]) - - return ports diff --git a/sahara/plugins/cdh/v5_3_0/edp_engine.py b/sahara/plugins/cdh/v5_3_0/edp_engine.py deleted file mode 100644 index 31f14d58..00000000 --- a/sahara/plugins/cdh/v5_3_0/edp_engine.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2014 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.plugins.cdh import confighints_helper as ch_helper -from sahara.plugins.cdh import edp_engine -from sahara.plugins.cdh.v5_3_0 import cloudera_utils as cu -from sahara.service.edp.oozie import engine as oozie_engine -from sahara.utils import edp - - -class EdpOozieEngine(edp_engine.EdpOozieEngine): - - def __init__(self, cluster): - super(EdpOozieEngine, self).__init__(cluster) - self.cloudera_utils = cu.ClouderaUtilsV530() - - @staticmethod - def get_possible_job_config(job_type): - if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE): - return {'job_config': ch_helper.get_possible_hive_config_from( - 'plugins/cdh/v5_3_0/resources/hive-site.xml')} - if edp.compare_job_type(job_type, - edp.JOB_TYPE_MAPREDUCE, - edp.JOB_TYPE_MAPREDUCE_STREAMING): - return {'job_config': ch_helper.get_possible_mapreduce_config_from( - 'plugins/cdh/v5_3_0/resources/mapred-site.xml')} - if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG): - return {'job_config': ch_helper.get_possible_pig_config_from( - 'plugins/cdh/v5_3_0/resources/mapred-site.xml')} - return oozie_engine.OozieJobEngine.get_possible_job_config(job_type) - - -class EdpSparkEngine(edp_engine.EdpSparkEngine): - - edp_base_version = "5.3.0" diff --git a/sahara/plugins/cdh/v5_3_0/plugin_utils.py b/sahara/plugins/cdh/v5_3_0/plugin_utils.py deleted file mode 100644 index c9998f80..00000000 --- a/sahara/plugins/cdh/v5_3_0/plugin_utils.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright (c) 2014 Intel Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.conductor import resource as res -from sahara.plugins.cdh import plugin_utils as pu -from sahara.plugins.cdh.v5_3_0 import config_helper -from sahara.plugins.cdh.v5_3_0 import db_helper -from sahara.plugins import utils as u - - -class PluginUtilsV530(pu.AbstractPluginUtils): - - def __init__(self): - self.c_helper = config_helper.ConfigHelperV530() - self.db_helper = db_helper - - def get_role_name(self, instance, service): - # NOTE: role name must match regexp "[_A-Za-z][-_A-Za-z0-9]{0,63}" - shortcuts = { - 'AGENT': 'A', - 'ALERTPUBLISHER': 'AP', - 'CATALOGSERVER': 'ICS', - 'DATANODE': 'DN', - 'EVENTSERVER': 'ES', - 'HBASE_INDEXER': 'LHBI', - 'HIVEMETASTORE': 'HVM', - 'HIVESERVER2': 'HVS', - 'HOSTMONITOR': 'HM', - 'IMPALAD': 'ID', - 'JOBHISTORY': 'JS', - 'MASTER': 'M', - 'NAMENODE': 'NN', - 'NODEMANAGER': 'NM', - 'OOZIE_SERVER': 'OS', - 'REGIONSERVER': 'RS', - 'RESOURCEMANAGER': 'RM', - 'SECONDARYNAMENODE': 'SNN', - 'SENTRY_SERVER': 'SNT', - 'SERVER': 'S', - 'SERVICEMONITOR': 'SM', - 'SOLR_SERVER': 'SLR', - 'SPARK_YARN_HISTORY_SERVER': 'SHS', - 'SQOOP_SERVER': 'S2S', - 'STATESTORE': 'ISS', - 'WEBHCAT': 'WHC', - 'HDFS_GATEWAY': 'HG', - 'YARN_GATEWAY': 'YG' - } - return '%s_%s' % (shortcuts.get(service, service), - instance.hostname().replace('-', '_')) - - def get_sentry(self, cluster): - return u.get_instance(cluster, 'SENTRY_SERVER') - - def get_flumes(self, cluster): - return u.get_instances(cluster, 'FLUME_AGENT') - - def get_solrs(self, cluster): - return u.get_instances(cluster, 'SOLR_SERVER') - - def get_sqoop(self, cluster): - return u.get_instance(cluster, 'SQOOP_SERVER') - - def get_hbase_indexers(self, cluster): - return u.get_instances(cluster, 'KEY_VALUE_STORE_INDEXER') - - def get_catalogserver(self, cluster): - return u.get_instance(cluster, 'IMPALA_CATALOGSERVER') - - def get_statestore(self, cluster): - return u.get_instance(cluster, 'IMPALA_STATESTORE') - - def get_impalads(self, cluster): - return u.get_instances(cluster, 'IMPALAD') - - def convert_process_configs(self, configs): - p_dict = { - "CLOUDERA": ['MANAGER'], - "NAMENODE": ['NAMENODE'], - "DATANODE": ['DATANODE'], - "SECONDARYNAMENODE": ['SECONDARYNAMENODE'], - "RESOURCEMANAGER": ['RESOURCEMANAGER'], - "NODEMANAGER": ['NODEMANAGER'], - "JOBHISTORY": ['JOBHISTORY'], - "OOZIE": ['OOZIE_SERVER'], - "HIVESERVER": ['HIVESERVER2'], - "HIVEMETASTORE": ['HIVEMETASTORE'], - "WEBHCAT": ['WEBHCAT'], - "HUE": ['HUE_SERVER'], - "SPARK_ON_YARN": ['SPARK_YARN_HISTORY_SERVER'], - "ZOOKEEPER": ['SERVER'], - "MASTER": ['MASTER'], - "REGIONSERVER": ['REGIONSERVER'], - "FLUME": ['AGENT'], - "CATALOGSERVER": ['CATALOGSERVER'], - "STATESTORE": ['STATESTORE'], - "IMPALAD": ['IMPALAD'], - "KS_INDEXER": ['HBASE_INDEXER'], - "SENTRY": ['SENTRY_SERVER'], - "SOLR": ['SOLR_SERVER'], - "SQOOP": ['SQOOP_SERVER'], - 'YARN_GATEWAY': ['YARN_GATEWAY'], - 'HDFS_GATEWAY': ['HDFS_GATEWAY'] - } - if isinstance(configs, res.Resource): - configs = configs.to_dict() - for k in configs.keys(): - if k in p_dict.keys(): - item = configs[k] - del configs[k] - newkey = p_dict[k][0] - configs[newkey] = item - return res.Resource(configs) - - def configure_sentry(self, cluster): - manager = self.get_manager(cluster) - with manager.remote() as r: - self.db_helper.create_sentry_database(cluster, r) diff --git a/sahara/plugins/cdh/v5_3_0/resources/cdh_config.py b/sahara/plugins/cdh/v5_3_0/resources/cdh_config.py deleted file mode 100644 index 65dc3185..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/cdh_config.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright (c) 2014 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from oslo_serialization import jsonutils as json -import six - -from sahara.plugins.cdh.client import api_client - -# -- cm config -- - -cm_address = 'localhost' -cm_port = 7180 -cm_username = 'admin' -cm_password = 'admin' - -hdfs_service_name = 'hdfs01' -yarn_service_name = 'yarn01' -oozie_service_name = 'oozie01' -hive_service_name = 'hive01' -hue_service_name = 'hue01' -spark_service_name = 'spark_on_yarn01' -zookeeper_service_name = 'zookeeper01' -hbase_service_name = 'hbase01' -flume_service_name = 'flume01' -sqoop_service_name = 'sqoop01' -solr_service_name = 'solr01' -ks_indexer_service_name = 'ks_indexer01' -impala_service_name = 'impala01' -sentry_service_name = 'sentry01' - - -def get_cm_api(): - return api_client.ApiResource(cm_address, server_port=cm_port, - username=cm_username, password=cm_password) - - -def get_cluster(api): - return api.get_all_clusters()[0] - - -def process_service(service, service_name): - for role_cfgs in service.get_all_role_config_groups(): - role_cm_cfg = role_cfgs.get_config(view='full') - role_cfg = parse_config(role_cm_cfg) - role_name = role_cfgs.roleType.lower() - write_cfg(role_cfg, '%s-%s.json' % (service_name, role_name)) - - service_cm_cfg = service.get_config(view='full')[0] - service_cfg = parse_config(service_cm_cfg) - write_cfg(service_cfg, '%s-service.json' % service_name) - - -def parse_config(config): - cfg = [] - for name, value in six.iteritems(config): - p = { - 'name': value.name, - 'value': value.default, - 'display_name': value.displayName, - 'desc': value.description - } - cfg.append(p) - - return cfg - - -def write_cfg(cfg, file_name): - to_write = json.dumps(cfg, sort_keys=True, indent=4, - separators=(',', ': ')) - - with open(file_name, 'w') as f: - f.write(to_write) - - -def main(): - client = get_cm_api() - cluster = get_cluster(client) - - hdfs = cluster.get_service(hdfs_service_name) - process_service(hdfs, 'hdfs') - - yarn = cluster.get_service(yarn_service_name) - process_service(yarn, 'yarn') - - oozie = cluster.get_service(oozie_service_name) - process_service(oozie, 'oozie') - - hive = cluster.get_service(hive_service_name) - process_service(hive, 'hive') - - hue = cluster.get_service(hue_service_name) - process_service(hue, 'hue') - - spark = cluster.get_service(spark_service_name) - process_service(spark, 'spark') - - zookeeper = cluster.get_service(zookeeper_service_name) - process_service(zookeeper, 'zookeeper') - - hbase = cluster.get_service(hbase_service_name) - process_service(hbase, 'hbase') - - flume = cluster.get_service(flume_service_name) - process_service(flume, 'flume') - - sqoop = cluster.get_service(sqoop_service_name) - process_service(sqoop, 'sqoop') - - solr = cluster.get_service(solr_service_name) - process_service(solr, 'solr') - - ks_indexer = cluster.get_service(ks_indexer_service_name) - process_service(ks_indexer, 'ks_indexer') - - impala = cluster.get_service(impala_service_name) - process_service(impala, 'impala') - - sentry = cluster.get_service(sentry_service_name) - process_service(sentry, 'sentry') - - -if __name__ == '__main__': - main() diff --git a/sahara/plugins/cdh/v5_3_0/resources/cdh_config.sh b/sahara/plugins/cdh/v5_3_0/resources/cdh_config.sh deleted file mode 100755 index 4cfb7b7a..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/cdh_config.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -tox -evenv -- python $(dirname $0)/cdh_config.py $* diff --git a/sahara/plugins/cdh/v5_3_0/resources/create_hive_db.sql b/sahara/plugins/cdh/v5_3_0/resources/create_hive_db.sql deleted file mode 100644 index 7545eeb7..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/create_hive_db.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE ROLE hive LOGIN PASSWORD '%s'; -CREATE DATABASE metastore OWNER hive encoding 'UTF8'; -GRANT ALL PRIVILEGES ON DATABASE metastore TO hive; -COMMIT; diff --git a/sahara/plugins/cdh/v5_3_0/resources/create_sentry_db.sql b/sahara/plugins/cdh/v5_3_0/resources/create_sentry_db.sql deleted file mode 100644 index 30cb6ab3..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/create_sentry_db.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE ROLE sentry LOGIN PASSWORD '%s'; -CREATE DATABASE sentry OWNER sentry encoding 'UTF8'; -GRANT ALL PRIVILEGES ON DATABASE sentry TO sentry; -COMMIT; diff --git a/sahara/plugins/cdh/v5_3_0/resources/flume-agent.json b/sahara/plugins/cdh/v5_3_0/resources/flume-agent.json deleted file mode 100644 index 415d9663..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/flume-agent.json +++ /dev/null @@ -1,260 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Used to select an agent configuration to use from flume.conf. Multiple agents may share the same agent name, in which case they will be assigned the same agent configuration.", - "display_name": "Agent Name", - "name": "agent_name", - "value": "tier1" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "Text that goes into morphlines.conf file used by the Flume-NG Solr sink. The text goes verbatim into the config file except that $ZK_HOST is replaced by the ZooKeeper quorum of the Solr service.", - "display_name": "Morphlines File", - "name": "agent_morphlines_conf_file", - "value": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Application configuration file in HOCON format (Human-Optimized Config Object Notation). \n# HOCON syntax is defined at http://github.com/typesafehub/config/blob/master/HOCON.md\n# and also used by Akka (http://www.akka.io) and Play (http://www.playframework.org/).\n# For more examples see http://doc.akka.io/docs/akka/2.1.2/general/configuration.html\n\n# morphline.conf example file\n# this is a comment\n\n# Specify server locations in a SOLR_LOCATOR variable; used later in variable substitutions:\nSOLR_LOCATOR : {\n # Name of solr collection\n collection : collection1\n \n # ZooKeeper ensemble\n zkHost : \"$ZK_HOST\"\n \n # Relative or absolute path to a directory containing conf/solrconfig.xml and conf/schema.xml\n # If this path is uncommented it takes precedence over the configuration stored in ZooKeeper. \n # solrHomeDir : \"example/solr/collection1\"\n \n # The maximum number of documents to send to Solr per network batch (throughput knob)\n # batchSize : 100\n}\n\n# Specify an array of one or more morphlines, each of which defines an ETL \n# transformation chain. A morphline consists of one or more (potentially \n# nested) commands. A morphline is a way to consume records (e.g. Flume events, \n# HDFS files or blocks), turn them into a stream of records, and pipe the stream \n# of records through a set of easily configurable transformations on it's way to \n# Solr (or a MapReduceIndexerTool RecordWriter that feeds via a Reducer into Solr).\nmorphlines : [\n {\n # Name used to identify a morphline. E.g. used if there are multiple morphlines in a \n # morphline config file\n id : morphline1 \n \n # Import all morphline commands in these java packages and their subpackages.\n # Other commands that may be present on the classpath are not visible to this morphline.\n importCommands : [\"org.kitesdk.**\", \"org.apache.solr.**\"]\n \n commands : [ \n { \n # Parse Avro container file and emit a record for each avro object\n readAvroContainer {\n # Optionally, require the input record to match one of these MIME types:\n # supportedMimeTypes : [avro/binary]\n \n # Optionally, use a custom Avro schema in JSON format inline:\n # schemaString : \"\"\"$KERBEROS_PRINCIPAL\" and \"$KERBEROS_KEYTAB\", which will be replaced by the principal name and the keytab path respectively.",
- "display_name": "Configuration File",
- "name": "agent_config_file",
- "value": "# Please paste flume.conf here. Example:\n\n# Sources, channels, and sinks are defined per\n# agent name, in this case 'tier1'.\ntier1.sources = source1\ntier1.channels = channel1\ntier1.sinks = sink1\n\n# For each source, channel, and sink, set\n# standard properties.\ntier1.sources.source1.type = netcat\ntier1.sources.source1.bind = 127.0.0.1\ntier1.sources.source1.port = 9999\ntier1.sources.source1.channels = channel1\ntier1.channels.channel1.type = memory\ntier1.sinks.sink1.type = logger\ntier1.sinks.sink1.channel = channel1\n\n# Other properties are specific to each type of\n# source, channel, or sink. In this case, we\n# specify the capacity of the memory channel.\ntier1.channels.channel1.capacity = 100\n"
- },
- {
- "desc": "Whether or not periodic stacks collection is enabled.",
- "display_name": "Stacks Collection Enabled",
- "name": "stacks_collection_enabled",
- "value": "false"
- },
- {
- "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
- "display_name": "Automatically Restart Process",
- "name": "process_auto_restart",
- "value": "true"
- },
- {
- "desc": "The maximum number of rolled log files to keep for Agent logs. Typically used by log4j.",
- "display_name": "Agent Maximum Log File Backups",
- "name": "max_log_backup_index",
- "value": "10"
- },
- {
- "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Enables the health test that the Agent's process state is consistent with the role configuration", - "display_name": "Agent Process Health Test", - "name": "flume_agent_scm_health_enabled", - "value": "true" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "flume_agent_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The port on which the Flume web server listens for requests.", - "display_name": "HTTP Port", - "name": "agent_http_port", - "value": "41414" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Agent Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Agent in Bytes", - "name": "agent_java_heapsize", - "value": "1073741824" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Disables import of ZooKeeper configuration from the HBase classpath. This prevents zoo.cfg from overriding hbase-site.xml for Zookeeper quorum information. This option is only supported on CDH 4.4 or later deployments.", - "display_name": "HBase sink prefer hbase-site.xml over Zookeeper config", - "name": "agent_disable_zoo_cfg", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Agent logs. Typically used by log4j.", - "display_name": "Agent Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "Directory where Flume Agent will place its log files.", - "display_name": "Flume Agent Log Directory", - "name": "flume_agent_log_dir", - "value": "/var/log/flume-ng" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Text that goes verbatim into custom-mimetypes.xml file used by the Flume-NG Solr sink.", - "display_name": "Custom Mime-types File", - "name": "agent_custom_mimetypes_file", - "value": "\n\n\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "Name of the HDFS service that this Flume service instance depends on", - "display_name": "HDFS Service", - "name": "hdfs_service", - "value": null - }, - { - "desc": "The health test thresholds of the overall Agent health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Agents falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Agents falls below the critical threshold.", - "display_name": "Healthy Agent Monitoring Thresholds", - "name": "flume_agents_healthy_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"95.0\"}" - }, - { - "desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.", - "display_name": "Enable Log Event Capture", - "name": "catch_events", - "value": "true" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hbase-gateway.json b/sahara/plugins/cdh/v5_3_0/resources/hbase-gateway.json deleted file mode 100644 index 8662040d..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hbase-gateway.json +++ /dev/null @@ -1,44 +0,0 @@ -[ - { - "desc": "For advanced use only, a string to be inserted into the client configuration for hbase-site.xml.", - "display_name": "HBase Client Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", - "name": "hbase_client_config_safety_valve", - "value": null - }, - { - "desc": "The directory where the client configs will be deployed", - "display_name": "Deploy Directory", - "name": "client_config_root_dir", - "value": "/etc/hbase" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the client configuration for hbase-env.sh", - "display_name": "HBase Client Environment Advanced Configuration Snippet for hbase-env.sh (Safety Valve)", - "name": "hbase_client_env_safety_valve", - "value": null - }, - { - "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", - "display_name": "Alternatives Priority", - "name": "client_config_priority", - "value": "90" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "These are Java command line arguments. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Client Java Configuration Options", - "name": "hbase_client_java_opts", - "value": "-XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:+CMSIncrementalMode -Djava.net.preferIPv4Stack=true" - }, - { - "desc": "Maximum size in bytes for the Java process heap memory. Passed to Java -Xmx.", - "display_name": "Client Java Heap Size in Bytes", - "name": "hbase_client_java_heapsize", - "value": "268435456" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hbase-hbaserestserver.json b/sahara/plugins/cdh/v5_3_0/resources/hbase-hbaserestserver.json deleted file mode 100644 index a1e47599..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hbase-hbaserestserver.json +++ /dev/null @@ -1,254 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for HBase REST Server", - "name": "hbase_restserver_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Maximum size of the HBase REST Server thread pool. The server can process this number of concurrent requests. Setting this too high can lead to out of memory errors.", - "display_name": "HBase REST Server Maximum Threads", - "name": "hbase_restserver_threads_max", - "value": "100" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "hbaserestserver_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", - "display_name": "HBase REST Server Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", - "name": "hbase_restserver_config_safety_valve", - "value": null - }, - { - "desc": "Directory where HBase REST Server will place its log files.", - "display_name": "HBase REST Server Log Directory", - "name": "hbase_restserver_log_dir", - "value": "/var/log/hbase" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for HBase REST Server logs. Typically used by log4j.", - "display_name": "HBase REST Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of HBase REST Server in Bytes", - "name": "hbase_restserver_java_heapsize", - "value": "1073741824" - }, - { - "desc": "Enables the health test that the HBase REST Server's process state is consistent with the role configuration", - "display_name": "HBase REST Server Process Health Test", - "name": "hbaserestserver_scm_health_enabled", - "value": "true" - }, - { - "desc": "The port that HBase REST Server binds to.", - "display_name": "HBase REST Server Port", - "name": "hbase_restserver_port", - "value": "20550" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "If true, HBase REST Server Web UI will bind to a wildcard address (0.0.0.0). Otherwise it will bind to a host name. Only available in CDH 4.3 and later.", - "display_name": "HBase REST Server Web UI Bind to Wildcard Address", - "name": "hbase_restserver_info_bind_to_wildcard", - "value": "true" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "HBase REST Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "When false, all HTTP methods are permitted (GET/PUT/POST/DELETE). When true, only GET is permitted.", - "display_name": "Enable HBase REST Server Read Only Mode", - "name": "hbase_restserver_readonly", - "value": "false" - }, - { - "desc": "The port that HBase REST Server Web UI binds to.", - "display_name": "HBase REST Server Web UI Port", - "name": "hbase_restserver_info_port", - "value": "8085" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When computing the overall HBase REST Server health, consider the host's health.", - "display_name": "HBase REST Server Host Health Test", - "name": "hbaserestserver_host_health_enabled", - "value": "true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for HBase REST Server logs. Typically used by log4j.", - "display_name": "HBase REST Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The host name or IP address of the DNS name server which an HBase REST Server should use to determine the host name used for communication and display purposes.", - "display_name": "HBase REST Server DNS Name Server", - "name": "hbase_restserver_dns_nameserver", - "value": null - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "HBase REST Server will bind to this address.", - "display_name": "HBase REST Server Host Address", - "name": "hbase_restserver_host", - "value": "0.0.0.0" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The name of the DNS network interface from which an HBase REST Server should report its IP address.", - "display_name": "HBase REST Server DNS Network Interface", - "name": "hbase_restserver_dns_interface", - "value": null - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Minimum size of the HBase REST Server thread pool. The server will maintain at least this number of threads in the pool at all times. The thread pool can grow up to the maximum size set by hbase.rest.threads.max.", - "display_name": "HBase REST Server Minimum Threads", - "name": "hbase_restserver_threads_min", - "value": "2" - }, - { - "desc": "The minimum log level for HBase REST Server logs", - "display_name": "HBase REST Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hbase-hbasethriftserver.json b/sahara/plugins/cdh/v5_3_0/resources/hbase-hbasethriftserver.json deleted file mode 100644 index 78299a29..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hbase-hbasethriftserver.json +++ /dev/null @@ -1,260 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "The \"core size\" of the thread pool. New threads are created on every connection until this many threads are created.", - "display_name": "HBase Thrift Server Min Worker Threads", - "name": "hbase_thriftserver_min_worker_threads", - "value": "200" - }, - { - "desc": "The port that HBase Thrift Server binds to.", - "display_name": "HBase Thrift Server Port", - "name": "hbase_thriftserver_port", - "value": "9090" - }, - { - "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", - "display_name": "HBase Thrift Server Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", - "name": "hbase_thriftserver_config_safety_valve", - "value": null - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for HBase Thrift Server logs. Typically used by log4j.", - "display_name": "HBase Thrift Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of HBase Thrift Server in Bytes", - "name": "hbase_thriftserver_java_heapsize", - "value": "1073741824" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "If true, HBase Thrift Server Web UI will bind to a wildcard address (0.0.0.0). Otherwise it will bind to a host name. Only available in CDH 4.3 and later.", - "display_name": "HBase Thrift Server Web UI Bind to Wildcard Address", - "name": "hbase_thriftserver_info_bind_to_wildcard", - "value": "true" - }, - { - "desc": "Address to bind the HBase Thrift Server to. When using the THsHaServer or the TNonblockingServer, always binds to 0.0.0.0 irrespective of this configuration value.", - "display_name": "HBase Thrift Server Bind Address", - "name": "hbase_thriftserver_bindaddress", - "value": "0.0.0.0" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "Type of HBase Thrift Server.", - "display_name": "HBase Thrift Server Type", - "name": "hbase_thriftserver_type", - "value": "threadpool" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "HBase Thrift Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "When computing the overall HBase Thrift Server health, consider the host's health.", - "display_name": "HBase Thrift Server Host Health Test", - "name": "hbasethriftserver_host_health_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Enables the health test that the HBase Thrift Server's process state is consistent with the role configuration", - "display_name": "HBase Thrift Server Process Health Test", - "name": "hbasethriftserver_scm_health_enabled", - "value": "true" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for HBase Thrift Server", - "name": "hbase_thriftserver_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "Use framed transport. When using the THsHaServer or TNonblockingServer, framed transport is always used irrespective of this configuration value.", - "display_name": "Enable HBase Thrift Server Framed Transport", - "name": "hbase_thriftserver_framed", - "value": "false" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for HBase Thrift Server logs. Typically used by log4j.", - "display_name": "HBase Thrift Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "The port that HBase Thrift Server Web UI binds to.", - "display_name": "HBase Thrift Server Web UI Port", - "name": "hbase_thriftserver_info_port", - "value": "9095" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The host name or IP address of the DNS name server which an HBase Thrift Server should use to determine the host name used for communication and display purposes.", - "display_name": "HBase Thrift Server DNS Name Server", - "name": "hbase_thriftserver_dns_nameserver", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The name of the DNS network interface from which an HBase Thrift Server should report its IP address.", - "display_name": "HBase Thrift Server DNS Network Interface", - "name": "hbase_thriftserver_dns_interface", - "value": null - }, - { - "desc": "Use the TCompactProtocol instead of the default TBinaryProtocol. TCompactProtocol is a binary protocol that is more compact than the default and typically more efficient.", - "display_name": "Enable HBase Thrift Server Compact Protocol", - "name": "hbase_thriftserver_compact", - "value": "false" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Directory where HBase Thrift Server will place its log files.", - "display_name": "HBase Thrift Server Log Directory", - "name": "hbase_thriftserver_log_dir", - "value": "/var/log/hbase" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "hbasethriftserver_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The minimum log level for HBase Thrift Server logs", - "display_name": "HBase Thrift Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hbase-master.json b/sahara/plugins/cdh/v5_3_0/resources/hbase-master.json deleted file mode 100644 index 9dbfd71f..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hbase-master.json +++ /dev/null @@ -1,326 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"IPC Server handler.*ClosedChannelException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"IPC Server Responder, call.*output error\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Daughter regiondir does not exist: .*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"File.*might still be open.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"File.*might still be open.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Moving table .+ state to enabled but was already enabled\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Received OPENED for region.*but region was in the state.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", - "display_name": "Health Check Startup Tolerance", - "name": "master_startup_tolerance", - "value": "5" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Number of pooled threads to handle region closing in the master.", - "display_name": "Region Closing Threads", - "name": "hbase_master_executor_closeregion_threads", - "value": "5" - }, - { - "desc": "When computing the overall Master health, consider the host's health.", - "display_name": "Master Host Health Test", - "name": "master_host_health_enabled", - "value": "true" - }, - { - "desc": "Number of pooled threads to handle the recovery of the region servers in the master.", - "display_name": "RegionServer Recovery Threads", - "name": "hbase_master_executor_serverops_threads", - "value": "5" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Directory where Master will place its log files.", - "display_name": "Master Log Directory", - "name": "hbase_master_log_dir", - "value": "/var/log/hbase" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "master_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Time period in seconds to reset long-running metrics (e.g. compactions). This is an HBase specific configuration.", - "display_name": "Extended Period", - "name": "hbase_metrics_extended_period", - "value": "3600" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Master Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "master_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The port that the HBase Master binds to.", - "display_name": "HBase Master Port", - "name": "hbase_master_port", - "value": "60000" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "master_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "List of org.apache.hadoop.hbase.coprocessor.MasterObserver coprocessors that are loaded by default on the active HMaster process. For any implemented coprocessor methods, the listed classes will be called in order. After implementing your own MasterObserver, just put it in HBase's classpath and add the fully qualified class name here.", - "display_name": "HBase Coprocessor Master Classes", - "name": "hbase_coprocessor_master_classes", - "value": "" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of HBase Master in Bytes", - "name": "hbase_master_java_heapsize", - "value": "1073741824" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "master_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "When true, HBase Master will bind to 0.0.0.0. Only available with CDH 4.3 and later.", - "display_name": "HBase Master Bind to Wildcard Address", - "name": "hbase_master_bind_to_wildcard_address", - "value": "true" - }, - { - "desc": "Enables the health test that a client can connect to the HBase Master", - "display_name": "HBase Master Canary Health Test", - "name": "master_canary_health_enabled", - "value": "true" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The port for the HBase Master web UI. Set to -1 to disable the HBase Master web UI.", - "display_name": "HBase Master Web UI Port", - "name": "hbase_master_info_port", - "value": "60010" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "master_gc_duration_window", - "value": "5" - }, - { - "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", - "display_name": "Master Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", - "name": "hbase_master_config_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for HBase Master", - "name": "hbase_master_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Master logs. Typically used by log4j.", - "display_name": "Master Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "Maximum time an HLog remains in the .oldlogdir directory until an HBase Master thread deletes it.", - "display_name": "Maximum Time to Keep HLogs", - "name": "hbase_master_logcleaner_ttl", - "value": "60000" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Number of RPC Server instances spun up on HBase Master.", - "display_name": "HBase Master Handler Count", - "name": "hbase_master_handler_count", - "value": "25" - }, - { - "desc": "The address for the HBase Master web UI", - "display_name": "HBase Master Web UI Address", - "name": "hbase_master_info_bindAddress", - "value": null - }, - { - "desc": "A comma-separated list of LogCleanerDelegate(s) that are used in LogsCleaner. WAL/HLog cleaner(s) are called in order, so put the log cleaner that prunes the most log files in the front. To implement your own LogCleanerDelegate, add it to HBase's classpath and add the fully-qualified class name here. You should always add the above default log cleaners in the list, unless you have a special reason not to.", - "display_name": "HBase Master Log Cleaner Plugins", - "name": "hbase_master_logcleaner_plugins", - "value": null - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "The maximum number of rolled log files to keep for Master logs. Typically used by log4j.", - "display_name": "Master Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The host name or IP address of the DNS name server which an HBase Master should use to determine the host name used for communication and display purposes.", - "display_name": "HBase Master DNS Name Server", - "name": "hbase_master_dns_nameserver", - "value": null - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "Number of pooled threads to handle region opening in the master.", - "display_name": "Region Opening Threads", - "name": "hbase_master_executor_openregion_threads", - "value": "5" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The name of the DNS network interface from which an HBase Master should report its IP address.", - "display_name": "HBase Master DNS Network Interface", - "name": "hbase_master_dns_interface", - "value": null - }, - { - "desc": "The minimum log level for Master logs", - "display_name": "Master Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Enables the health test that the Master's process state is consistent with the role configuration", - "display_name": "Master Process Health Test", - "name": "master_scm_health_enabled", - "value": "true" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hbase-regionserver.json b/sahara/plugins/cdh/v5_3_0/resources/hbase-regionserver.json deleted file mode 100644 index e64baae2..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hbase-regionserver.json +++ /dev/null @@ -1,548 +0,0 @@ -[ - { - "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", - "display_name": "RegionServer Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", - "name": "hbase_regionserver_config_safety_valve", - "value": null - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Maximum number of HStoreFiles to compact per minor compaction.", - "display_name": "Maximum Number of HStoreFiles Compaction", - "name": "hbase_hstore_compaction_max", - "value": null - }, - { - "desc": "The address for the HBase RegionServer web UI", - "display_name": "HBase RegionServer Web UI Address", - "name": "hbase_regionserver_info_bindAddress", - "value": null - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "Number of threads to use while loading and unloading regions to or from a RegionServer. Can be used to increase the speed of decommissioning or rolling restart operations.", - "display_name": "Region Mover Threads", - "name": "hbase_regionserver_regionmover_thread_count", - "value": "1" - }, - { - "desc": "The health test thresholds of the average size of the HBase RegionServer flush queue over a recent period. See HBase RegionServer Flush Queue Monitoring Period.", - "display_name": "HBase RegionServer Flush Queue Monitoring Thresholds", - "name": "regionserver_flush_queue_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10.0\"}" - }, - { - "desc": "The amount of time to wait for the HBase Region Server to fully start up and connect to the HBase Master before enforcing the connectivity check.", - "display_name": "HBase Region Server Connectivity Tolerance at Startup", - "name": "regionserver_connectivity_tolerance", - "value": "180" - }, - { - "desc": "The hashing algorithm for use in HashFunction. Two values are supported: 'murmur' (for MurmurHash) and 'jenkins' (for JenkinsHash).", - "display_name": "HBase Hash Type", - "name": "hbase_hash_type", - "value": "murmur" - }, - { - "desc": "The port for the HBase RegionServer web UI. Set to -1 to disable RegionServer web UI.", - "display_name": "HBase RegionServer Web UI port", - "name": "hbase_regionserver_info_port", - "value": "60030" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "When memstores are being forced to flush to make room in memory, keep flushing until this amount is reached. If this amount is equal to 'hbase.regionserver.global.memstore.upperLimit', then minimum possible flushing will occur when updates are blocked due to memstore limiting.", - "display_name": "Low Watermark for Memstore Flush", - "name": "hbase_regionserver_global_memstore_lowerLimit", - "value": "0.38" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for HBase RegionServer", - "name": "hbase_regionserver_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "RegionServer Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds of the latency that the RegionServer sees for HDFS read operations", - "display_name": "HBase RegionServer HDFS Read Latency Thresholds", - "name": "regionserver_read_latency_thresholds", - "value": "{\"critical\":\"100.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds of the size used by the HStoreFile index. Specified as a percentage of the total heap size.", - "display_name": "Percentage of Heap Used by HStoreFile Index", - "name": "regionserver_store_file_idx_size_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10.0\"}" - }, - { - "desc": "Maximum number of Write-Ahead Log (WAL) files. This value multiplied by HDFS Block Size (dfs.blocksize) is the size of the WAL that will need to be replayed when a server crashes. This value is inversely proportional to the frequency of flushes to disk.", - "display_name": "Maximum number of Write-Ahead Log (WAL) files", - "name": "hbase_regionserver_maxlogs", - "value": "32" - }, - { - "desc": "List of coprocessors that are loaded by default on all tables. For any override coprocessor method, these classes will be called in order. After implementing your own coprocessor, just put it in HBase's classpath and add the fully qualified class name here. A coprocessor can also be loaded on demand by setting HTableDescriptor.", - "display_name": "HBase Coprocessor Region Classes", - "name": "hbase_coprocessor_region_classes", - "value": "" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "regionserver_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "Blocks writes if the size of the memstore increases to the value of 'hbase.hregion.block.memstore' multiplied by the value of 'hbase.hregion.flush.size' bytes. This setting is useful for preventing runaway memstore during spikes in update traffic. Without an upper-bound, memstore fills such that when it flushes, the resultant process of flushing files take a long time to compact or split, or worse, an \"out of memory\" error occurs.", - "display_name": "HBase Memstore Block Multiplier", - "name": "hbase_hregion_memstore_block_multiplier", - "value": "2" - }, - { - "desc": "Sync the HLog to HDFS after this interval, in milliseconds, if it has not accumulated the number of HLog Entries specified to trigger a sync.", - "display_name": "Synch Interval of HLog Entries", - "name": "hbase_regionserver_optionallogflushinterval", - "value": "1000" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \"IPC Server handler.*took.*appending an edit to hlog.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"ABORTING region server serverName.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"DFSOutputStream ResponseProcessor exception.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Error Recovery for block blk.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Failed init\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Problem renewing lease for DFSClient.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"remote error telling master we are up\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Session.*for server.*closing socket connection and attempting reconnect.\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Starting .*compaction on region (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_REGION_STARTED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^completed compaction on region (.+) after (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_REGION_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Starting compaction on (.+) in region (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_COLUMN_FAMILY_STARTED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"COLUMN_FAMILY\", \"group1\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^completed compaction: regionName=(.+), storeName=(.+), fileCount=(.+), fileSize=(.+), priority=(.+), time=(.+); duration=(.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_COLUMN_FAMILY_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"COLUMN_FAMILY\", \"group2\": \"FILE_COUNT\", \"group3\": \"FILE_SIZE\", \"group4\": \"PRIORITY\", \"group5\": \"REQUEST_TIME_NANOS\", \"group6\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Completed compaction: Request = regionName=(.+), storeName=(.+), fileCount=(.+), fileSize=(.+), priority=(.+), time=(.+); duration=(.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_COLUMN_FAMILY_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"COLUMN_FAMILY\", \"group2\": \"FILE_COUNT\", \"group3\": \"FILE_SIZE\", \"group4\": \"PRIORITY\", \"group5\": \"REQUEST_TIME_NANOS\", \"group6\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^aborted compaction: regionName=(.+), storeName=(.+), fileCount=(.+), fileSize=(.+), priority=(.+), time=(.+); duration=(.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_COLUMN_FAMILY_ABORTED\", \"attribute:SEVERITY\": \"IMPORTANT\", \"group0\": \"REGION\", \"group1\": \"COLUMN_FAMILY\", \"group2\": \"FILE_COUNT\", \"group3\": \"FILE_SIZE\", \"group4\": \"PRIORITY\", \"group5\": \"REQUEST_TIME_NANOS\", \"group6\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Finished memstore flush of .+ for region (.+) in (.+), sequenceid=(.+), compaction requested=(.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_FLUSH_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"DURATION\", \"group2\": \"SEQUENCE_ID\", \"group3\": \"COMPACTION_REQUESTED\" },\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Flush of region (.+) due to global heap pressure$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_FLUSH_DUE_TO_HEAP_PRESSURE\", \"attribute:SEVERITY\": \"IMPORTANT\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"WARN\", \"content\": \"^Region (.+) has too many store files; delaying flush up to 90000ms$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_FLUSH_DELAYED_TOO_MANY_STORE_FILES\", \"attribute:SEVERITY\": \"CRITICAL\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Starting split of region (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_SPLIT_STARTED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Running rollback/cleanup of failed split of (.+);.+$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_SPLIT_ABORTED\", \"attribute:SEVERITY\": \"IMPORTANT\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Region split, hbase:meta updated, and report to master. Parent=(.+), new regions: (.+,.*,.+), (.+,.*,.+). Split took (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_SPLIT_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"DAUGHTER_REGIONS\", \"group2\": \"DAUGHTER_REGIONS\", \"group3\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Region split, META updated, and report to master. Parent=(.+), new regions: (.+,.*,.+), (.+,.*,.+). Split took (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_SPLIT_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"DAUGHTER_REGIONS\", \"group2\": \"DAUGHTER_REGIONS\", \"group3\": \"DURATION\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" - }, - { - "desc": "The host name or IP address of the DNS name server which a RegionServer should use to determine the host name used by the HBase Master for communication and display purposes.", - "display_name": "RegionServer DNS Nameserver", - "name": "hbase_regionserver_dns_nameserver", - "value": null - }, - { - "desc": "The health test thresholds of the total size of RegionServer's memstores. Specified as a percentage of the configured upper limit. See Maximum Size of All Memstores in RegionServer.", - "display_name": "HBase RegionServer Memstore Size Thresholds", - "name": "regionserver_memstore_size_thresholds", - "value": "{\"critical\":\"100.0\",\"warning\":\"95.0\"}" - }, - { - "desc": "Percentage of maximum heap (-Xmx setting) to allocate to block cache used by HFile/StoreFile. To disable, set this value to 0 .", - "display_name": "HFile Block Cache Size", - "name": "hfile_block_cache_size", - "value": "0.4" - }, - { - "desc": "The maximum size, in megabytes, per log file for RegionServer logs. Typically used by log4j.", - "display_name": "RegionServer Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Memstore will be flushed to disk if size of the memstore exceeds this value in number of bytes. This value is checked by a thread that runs the frequency specified by hbase.server.thread.wakefrequency.", - "display_name": "HBase Memstore Flush Size", - "name": "hbase_hregion_memstore_flush_size", - "value": "134217728" - }, - { - "desc": "The HLog file reader implementation.", - "display_name": "HLog Reader Implementation", - "name": "hbase_regionserver_hlog_reader_impl", - "value": null - }, - { - "desc": "If this number of HStoreFiles in any one HStore is exceeded, then a compaction is run to rewrite all HStoreFiles files as one HStoreFile. (One HStoreFile is written per flush of memstore.) You can delay compaction by specifying a larger number, but the compaction will take longer when it does run. During a compaction, updates cannot be flushed to disk. Long compactions require memory sufficient to carry the logging of all updates across the duration of the compaction. If too large, clients timeout during compaction.", - "display_name": "HStore Compaction Threshold", - "name": "hbase_hstore_compactionThreshold", - "value": "3" - }, - { - "desc": "The period over which to compute the moving average of the HDFS sync latency of the HBase RegionServer.", - "display_name": "HBase RegionServer HDFS Sync Latency Monitoring Period", - "name": "regionserver_sync_latency_window", - "value": "5" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The maximum size byte array that should come from the MSLAB, in bytes.", - "display_name": "Maximum Byte Array from MSLAB Allocation Scheme", - "name": "hbase_hregion_memstore_mslab_max_allocation", - "value": "262144" - }, - { - "desc": "The health test thresholds for the latency of HDFS write operations that the RegionServer detects", - "display_name": "HBase RegionServer HDFS Sync Latency Thresholds", - "name": "regionserver_sync_latency_thresholds", - "value": "{\"critical\":\"5000.0\",\"warning\":\"500.0\"}" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Directory where RegionServer will place its log files.", - "display_name": "RegionServer Log Directory", - "name": "hbase_regionserver_log_dir", - "value": "/var/log/hbase" - }, - { - "desc": "The period of time that an HRegion will block updates after reaching the HStoreFile limit that is specified by 'hbase.hstore.blockingStoreFiles'. After this time has elapsed, the HRegion will stop blocking updates even if a compaction has not been completed.", - "display_name": "HStore Blocking Wait Time", - "name": "hbase_hstore_blockingWaitTime", - "value": "90000" - }, - { - "desc": "Timeout for Canary to perform its checks.", - "display_name": "Canary Timeout", - "name": "hbase_regionserver_canary_timeout", - "value": "15" - }, - { - "desc": "Duration between consecutive checks done by the Canary.", - "display_name": "Canary Interval", - "name": "hbase_regionserver_canary_interval", - "value": "6" - }, - { - "desc": "The lease period, in milliseconds, for the HBase RegionServer. Clients must report in within this period or else they are considered dead.", - "display_name": "HBase RegionServer Lease Period", - "name": "hbase_regionserver_lease_period", - "value": "60000" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "The minimum log level for RegionServer logs", - "display_name": "RegionServer Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "regionserver_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "The name of the DNS Network Interface from which a RegionServer should report its IP address.", - "display_name": "RegionServer DNS Network Interface", - "name": "hbase_regionserver_dns_interface", - "value": null - }, - { - "desc": "The number of reservation blocks that are used to prevent unstable RegionServers caused by an OOME.", - "display_name": "RegionServer Reservation Blocks", - "name": "hbase_regionserver_nbreservationblocks", - "value": "4" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "regionserver_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The size of the chunks allocated by MSLAB, in bytes.", - "display_name": "Chunk Size Allocated by MSLAB Allocation Scheme", - "name": "hbase_hregion_memstore_mslab_chunksize", - "value": "2097152" - }, - { - "desc": "If the memstores in a region are this size or larger when closing, run a pre-flush process to clear out memstores before putting up the region closed flag and taking the region offline. On close, a flush process is run under the close flag up to empty memory. During this time, the region is offline and no writes are taken. If the memstore content is large, the flush process could take a long time to complete. The pre-flush process cleans out the bulk of the memstore before putting up the close flag and taking the region offline, so that the flush process that runs under the close flag has little to do.", - "display_name": "HBase Memstore Pre-close Flush Size", - "name": "hbase_hregion_preclose_flush_size", - "value": "5242880" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The period over which to compute the moving average of the HDFS read latency of the HBase RegionServer.", - "display_name": "HBase RegionServer HDFS Read Latency Monitoring Period", - "name": "regionserver_read_latency_window", - "value": "5" - }, - { - "desc": "The directory for HBase secure bulk Load.", - "display_name": "HBase Secure Bulk Load Directory", - "name": "hbase_bulkload_staging_dir", - "value": "/tmp/hbase-staging" - }, - { - "desc": "An interface that is assignable to HRegionInterface. Used in HBase Client for opening a proxy to a remote HBase RegionServer.", - "display_name": "HBase RegionServer Interface Class", - "name": "hbase_regionserver_class", - "value": null - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "When true, HBase RegionServer will bind to 0.0.0.0. Only available in CDH 4.3 and later.", - "display_name": "HBase RegionServer Bind to Wildcard Address", - "name": "hbase_regionserver_bind_to_wildcard_address", - "value": "true" - }, - { - "desc": "The time, in milliseconds, between 'major' compactions of all HStoreFiles in a region. To disable automated major compactions, set this value to 0.", - "display_name": "HBase HRegion Major Compaction", - "name": "hbase_hregion_majorcompaction", - "value": "604800000" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Enable MemStore-Local Allocation Buffer (MSLAB) Allocation Scheme. Note: This feature is experimental in CDH3.", - "display_name": "Enable MSLAB Allocation Scheme", - "name": "hbase_hregion_memstore_mslab_enabled", - "value": "true" - }, - { - "desc": "Enables the health test that the RegionServer is connected to the Master", - "display_name": "HBase RegionServer to Master Connectivity Test", - "name": "regionserver_master_connectivity_enabled", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": "32768" - }, - { - "desc": "Limit for the number of regions after which no more region splitting should take place. This is not a hard limit for the number of regions but acts as a guideline for the RegionServer to stop splitting after a certain limit.", - "display_name": "HBase Region Split Limit", - "name": "hbase_regionserver_regionSplitLimit", - "value": "2147483647" - }, - { - "desc": "Enables the health test that the RegionServer's process state is consistent with the role configuration", - "display_name": "RegionServer Process Health Test", - "name": "regionserver_scm_health_enabled", - "value": "true" - }, - { - "desc": "Interval, in milliseconds, between messages from the RegionServer to the HBase Master. Use a high value such as 3000 for clusters that have more than 10 hosts.", - "display_name": "HBase RegionServer Message Interval", - "name": "hbase_regionserver_msginterval", - "value": "3000" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The period over which to compute the moving average of the compaction queue size.", - "display_name": "HBase RegionServer Compaction Queue Monitoring Period", - "name": "regionserver_compaction_queue_window", - "value": "5" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "regionserver_gc_duration_window", - "value": "5" - }, - { - "desc": "Time period in seconds to reset long-running metrics (e.g. compactions). This is an HBase specific configuration.", - "display_name": "Extended Period", - "name": "hbase_metrics_extended_period", - "value": "3600" - }, - { - "desc": "Comma-separated list of codecs that the RegionServer requires to start. Use this setting to make sure that all RegionServers joining a cluster are installed with a particular set of codecs.", - "display_name": "RegionServer Codecs", - "name": "hbase_regionserver_codecs", - "value": "" - }, - { - "desc": "Maximum size of all memstores in a RegionServer before new updates are blocked and flushes are forced.", - "display_name": "Maximum Size of All Memstores in RegionServer", - "name": "hbase_regionserver_global_memstore_upperLimit", - "value": "0.4" - }, - { - "desc": "Number of handlers for processing priority requests in a RegionServer.", - "display_name": "HBase RegionServer Meta-Handler Count", - "name": "hbase_regionserver_metahandler_count", - "value": "10" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Period, in milliseconds, at which to roll the commit log.", - "display_name": "HBase RegionServer Log Roll Period", - "name": "hbase_regionserver_logroll_period", - "value": "3600000" - }, - { - "desc": "The period over which to compute the moving average of the flush queue size.", - "display_name": "HBase RegionServer Flush Queue Monitoring Period", - "name": "regionserver_flush_queue_window", - "value": "5" - }, - { - "desc": "Maximum HStoreFile size. If any one of a column families' HStoreFiles has grown to exceed this value, the hosting HRegion is split in two.", - "display_name": "HBase Maximum File Size", - "name": "hbase_hregion_max_filesize", - "value": "10737418240" - }, - { - "desc": "The health test thresholds of the weighted average size of the HBase RegionServer compaction queue over a recent period. See HBase RegionServer Compaction Queue Monitoring Period.", - "display_name": "HBase RegionServer Compaction Queue Monitoring Thresholds", - "name": "regionserver_compaction_queue_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10.0\"}" - }, - { - "desc": "If there are more than this number of HStoreFiles in any one HStore, then updates are blocked for this HRegion until a compaction is completed, or until the value specified for 'hbase.hstore.blockingWaitTime' has been exceeded.", - "display_name": "HStore Blocking Store Files", - "name": "hbase_hstore_blockingStoreFiles", - "value": "10" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "regionserver_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "The maximum number of rolled log files to keep for RegionServer logs. Typically used by log4j.", - "display_name": "RegionServer Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "The HLog file writer implementation.", - "display_name": "HLog Writer Implementation", - "name": "hbase_regionserver_hlog_writer_impl", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The port that an HBase RegionServer binds to.", - "display_name": "HBase RegionServer Port", - "name": "hbase_regionserver_port", - "value": "60020" - }, - { - "desc": "Number of RPC Server instances spun up on RegionServers.", - "display_name": "HBase RegionServer Handler Count", - "name": "hbase_regionserver_handler_count", - "value": "30" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of HBase RegionServer in Bytes", - "name": "hbase_regionserver_java_heapsize", - "value": "4294967296" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "When computing the overall RegionServer health, consider the host's health.", - "display_name": "RegionServer Host Health Test", - "name": "regionserver_host_health_enabled", - "value": "true" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hbase-service.json b/sahara/plugins/cdh/v5_3_0/resources/hbase-service.json deleted file mode 100644 index 3bc56e65..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hbase-service.json +++ /dev/null @@ -1,482 +0,0 @@ -[ - { - "desc": "Comma-delimited list of hosts where you want to allow the HBase user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that does not correspond to a host name, such as '_no_host'. Note: This property is used only if HBase REST/Thrift Server Authentication is enabled.", - "display_name": "HBase Proxy User Hosts", - "name": "hbase_proxy_user_hosts_list", - "value": "*" - }, - { - "desc": "The user that this service's processes should run as.", - "display_name": "System User", - "name": "process_username", - "value": "hbase" - }, - { - "desc": "The frequency in which the log4j event publication appender will retry sending undelivered log events to the Event server, in seconds", - "display_name": "Log Event Retry Frequency", - "name": "log_event_retry_frequency", - "value": "30" - }, - { - "desc": "A general client pause time value. Used mostly as a time period to wait before retrying operations such as a failed get or region lookup.", - "display_name": "HBase Client Pause", - "name": "hbase_client_pause", - "value": "100" - }, - { - "desc": "Comma-delimited list of groups that you want to allow the HBase user to impersonate. The default '*' allows all groups. To disable entirely, use a string that does not correspond to a group name, such as '_no_group_'. Note: This property is used only if HBase REST/Thrift Server Authentication is enabled.", - "display_name": "HBase Proxy User Groups", - "name": "hbase_proxy_user_groups_list", - "value": "*" - }, - { - "desc": "Allow indexing of tables in HBase by Lily HBase Indexer. Note: Replication must be enabled for indexing to work.", - "display_name": "Enable Indexing", - "name": "hbase_enable_indexing", - "value": "false" - }, - { - "desc": "\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n
\n\n\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n
\n\nEvent trackers are defined in a JSON object like the following:
\n\n\n{\n \"timeToLive\" : [integer],\n \"fields\" : [\n {\n \"type\" : [string],\n \"name\" : [string]\n }\n ]\n}\n\n\n\nWhere:\n
\n\n\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n
\n\n\nThe following is the list of fields that can used to compare HBase events:\n
\nEvent filters are defined in a JSON object like the following:
\n\n\n{\n \"defaultAction\" : (\"accept\", \"discard\"),\n \"rules\" : [\n {\n \"action\" : (\"accept\", \"discard\"),\n \"fields\" : [\n {\n \"name\" : \"fieldName\",\n \"match\" : \"regex\"\n }\n ]\n }\n ]\n}\n\n\n\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n
\n\n\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n
\n\n\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n
\n\n\nThe following is the list of fields that can be filtered for HBase events:\n
\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "An alert is published if the HBase region health canary runs slowly.", - "display_name": "HBase Region Health Canary Slow Run Alert Enabled", - "name": "hbase_region_health_canary_slow_run_alert_enabled", - "value": "true" - }, - { - "desc": "Set to true to cause the hosting server (Master or RegionServer) to abort if a coprocessor throws a Throwable object that is not IOException or a subclass of IOException. Setting it to true might be useful in development environments where one wants to terminate the server as soon as possible to simplify coprocessor failure analysis.", - "display_name": "HBase Coprocessor Abort on Error", - "name": "hbase_coprocessor_abort_on_error", - "value": "false" - }, - { - "desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.", - "display_name": "Enable Log Event Capture", - "name": "catch_events", - "value": "true" - }, - { - "desc": "Allow HBase tables to be replicated.", - "display_name": "Enable Replication", - "name": "hbase_enable_replication", - "value": "false" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Path to the keystore file containing the server certificate and private key used for encrypted web UIs.", - "display_name": "SSL Server Keystore File Location", - "name": "ssl_server_keystore_location", - "value": null - }, - { - "desc": "Enable HBase authorization.", - "display_name": "HBase Secure Authorization", - "name": "hbase_security_authorization", - "value": "false" - }, - { - "desc": "Name of the scheduler pool to use for MR jobs created during export/import of remote snapshots in AWS S3.", - "display_name": "Scheduler pool for remote snapshots in AWS S3.", - "name": "hbase_snapshot_s3_scheduler_pool", - "value": null - }, - { - "desc": "Period of time, in milliseconds, to pause between searches for work. Used as a sleep interval by service threads such as a META scanner and log roller.", - "display_name": "HBase Server Thread Wake Frequency", - "name": "hbase_server_thread_wakefrequency", - "value": "10000" - }, - { - "desc": "Number of rows to fetch when calling next on a scanner if it is not served from memory. Higher caching values enable faster scanners but require more memory and some calls of next may take longer when the cache is empty.", - "display_name": "HBase Client Scanner Caching", - "name": "hbase_client_scanner_caching", - "value": "100" - }, - { - "desc": "An alert is published if the HBase region health canary detects at least this percentage of total regions are unhealthy. This threshold is used if the explicit count is not set via the hbase_canary_alert_unhealthy_region_count_threshold config.", - "display_name": "HBase Canary Unhealthy Region Percentage Alert Threshold", - "name": "hbase_canary_alert_unhealthy_region_percent_threshold", - "value": "0.1" - }, - { - "desc": "The health test thresholds of the overall RegionServer health. The check returns \"Concerning\" health if the percentage of \"Healthy\" RegionServers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" RegionServers falls below the critical threshold.", - "display_name": "Healthy RegionServer Monitoring Thresholds", - "name": "hbase_regionservers_healthy_thresholds", - "value": "{\"critical\":\"90.0\",\"warning\":\"95.0\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for navigator.client.properties.", - "display_name": "HBASE Client Advanced Configuration Snippet (Safety Valve) for navigator.client.properties", - "name": "navigator_client_config_safety_valve", - "value": null - }, - { - "desc": "Duration to wait before starting up a 'hedged' read.", - "display_name": "HDFS Hedged Read Delay Threshold", - "name": "hbase_server_dfs_client_hedged_read_threshold_millis", - "value": "500" - }, - { - "desc": "Enables the HBase Hbck Poller so that Hbck reports will be available. Enabling the Hbck poller will increase the amount of memory used by the Service Monitor. Consider increasing the Service Monitor Java heap size by an additional 3KB per region. For example, for a cluster with 10,000 regions, increase the JVM heap size by approximately 30MB.", - "display_name": "HBase Hbck Poller", - "name": "hbase_hbck_poller_enabled", - "value": "false" - }, - { - "desc": "List of users or groups, who are allowed full privileges, regardless of stored ACLs, across the cluster. Only used when HBase security is enabled.", - "display_name": "HBase Superusers", - "name": "hbase_superuser", - "value": "" - }, - { - "desc": "Maximum number of hlog entries to replicate in one go. If this is large, and a consumer takes a while to process the events, the HBase RPC call will time out.", - "display_name": "Replication Batch Size", - "name": "hbase_replication_source_nb_capacity", - "value": "1000" - }, - { - "desc": "An alert is published if the HBase region health canary detects at least this many unhealthy regions. This setting takes precedence over the hbase_canary_alert_unhealthy_region_percent_threshold config.", - "display_name": "HBase Canary Unhealthy Region Count Alert Threshold", - "name": "hbase_canary_alert_unhealthy_region_count_threshold", - "value": null - }, - { - "desc": "ZooKeeper session timeout in milliseconds. HBase passes this to the ZooKeeper quorum as the suggested maximum time for a session. See http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions The client sends a requested timeout, the server responds with the timeout that it can give the client.", - "display_name": "ZooKeeper Session Timeout", - "name": "zookeeper_session_timeout", - "value": "60000" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "Write buffer size in bytes. A larger buffer requires more memory on both the client and the server because the server instantiates the passed write buffer to process it but reduces the number of remote procedure calls (RPC). To estimate the amount of server memory used, multiply the value of 'hbase.client.write.buffer' by the value of 'hbase.regionserver.handler.count'.", - "display_name": "HBase Client Write Buffer", - "name": "hbase_client_write_buffer", - "value": "2097152" - }, - { - "desc": "The user the management services will impersonate when connecting to HBase. Defaults to 'hbase', a superuser.", - "display_name": "HBase User to Impersonate", - "name": "hbase_user_to_impersonate", - "value": "hbase" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hdfs-balancer.json b/sahara/plugins/cdh/v5_3_0/resources/hdfs-balancer.json deleted file mode 100644 index 6f6e669d..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hdfs-balancer.json +++ /dev/null @@ -1,44 +0,0 @@ -[ - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Balancer", - "name": "balancer_java_opts", - "value": "" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "Balancer Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "balancer_config_safety_valve", - "value": null - }, - { - "desc": "The policy that should be used to rebalance HDFS storage. The default DataNode policy balances the storage at the DataNode level. This is similar to the balancing policy from prior releases. The BlockPool policy balances the storage at the block pool level as well as at the Datanode level. The BlockPool policy is relevant only to a Federated HDFS service.", - "display_name": "Rebalancing Policy", - "name": "rebalancing_policy", - "value": "DataNode" - }, - { - "desc": "The percentage deviation from average utilization, after which a node will be rebalanced. (for example, '10.0' for 10%)", - "display_name": "Rebalancing Threshold", - "name": "rebalancer_threshold", - "value": "10.0" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Balancer in Bytes", - "name": "balancer_java_heapsize", - "value": "1073741824" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hdfs-datanode.json b/sahara/plugins/cdh/v5_3_0/resources/hdfs-datanode.json deleted file mode 100644 index 6966ede5..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hdfs-datanode.json +++ /dev/null @@ -1,416 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 5, \"content\":\"Datanode registration failed\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Got a command from standby NN - ignoring command:.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "Specifies the maximum number of threads to use for transferring data in and out of the DataNode.", - "display_name": "Maximum Number of Transfer Threads", - "name": "dfs_datanode_max_xcievers", - "value": "4096" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Comma-separated list of DataNode plug-ins to be activated. If one plug-in cannot be loaded, all the plug-ins are ignored.", - "display_name": "DateNode Plugins", - "name": "dfs_datanode_plugins_list", - "value": "" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "In some workloads, the data read from HDFS is known to be significantly large enough that it is unlikely to be useful to cache it in the operating system buffer cache. In this case, the DataNode may be configured to automatically purge all data from the buffer cache after it is delivered to the client. This may improve performance for some workloads by freeing buffer cache spare usage for more cacheable data. This behavior will always be disabled for workloads that read only short sections of a block (e.g HBase random-IO workloads). This property is supported in CDH3u3 or later deployments.", - "display_name": "Enable purging cache after reads", - "name": "dfs_datanode_drop_cache_behind_reads", - "value": "false" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Comma-delimited list of directories on the local file system where the DataNode stores HDFS block data. Typical values are /data/N/dfs/dn for N = 1, 2, 3... These directories should be mounted using the noatime option and the disks should be configured using JBOD. RAID is not recommended.", - "display_name": "DataNode Data Directory", - "name": "dfs_data_dir_list", - "value": null - }, - { - "desc": "The health test thresholds for the weighted average extra time the pause monitor spent paused. Specified as a percentage of elapsed wall clock time.", - "display_name": "Pause Duration Thresholds", - "name": "datanode_pause_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "The number of volumes that are allowed to fail before a DataNode stops offering service. By default, any volume failure will cause a DataNode to shutdown.", - "display_name": "DataNode Failed Volumes Tolerated", - "name": "dfs_datanode_failed_volumes_tolerated", - "value": "0" - }, - { - "desc": "In some workloads, the data written to HDFS is known to be significantly large enough that it is unlikely to be useful to cache it in the operating system buffer cache. In this case, the DataNode may be configured to automatically purge all data from the buffer cache after it is written to disk. This may improve performance for some workloads by freeing buffer cache spare usage for more cacheable data. This property is supported in CDH3u3 or later deployments.", - "display_name": "Enable purging cache after writes", - "name": "dfs_datanode_drop_cache_behind_writes", - "value": "false" - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "If enabled, the DataNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind DataNode to Wildcard Address", - "name": "dfs_datanode_bind_wildcard", - "value": "false" - }, - { - "desc": "The number of server threads for the DataNode.", - "display_name": "Handler Count", - "name": "dfs_datanode_handler_count", - "value": "3" - }, - { - "desc": "When computing the overall DataNode health, consider the host's health.", - "display_name": "DataNode Host Health Test", - "name": "datanode_host_health_enabled", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "The maximum number of rolled log files to keep for DataNode logs. Typically used by log4j.", - "display_name": "DataNode Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Timeout in seconds for the Hue Thrift server running on each DataNode", - "display_name": "Hue Thrift Server Timeout", - "name": "dfs_thrift_timeout", - "value": "60" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Whether DataNodes should use DataNode hostnames when connecting to DataNodes for data transfer. This property is supported in CDH3u4 or later deployments.", - "display_name": "Use DataNode Hostname", - "name": "dfs_datanode_use_datanode_hostname", - "value": "false" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for DataNode", - "name": "datanode_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "datanode_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "datanode_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Enables the health test that the DataNode's process state is consistent with the role configuration", - "display_name": "DataNode Process Health Test", - "name": "datanode_scm_health_enabled", - "value": "true" - }, - { - "desc": "Port for DataNode's XCeiver Protocol. Combined with the DataNode's hostname to build its address.", - "display_name": "DataNode Transceiver Port", - "name": "dfs_datanode_port", - "value": "50010" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "DataNode Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "DataNode Policy for picking which volume should get a new block. The Available Space policy is only available starting with CDH 4.3.", - "display_name": "DataNode Volume Choosing Policy", - "name": "dfs_datanode_volume_choosing_policy", - "value": "org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Port for the DataNode HTTP web UI. Combined with the DataNode's hostname to build its HTTP address.", - "display_name": "DataNode HTTP Web UI Port", - "name": "dfs_datanode_http_port", - "value": "50075" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "Directory where DataNode will place its log files.", - "display_name": "DataNode Log Directory", - "name": "datanode_log_dir", - "value": "/var/log/hadoop-hdfs" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "DataNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "datanode_config_safety_valve", - "value": null - }, - { - "desc": "While reading block files, the DataNode can use the posix_fadvise system call to explicitly page data into the operating system buffer cache ahead of the current reader's position. This can improve performance especially when disks are highly contended. This configuration specifies the number of bytes ahead of the current read position which the DataNode will attempt to read ahead. A value of 0 disables this feature. This property is supported in CDH3u3 or later deployments.", - "display_name": "Number of read ahead bytes", - "name": "dfs_datanode_readahead_bytes", - "value": "4194304" - }, - { - "desc": "The maximum size, in megabytes, per log file for DataNode logs. Typically used by log4j.", - "display_name": "DataNode Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The maximum amount of memory a DataNode may use to cache data blocks in memory. Setting it to zero will disable caching.", - "display_name": "Maximum Memory Used for Caching", - "name": "dfs_datanode_max_locked_memory", - "value": "4294967296" - }, - { - "desc": "Only used when the DataNode Volume Choosing Policy is set to Available Space. Controls how much DataNode volumes are allowed to differ in terms of bytes of free disk space before they are considered imbalanced. If the free space of all the volumes are within this range of each other, the volumes will be considered balanced and block assignments will be done on a pure round robin basis.", - "display_name": "Available Space Policy Balanced Threshold", - "name": "dfs_datanode_available_space_balanced_threshold", - "value": "10737418240" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds of free space in a DataNode. Specified as a percentage of the capacity on the DataNode.", - "display_name": "DataNode Free Space Monitoring Thresholds", - "name": "datanode_free_space_thresholds", - "value": "{\"critical\":\"10.0\",\"warning\":\"20.0\"}" - }, - { - "desc": "Port for the various DataNode Protocols. Combined with the DataNode's hostname to build its IPC port address.", - "display_name": "DataNode Protocol Port", - "name": "dfs_datanode_ipc_port", - "value": "50020" - }, - { - "desc": "The period to review when computing the moving average of extra time the pause monitor spent paused.", - "display_name": "Pause Duration Monitoring Period", - "name": "datanode_pause_duration_window", - "value": "5" - }, - { - "desc": "Minimum number of running threads for the Hue Thrift server running on each DataNode", - "display_name": "Hue Thrift Server Min Threadcount", - "name": "dfs_thrift_threads_min", - "value": "10" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "If this configuration is enabled, the DataNode will instruct the operating system to enqueue all written data to the disk immediately after it is written. This differs from the usual OS policy which may wait for up to 30 seconds before triggering writeback. This may improve performance for some workloads by smoothing the IO profile for data written to disk. This property is supported in CDH3u3 or later deployments.", - "display_name": "Enable immediate enqueuing of data to disk after writes", - "name": "dfs_datanode_sync_behind_writes", - "value": "false" - }, - { - "desc": "The health test thresholds of the number of blocks on a DataNode", - "display_name": "DataNode Block Count Thresholds", - "name": "datanode_block_count_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"500000.0\"}" - }, - { - "desc": "Permissions for the directories on the local file system where the DataNode stores its blocks. The permissions must be octal. 755 and 700 are typical values.", - "display_name": "DataNode Data Directory Permissions", - "name": "dfs_datanode_data_dir_perm", - "value": "700" - }, - { - "desc": "Reserved space in bytes per volume for non Distributed File System (DFS) use.", - "display_name": "Reserved Space for Non DFS Use", - "name": "dfs_datanode_du_reserved", - "value": "10737418240" - }, - { - "desc": "The amount of time to wait for the DataNode to fully start up and connect to the NameNode before enforcing the connectivity check.", - "display_name": "DataNode Connectivity Tolerance at Startup", - "name": "datanode_connectivity_tolerance", - "value": "180" - }, - { - "desc": "The base port where the secure DataNode web UI listens. Combined with the DataNode's hostname to build its secure web UI address.", - "display_name": "Secure DataNode Web UI Port (SSL)", - "name": "dfs_datanode_https_port", - "value": "50475" - }, - { - "desc": "Maximum amount of bandwidth that each DataNode can use for balancing. Specified in bytes per second.", - "display_name": "DataNode Balancing Bandwidth", - "name": "dfs_balance_bandwidthPerSec", - "value": "10485760" - }, - { - "desc": "Maximum number of running threads for the Hue Thrift server running on each DataNode", - "display_name": "Hue Thrift Server Max Threadcount", - "name": "dfs_thrift_threads_max", - "value": "20" - }, - { - "desc": "The health test thresholds of transceivers usage in a DataNode. Specified as a percentage of the total configured number of transceivers.", - "display_name": "DataNode Transceivers Usage Thresholds", - "name": "datanode_transceivers_usage_thresholds", - "value": "{\"critical\":\"95.0\",\"warning\":\"75.0\"}" - }, - { - "desc": "Only used when the DataNode Volume Choosing Policy is set to Available Space. Controls what percentage of new block allocations will be sent to volumes with more available disk space than others. This setting should be in the range 0.0 - 1.0, though in practice 0.5 - 1.0, since there should be no reason to prefer that volumes with less available disk space receive more block allocations.", - "display_name": "Available Space Policy Balanced Preference", - "name": "dfs_datanode_available_space_balanced_preference", - "value": "0.75" - }, - { - "desc": "The health test thresholds of failed volumes in a DataNode.", - "display_name": "DataNode Volume Failures Thresholds", - "name": "datanode_volume_failures_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "The minimum log level for DataNode logs", - "display_name": "DataNode Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "datanode_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of DataNode in Bytes", - "name": "datanode_java_heapsize", - "value": "1073741824" - }, - { - "desc": "Enables the health test that verifies the DataNode is connected to the NameNode", - "display_name": "DataNode Connectivity Health Test", - "name": "datanode_connectivity_health_enabled", - "value": "true" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hdfs-failovercontroller.json b/sahara/plugins/cdh/v5_3_0/resources/hdfs-failovercontroller.json deleted file mode 100644 index fdf41dde..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hdfs-failovercontroller.json +++ /dev/null @@ -1,200 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Directory where Failover Controller will place its log files.", - "display_name": "Failover Controller Log Directory", - "name": "failover_controller_log_dir", - "value": "/var/log/hadoop-hdfs" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "Failover Controller Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "fc_config_safety_valve", - "value": null - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for Failover Controller logs. Typically used by log4j.", - "display_name": "Failover Controller Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Failover Controller in Bytes", - "name": "failover_controller_java_heapsize", - "value": "268435456" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Failover Controller Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When computing the overall Failover Controller health, consider the host's health.", - "display_name": "Failover Controller Host Health Test", - "name": "failovercontroller_host_health_enabled", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Failover Controller logs. Typically used by log4j.", - "display_name": "Failover Controller Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "failovercontroller_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Enables the health test that the Failover Controller's process state is consistent with the role configuration", - "display_name": "Failover Controller Process Health Test", - "name": "failovercontroller_scm_health_enabled", - "value": "true" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Failover Controller", - "name": "failover_controller_java_opts", - "value": "" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The minimum log level for Failover Controller logs", - "display_name": "Failover Controller Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hdfs-gateway.json b/sahara/plugins/cdh/v5_3_0/resources/hdfs-gateway.json deleted file mode 100644 index 945e04fc..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hdfs-gateway.json +++ /dev/null @@ -1,56 +0,0 @@ -[ - { - "desc": "For advanced use only, a string to be inserted into the client configuration for hdfs-site.xml.", - "display_name": "HDFS Client Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "hdfs_client_config_safety_valve", - "value": null - }, - { - "desc": "The directory where the client configs will be deployed", - "display_name": "Deploy Directory", - "name": "client_config_root_dir", - "value": "/etc/hadoop" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the client configuration for hadoop-env.sh", - "display_name": "HDFS Client Environment Advanced Configuration Snippet for hadoop-env.sh (Safety Valve)", - "name": "hdfs_client_env_safety_valve", - "value": null - }, - { - "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", - "display_name": "Alternatives Priority", - "name": "client_config_priority", - "value": "90" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Maximum size in bytes for the Java process heap memory. Passed to Java -Xmx.", - "display_name": "Client Java Heap Size in Bytes", - "name": "hdfs_client_java_heapsize", - "value": "268435456" - }, - { - "desc": "Enable HDFS short circuit read. This allows a client co-located with the DataNode to read HDFS file blocks directly. This gives a performance boost to distributed clients that are aware of locality.", - "display_name": "Enable HDFS Short Circuit Read", - "name": "dfs_client_read_shortcircuit", - "value": "false" - }, - { - "desc": "These are Java command line arguments. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Client Java Configuration Options", - "name": "hbase_client_java_opts", - "value": "-Djava.net.preferIPv4Stack=true" - }, - { - "desc": "Move deleted files to the trash so that they can be recovered if necessary. This client side configuration takes effect only if the HDFS service-wide trash is disabled (NameNode Filesystem Trash Interval set to 0) and is ignored otherwise. The trash is not automatically emptied when enabled with this configuration.", - "display_name": "Use Trash", - "name": "dfs_client_use_trash", - "value": "false" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hdfs-httpfs.json b/sahara/plugins/cdh/v5_3_0/resources/hdfs-httpfs.json deleted file mode 100644 index 34343818..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hdfs-httpfs.json +++ /dev/null @@ -1,242 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "httpfs_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "The user that the HttpFS server process should run as.", - "display_name": "System User", - "name": "httpfs_process_username", - "value": "httpfs" - }, - { - "desc": "The group that the HttpFS server process should run as.", - "display_name": "System Group", - "name": "httpfs_process_groupname", - "value": "httpfs" - }, - { - "desc": "When computing the overall HttpFS health, consider the host's health.", - "display_name": "HttpFS Host Health Test", - "name": "httpfs_host_health_enabled", - "value": "true" - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for HttpFS logs. Typically used by log4j.", - "display_name": "HttpFS Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Enables the health test that the HttpFS's process state is consistent with the role configuration", - "display_name": "HttpFS Process Health Test", - "name": "httpfs_scm_health_enabled", - "value": "true" - }, - { - "desc": "Password of the keystore used by HttpFS role for SSL.", - "display_name": "HttpFS Keystore Password", - "name": "httpfs_https_keystore_password", - "value": null - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "Directory where HttpFS will place its log files.", - "display_name": "HttpFS Log Directory", - "name": "httpfs_log_dir", - "value": "/var/log/hadoop-httpfs" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "HttpFS Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "For advanced use only, a string to be inserted into httpfs-site.xml for this role only.", - "display_name": "HttpFS Advanced Configuration Snippet (Safety Valve) for httpfs-site.xml", - "name": "httpfs_config_safety_valve", - "value": null - }, - { - "desc": "The maximum size, in megabytes, per log file for HttpFS logs. Typically used by log4j.", - "display_name": "HttpFS Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for HttpFS", - "name": "httpfs_java_opts", - "value": "" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The secret to use for signing client authentication tokens.", - "display_name": "Signature Secret", - "name": "hdfs_httpfs_signature_secret", - "value": "hadoop httpfs secret" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Location of the keystore file used by HttpFS role for SSL.", - "display_name": "HttpFS Keystore File", - "name": "httpfs_https_keystore_file", - "value": "/var/run/hadoop-httpfs/.keystore" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of HttpFS in Bytes", - "name": "httpfs_java_heapsize", - "value": "268435456" - }, - { - "desc": "The port for the administration interface.", - "display_name": "Administration Port", - "name": "hdfs_httpfs_admin_port", - "value": "14001" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "The HTTP port where the REST interface to HDFS is available.", - "display_name": "HTTP Port", - "name": "hdfs_httpfs_http_port", - "value": "14000" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The minimum log level for HttpFS logs", - "display_name": "HttpFS Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Use SSL for HttpFS.", - "display_name": "Use SSL", - "name": "httpfs_use_ssl", - "value": "false" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hdfs-journalnode.json b/sahara/plugins/cdh/v5_3_0/resources/hdfs-journalnode.json deleted file mode 100644 index 85740273..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hdfs-journalnode.json +++ /dev/null @@ -1,284 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "journalnode_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of JournalNode in Bytes", - "name": "journalNode_java_heapsize", - "value": "268435456" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "journalnode_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "Enables the health test that the JournalNode's process state is consistent with the role configuration", - "display_name": "JournalNode Process Health Test", - "name": "journalnode_scm_health_enabled", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "The maximum number of rolled log files to keep for JournalNode logs. Typically used by log4j.", - "display_name": "JournalNode Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The health check thresholds for monitoring of free space on the filesystem that contains the JournalNode's edits directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Edits Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Edits Directory Free Space Monitoring Percentage Thresholds", - "name": "journalnode_edits_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Directory on the local file system where NameNode edits are written.", - "display_name": "JournalNode Edits Directory", - "name": "dfs_journalnode_edits_dir", - "value": null - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "journalnode_gc_duration_window", - "value": "5" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "JournalNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "jn_config_safety_valve", - "value": null - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "When computing the overall JournalNode health, consider the host's health.", - "display_name": "JournalNode Host Health Test", - "name": "journalnode_host_health_enabled", - "value": "true" - }, - { - "desc": "The amount of time at JournalNode startup allowed for the active NameNode to get in sync with the JournalNode.", - "display_name": "Active NameNode Sync Status Startup Tolerance", - "name": "journalnode_sync_status_startup_tolerance", - "value": "180" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The base port where the secure JournalNode web UI listens. Combined with the JournalNode's hostname to build its secure web UI address.", - "display_name": "Secure JournalNode Web UI Port (SSL)", - "name": "dfs_journalnode_https_port", - "value": "8481" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "journalnode_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "JournalNode Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The health test thresholds for JournalNode fsync latency", - "display_name": "JournalNode Fsync Latency Thresholds", - "name": "journalnode_fsync_latency_thresholds", - "value": "{\"critical\":\"3000.0\",\"warning\":\"1000.0\"}" - }, - { - "desc": "Enables the health check that verifies the active NameNode's sync status to the JournalNode", - "display_name": "Active NameNode Sync Status Health Check", - "name": "journalnode_sync_status_enabled", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "journalnode_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "Port for the JournalNode's RPC. Combined with the JournalNode's hostname to build its RPC address.", - "display_name": "JournalNode RPC Port", - "name": "dfs_journalnode_rpc_port", - "value": "8485" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for JournalNode logs. Typically used by log4j.", - "display_name": "JournalNode Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "If enabled, the JournalNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind JournalNode to Wildcard Address", - "name": "journalnode_bind_wildcard", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The health check thresholds for monitoring of free space on the filesystem that contains the JournalNode's edits directory.", - "display_name": "Edits Directory Free Space Monitoring Absolute Thresholds", - "name": "journalnode_edits_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Directory where JournalNode will place its log files.", - "display_name": "JournalNode Log Directory", - "name": "journalnode_log_dir", - "value": "/var/log/hadoop-hdfs" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for JournalNode", - "name": "journalNode_java_opts", - "value": "" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Port for the JournalNode HTTP web UI. Combined with the JournalNode hostname to build its HTTP address.", - "display_name": "JournalNode HTTP Port", - "name": "dfs_journalnode_http_port", - "value": "8480" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "The minimum log level for JournalNode logs", - "display_name": "JournalNode Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hdfs-namenode.json b/sahara/plugins/cdh/v5_3_0/resources/hdfs-namenode.json deleted file mode 100644 index 9ba2a155..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hdfs-namenode.json +++ /dev/null @@ -1,488 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", - "display_name": "Health Check Startup Tolerance", - "name": "namenode_startup_tolerance", - "value": "5" - }, - { - "desc": "The health test thresholds of failed status directories in a NameNode.", - "display_name": "NameNode Directory Failures Thresholds", - "name": "namenode_directory_failures_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Name of the journal located on each JournalNode filesystem.", - "display_name": "Quorum-based Storage Journal name", - "name": "dfs_namenode_quorum_journal_name", - "value": null - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for NameNode", - "name": "namenode_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "The number of server threads for the NameNode.", - "display_name": "NameNode Handler Count", - "name": "dfs_namenode_handler_count", - "value": "30" - }, - { - "desc": "Enables the health test that the NameNode is not in safemode", - "display_name": "NameNode Safemode Health Test", - "name": "namenode_safe_mode_enabled", - "value": "true" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"},\n {\"alert\": false, \"rate\": 1, \"threshold\":\"INFO\", \"content\":\"Triggering checkpoint.*\"}\n ]\n}\n" - }, - { - "desc": "The base port where the DFS NameNode web UI listens. If the port number is 0, then the server starts on a free port. Combined with the NameNode's hostname to build its HTTP address.", - "display_name": "NameNode Web UI Port", - "name": "dfs_http_port", - "value": "50070" - }, - { - "desc": "For advanced use only, a string to be inserted into dfs_hosts_exclude.txt for this role only.", - "display_name": "NameNode Advanced Configuration Snippet (Safety Valve) for dfs_hosts_exclude.txt", - "name": "namenode_hosts_exclude_safety_valve", - "value": null - }, - { - "desc": "Mountpoints that are mapped to this NameNode's Nameservice.", - "display_name": "Mountpoints", - "name": "nameservice_mountpoints", - "value": "/" - }, - { - "desc": "The health test thresholds of the age of the HDFS namespace checkpoint. Specified as a percentage of the configured checkpoint interval.", - "display_name": "Filesystem Checkpoint Age Monitoring Thresholds", - "name": "namenode_checkpoint_age_thresholds", - "value": "{\"critical\":\"400.0\",\"warning\":\"200.0\"}" - }, - { - "desc": "The base port where the secure NameNode web UI listens.", - "display_name": "Secure NameNode Web UI Port (SSL)", - "name": "dfs_https_port", - "value": "50470" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Namenode in Bytes", - "name": "namenode_java_heapsize", - "value": "1073741824" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "NameNode Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "This determines the total amount of block transfers to begin in parallel at a DataNode for replication, when such a command list is being sent over a DataNode heartbeat by the NameNode. The actual number is obtained by multiplying this value by the total number of live nodes in the cluster. The result number is the number of blocks to transfer immediately, per DataNode heartbeat.", - "display_name": "Replication Work Multiplier Per Iteration", - "name": "dfs_namenode_replication_work_multiplier_per_iteration", - "value": "2" - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "The health test thresholds of the number of transactions since the last HDFS namespace checkpoint. Specified as a percentage of the configured checkpointing transaction limit.", - "display_name": "Filesystem Checkpoint Transactions Monitoring Thresholds", - "name": "namenode_checkpoint_transactions_thresholds", - "value": "{\"critical\":\"400.0\",\"warning\":\"200.0\"}" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "Determines extension of safemode in milliseconds after the threshold level is reached.", - "display_name": "Safemode Extension", - "name": "dfs_safemode_extension", - "value": "30000" - }, - { - "desc": "Determines where on the local file system the NameNode should store the name table (fsimage). For redundancy, enter a comma-delimited list of directories to replicate the name table in all of the directories. Typical values are /data/N/dfs/nn where N=1..3.", - "display_name": "NameNode Data Directories", - "name": "dfs_name_dir_list", - "value": null - }, - { - "desc": "Timeout in seconds for the Hue Thrift server running on the NameNode", - "display_name": "Hue Thrift Server Timeout", - "name": "dfs_thrift_timeout", - "value": "60" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The period to review when computing the moving average of extra time the pause monitor spent paused.", - "display_name": "Pause Duration Monitoring Period", - "name": "namenode_pause_duration_window", - "value": "5" - }, - { - "desc": "The health check thresholds of the NameNode's RPC latency.", - "display_name": "NameNode RPC Latency Thresholds", - "name": "namenode_rpc_latency_thresholds", - "value": "{\"critical\":\"5000.0\",\"warning\":\"1000.0\"}" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "namenode_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "The number of server threads for the NameNode used for service calls. Only used when NameNode Service RPC Port is configured.", - "display_name": "NameNode Service Handler Count", - "name": "dfs_namenode_service_handler_count", - "value": "30" - }, - { - "desc": "Full path to a custom topology script on the host file system. The topology script is used to determine the rack location of nodes. If left blank, a topology script will be provided that uses your hosts' rack information, visible in the \"Hosts\" page.", - "display_name": "Topology Script File Name", - "name": "topology_script_file_name", - "value": null - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "namenode_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The time between two periodic file system checkpoints.", - "display_name": "Filesystem Checkpoint Period", - "name": "fs_checkpoint_period", - "value": "3600" - }, - { - "desc": "Specifies the number of DataNodes that must be live before the name node exits safemode. Enter a value less than or equal to 0 to take the number of live DataNodes into account when deciding whether to remain in safemode during startup. Values greater than the number of DataNodes in the cluster will make safemode permanent.", - "display_name": "Safemode Minimum DataNodes", - "name": "dfs_safemode_min_datanodes", - "value": "0" - }, - { - "desc": "The port where the NameNode runs the HDFS protocol. Combined with the NameNode's hostname to build its address.", - "display_name": "NameNode Port", - "name": "namenode_port", - "value": "8020" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "NameNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "namenode_config_safety_valve", - "value": null - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The period to review when computing the moving average of the NameNode's RPC latency.", - "display_name": "NameNode RPC Latency Monitoring Window", - "name": "namenode_rpc_latency_window", - "value": "5" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Optional port for the service-rpc address which can be used by HDFS daemons instead of sharing the RPC address used by the clients.", - "display_name": "NameNode Service RPC Port", - "name": "dfs_namenode_servicerpc_address", - "value": null - }, - { - "desc": "The health test thresholds for the weighted average extra time the pause monitor spent paused. Specified as a percentage of elapsed wall clock time.", - "display_name": "Pause Duration Thresholds", - "name": "namenode_pause_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "When computing the overall NameNode health, consider the host's health.", - "display_name": "NameNode Host Health Test", - "name": "namenode_host_health_enabled", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for NameNode logs. Typically used by log4j.", - "display_name": "NameNode Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "Enables the health test that the NameNode's process state is consistent with the role configuration", - "display_name": "NameNode Process Health Test", - "name": "namenode_scm_health_enabled", - "value": "true" - }, - { - "desc": "The health check thresholds for the number of out-of-sync JournalNodes for this NameNode.", - "display_name": "NameNode Out-Of-Sync JournalNodes Thresholds", - "name": "namenode_out_of_sync_journal_nodes_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into dfs_hosts_allow.txt for this role only.", - "display_name": "NameNode Advanced Configuration Snippet (Safety Valve) for dfs_hosts_allow.txt", - "name": "namenode_hosts_allow_safety_valve", - "value": null - }, - { - "desc": "Number of minutes between trash checkpoints. Also controls the number of minutes after which a trash checkpoint directory is deleted. To disable the trash feature, enter 0.", - "display_name": "Filesystem Trash Interval", - "name": "fs_trash_interval", - "value": "1440" - }, - { - "desc": "The access time for HDFS file is precise upto this value. Setting the value of 0 disables access times for HDFS. When using the NFS Gateway role, make sure this property is enabled.", - "display_name": "Access Time Precision", - "name": "dfs_access_time_precision", - "value": "3600000" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Minimum number of running threads for the Hue Thrift server running on the NameNode", - "display_name": "Hue Thrift Server Min Threadcount", - "name": "dfs_thrift_threads_min", - "value": "10" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "If enabled, the NameNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind NameNode to Wildcard Address", - "name": "namenode_bind_wildcard", - "value": "false" - }, - { - "desc": "Comma-separated list of NameNode plug-ins to be activated. If one plug-in cannot be loaded, all the plug-ins are ignored.", - "display_name": "NameNode Plugins", - "name": "dfs_namenode_plugins_list", - "value": "" - }, - { - "desc": "Directory on a shared storage device, such as a Quorum-based Storage URI or a local directory that is an NFS mount from a NAS, to store the NameNode edits. The value of this configuration is automatically generated to be the Quourm Journal URI if there are JournalNodes and this NameNode is Highly Available.", - "display_name": "Shared Edits Directory", - "name": "dfs_namenode_shared_edits_dir", - "value": null - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Enables the health test of the rolling metadata upgrade status of the NameNode. This covers rolling metadata upgrades. Non-rolling metadata upgrades are covered in a separate health test.", - "display_name": "HDFS Rolling Metadata Upgrade Status Health Test", - "name": "namenode_rolling_upgrade_status_enabled", - "value": "true" - }, - { - "desc": "This determines the percentage amount of block invalidations (deletes) to do over a single DataNode heartbeat deletion command. The final deletion count is determined by applying this percentage to the number of live nodes in the system. The resultant number is the number of blocks from the deletion list chosen for proper invalidation over a single heartbeat of a single DataNode.", - "display_name": "Invalidate Work Percentage Per Iteration", - "name": "dfs_namenode_invalidate_work_pct_per_iteration", - "value": "0.32" - }, - { - "desc": "Enable Automatic Failover to maintain High Availability. Requires a ZooKeeper service and a High Availability NameNode partner.", - "display_name": "Enable Automatic Failover", - "name": "autofailover_enabled", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for NameNode logs. Typically used by log4j.", - "display_name": "NameNode Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "Maximum number of running threads for the Hue Thrift server running on the NameNode", - "display_name": "Hue Thrift Server Max Threadcount", - "name": "dfs_thrift_threads_max", - "value": "20" - }, - { - "desc": "Directory where NameNode will place its log files.", - "display_name": "NameNode Log Directory", - "name": "namenode_log_dir", - "value": "/var/log/hadoop-hdfs" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "namenode_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Enables the health test of the metadata upgrade status of the NameNode. This covers non-rolling metadata upgrades. Rolling metadata upgrades are covered in a separate health test.", - "display_name": "HDFS Metadata Upgrade Status Health Test", - "name": "namenode_upgrade_status_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystems that contain this role's data directories. Specified as a percentage of the capacity on the filesystem. This setting is not used if a Data Directories Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Data Directories Free Space Monitoring Percentage Thresholds", - "name": "namenode_data_directories_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Nameservice of this NameNode. The Nameservice represents the interface to this NameNode and its High Availability partner. The Nameservice also represents the namespace associated with a federated NameNode.", - "display_name": "NameNode Nameservice", - "name": "dfs_federation_namenode_nameservice", - "value": null - }, - { - "desc": "The number of transactions after which the NameNode or SecondaryNameNode will create a checkpoint of the namespace, regardless of whether the checkpoint period has expired.", - "display_name": "Filesystem Checkpoint Transaction Threshold", - "name": "fs_checkpoint_txns", - "value": "1000000" - }, - { - "desc": "The minimum log level for NameNode logs", - "display_name": "NameNode Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Specifies the percentage of blocks that should satisfy the minimal replication requirement defined by dfs.replication.min. Enter a value less than or equal to 0 to wait for any particular percentage of blocks before exiting safemode. Values greater than 1 will make safemode permanent.", - "display_name": "Safemode Threshold Percentage", - "name": "dfs_safemode_threshold_pct", - "value": "0.999" - }, - { - "desc": "Directories on the local file system to store the NameNode edits. If not set, the edits are stored in the NameNode's Data Directories. The value of this configuration is automatically generated to be the Quorum-based Storage URI if there are JournalNodes and this NameNode is not Highly Available.", - "display_name": "NameNode Edits Directories", - "name": "dfs_namenode_edits_dir", - "value": null - }, - { - "desc": "If set to false and if one of the replicas of the NameNode storage fails, such as temporarily failure of NFS, this directory is not used until the NameNode restarts. If enabled, failed storage is re-checked on every checkpoint and, if it becomes valid, the NameNode will try to restore the edits and fsimage.", - "display_name": "Restore NameNode Directories at Checkpoint Time", - "name": "dfs_name_dir_restore", - "value": "false" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystems that contain this role's data directories.", - "display_name": "Data Directories Free Space Monitoring Absolute Thresholds", - "name": "namenode_data_directories_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hdfs-nfsgateway.json b/sahara/plugins/cdh/v5_3_0/resources/hdfs-nfsgateway.json deleted file mode 100644 index b3a85b1d..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hdfs-nfsgateway.json +++ /dev/null @@ -1,230 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "nfsgateway_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "NFS clients often reorder writes. As a result, sequential writes can arrive at the NFS Gateway in random order. This directory is used to temporarily save out-of-order writes before writing to HDFS. For each file, the out-of-order writes are dumped after they are accumulated to exceed certain threshold (e.g., 1MB) in memory. Please make sure this directory has enough space. For example, if the application uploads 10 files with each having 100MB, it is recommended that this directory have roughly 1GB of space in case write reorder happens (in the worst case) to every file.", - "display_name": "Temporary Dump Directory", - "name": "dfs_nfs3_dump_dir", - "value": "/tmp/.hdfs-nfs" - }, - { - "desc": "Enables the health test that the NFS Gateway's process state is consistent with the role configuration", - "display_name": "NFS Gateway Process Health Test", - "name": "nfsgateway_scm_health_enabled", - "value": "true" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of NFS Gateway in Bytes", - "name": "nfsgateway_java_heapsize", - "value": "268435456" - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "NFS Gateway Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "nfsgateway_config_safety_valve", - "value": null - }, - { - "desc": "The port number of the system portmap or rpcbind service. This configuration is used by Cloudera Manager to verify if the system portmap or rpcbind service is running before starting NFS Gateway role. Cloudera Manager does not manage the system portmap or rpcbind service.", - "display_name": "Portmap (or Rpcbind) Port", - "name": "nfs3_portmap_port", - "value": "111" - }, - { - "desc": "The NFS Gateway server port.", - "display_name": "NFS Gateway Server Port", - "name": "nfs3_server_port", - "value": "2049" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for NFS Gateway", - "name": "nfsgateway_java_opts", - "value": "" - }, - { - "desc": "The maximum number of rolled log files to keep for NFS Gateway logs. Typically used by log4j.", - "display_name": "NFS Gateway Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "When computing the overall NFS Gateway health, consider the host's health.", - "display_name": "NFS Gateway Host Health Test", - "name": "nfsgateway_host_health_enabled", - "value": "true" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "By default, NFS Gateway exported directories can be mounted by any client. For better access control, update this property with a list of host names and access privileges separated by whitespace characters. Host name format can be a single host, a Java regular expression, or an IPv4 address. The access privilege uses rw to specify readwrite and ro to specify readonly access. If the access privilege is not provided, the default is read-only. Examples of host name format and access privilege: \"192.168.0.0/22 rw\", \"host.*.example.com\", \"host1.test.org ro\".", - "display_name": "Allowed Hosts and Privileges", - "name": "dfs_nfs_exports_allowed_hosts", - "value": "* rw" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The port number of the mount daemon implemented inside the NFS Gateway server role.", - "display_name": "NFS Gateway MountD Port", - "name": "nfs3_mountd_port", - "value": "4242" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "NFS Gateway Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The maximum size, in megabytes, per log file for NFS Gateway logs. Typically used by log4j.", - "display_name": "NFS Gateway Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The minimum log level for NFS Gateway logs", - "display_name": "NFS Gateway Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Directory where NFS Gateway will place its log files.", - "display_name": "NFS Gateway Log Directory", - "name": "nfsgateway_log_dir", - "value": "/var/log/hadoop-hdfs" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hdfs-secondarynamenode.json b/sahara/plugins/cdh/v5_3_0/resources/hdfs-secondarynamenode.json deleted file mode 100644 index fc96e8a9..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hdfs-secondarynamenode.json +++ /dev/null @@ -1,284 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"},\n {\"alert\": false, \"rate\": 1, \"threshold\":\"INFO\", \"content\":\"Triggering checkpoint.*\"}\n ]\n}\n" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Secondary NameNode", - "name": "secondarynamenode_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystems that contain this role's checkpoint directories.", - "display_name": "Checkpoint Directories Free Space Monitoring Absolute Thresholds", - "name": "secondarynamenode_checkpoint_directories_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Secondary namenode in Bytes", - "name": "secondary_namenode_java_heapsize", - "value": "1073741824" - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for SecondaryNameNode logs. Typically used by log4j.", - "display_name": "SecondaryNameNode Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "SecondaryNameNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "secondarynamenode_config_safety_valve", - "value": null - }, - { - "desc": "Enables the health test that the SecondaryNameNode's process state is consistent with the role configuration", - "display_name": "SecondaryNameNode Process Health Test", - "name": "secondarynamenode_scm_health_enabled", - "value": "true" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The time between two periodic file system checkpoints.", - "display_name": "Filesystem Checkpoint Period", - "name": "fs_checkpoint_period", - "value": "3600" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "If enabled, the SecondaryNameNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind SecondaryNameNode to Wildcard Address", - "name": "secondary_namenode_bind_wildcard", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "SecondaryNameNode Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The number of transactions after which the NameNode or SecondaryNameNode will create a checkpoint of the namespace, regardless of whether the checkpoint period has expired.", - "display_name": "Filesystem Checkpoint Transaction Threshold", - "name": "fs_checkpoint_txns", - "value": "1000000" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "secondarynamenode_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "secondarynamenode_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystems that contain this role's checkpoint directories. Specified as a percentage of the capacity on the filesystem. This setting is not used if a Checkpoint Directories Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Checkpoint Directories Free Space Monitoring Percentage Thresholds", - "name": "secondarynamenode_checkpoint_directories_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "When computing the overall SecondaryNameNode health, consider the host's health.", - "display_name": "SecondaryNameNode Host Health Test", - "name": "secondarynamenode_host_health_enabled", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for SecondaryNameNode logs. Typically used by log4j.", - "display_name": "SecondaryNameNode Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "secondarynamenode_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "secondarynamenode_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "secondarynamenode_gc_duration_window", - "value": "5" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Determines where on the local file system the DFS SecondaryNameNode should store the temporary images to merge. For redundancy, enter a comma-delimited list of directories to replicate the image in all of the directories. Typical values are /data/N/dfs/snn for N = 1, 2, 3...", - "display_name": "HDFS Checkpoint Directory", - "name": "fs_checkpoint_dir_list", - "value": null - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "Nameservice of this SecondaryNameNode", - "display_name": "SecondaryNameNode Nameservice", - "name": "dfs_secondarynamenode_nameservice", - "value": null - }, - { - "desc": "The minimum log level for SecondaryNameNode logs", - "display_name": "SecondaryNameNode Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "The base port where the secure SecondaryNameNode web UI listens.", - "display_name": "Secure SecondaryNameNode Web UI Port (SSL)", - "name": "dfs_secondary_https_port", - "value": "50495" - }, - { - "desc": "Directory where SecondaryNameNode will place its log files.", - "display_name": "SecondaryNameNode Log Directory", - "name": "secondarynamenode_log_dir", - "value": "/var/log/hadoop-hdfs" - }, - { - "desc": "The SecondaryNameNode HTTP port. If the port is 0, then the server starts on a free port. Combined with the SecondaryNameNode's hostname to build its HTTP address.", - "display_name": "SecondaryNameNode Web UI Port", - "name": "dfs_secondary_http_port", - "value": "50090" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hdfs-service.json b/sahara/plugins/cdh/v5_3_0/resources/hdfs-service.json deleted file mode 100644 index cafb2a29..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hdfs-service.json +++ /dev/null @@ -1,698 +0,0 @@ -[ - { - "desc": "Timeout in milliseconds for the parallel RPCs made in DistributedFileSystem#getFileBlockStorageLocations(). This value is only emitted for Impala.", - "display_name": "HDFS File Block Storage Location Timeout", - "name": "dfs_client_file_block_storage_locations_timeout", - "value": "10000" - }, - { - "desc": "The domain to use for the HTTP cookie that stores the authentication token. In order for authentiation to work correctly across all Hadoop nodes' web-consoles the domain must be correctly set. Important: when using IP addresses, browsers ignore cookies with domain settings. For this setting to work properly all nodes in the cluster must be configured to generate URLs with hostname.domain names on it.", - "display_name": "Hadoop HTTP Authentication Cookie Domain", - "name": "hadoop_http_auth_cookie_domain", - "value": "" - }, - { - "desc": "The user that this service's processes should run as (except the HttpFS server, which has its own user)", - "display_name": "System User", - "name": "process_username", - "value": "hdfs" - }, - { - "desc": "Event filters are defined in a JSON object like the following:
\n\n\n{\n \"defaultAction\" : (\"accept\", \"discard\"),\n \"rules\" : [\n {\n \"action\" : (\"accept\", \"discard\"),\n \"fields\" : [\n {\n \"name\" : \"fieldName\",\n \"match\" : \"regex\"\n }\n ]\n }\n ]\n}\n\n\n\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n
\n\n\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n
\n\n\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n
\n\n\nThe following is the list of fields that can be filtered for HDFS events:\n
\n\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "Enable SSL encryption for HDFS, MapReduce, and YARN web UIs, as well as encrypted shuffle for MapReduce and Yarn.", - "display_name": "Hadoop SSL Enabled", - "name": "hdfs_hadoop_ssl_enabled", - "value": "false" - }, - { - "desc": "List of Kerberos realms that Hadoop services should trust. If empty, defaults to the default_realm property configured in the krb5.conf file. After changing this value and restarting the service, all services depending on this service must also be restarted. Adds mapping rules for each domain to the hadoop.security.auth_to_local property in core-site.xml.", - "display_name": "Trusted Kerberos Realms", - "name": "trusted_realms", - "value": "" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "Enables the health check that verifies that the failover controllers associated with this service are healthy and running.", - "display_name": "Failover Controllers Healthy", - "name": "failover_controllers_healthy_enabled", - "value": "true" - }, - { - "desc": "The attribute of the group object that identifies the users that are members of the group. The default will usually be appropriate for any LDAP installation.", - "display_name": "Hadoop User Group Mapping LDAP Group Membership Attribute", - "name": "hadoop_group_mapping_ldap_member_attr", - "value": "member" - }, - { - "desc": "Comma separated list of users allowed to do short circuit read. A short circuit read allows a client co-located with the data to read HDFS file blocks directly from HDFS. If empty, will default to the DataNode process' user.", - "display_name": "DataNode Local Path Access Users", - "name": "dfs_block_local_path_access_user", - "value": null - }, - { - "desc": "The timeout, in milliseconds, to use with the Cloudera Manager agent-based fencer.", - "display_name": "Timeout for Cloudera Manager Fencing Strategy", - "name": "dfs_ha_fencing_cloudera_manager_timeout_millis", - "value": "10000" - }, - { - "desc": "Enable collection of audit events from the service's roles.", - "display_name": "Enable Collection", - "name": "navigator_audit_enabled", - "value": "true" - }, - { - "desc": "Maximum bandwidth used for image transfer in bytes per second. This can help keep normal NameNode operations responsive during checkpointing. A default value of 0 indicates that throttling is disabled.", - "display_name": "FsImage Transfer Bandwidth", - "name": "dfs_image_transfer_bandwidthPerSec", - "value": "0" - }, - { - "desc": "The user the management services will impersonate as when connecting to HDFS. Defaults to 'hdfs', a superuser.", - "display_name": "HDFS User to Impersonate", - "name": "hdfs_user_to_impersonate", - "value": "hdfs" - }, - { - "desc": "The short name of Hue's Kerberos principal", - "display_name": "Hue's Kerberos Principal Short Name", - "name": "hue_kerberos_principal_shortname", - "value": "hue" - }, - { - "desc": "File path to the SSL keystore containing the SSL certificate required by the LDAP server.", - "display_name": "Hadoop User Group Mapping LDAP SSL Keystore", - "name": "hadoop_group_mapping_ldap_keystore", - "value": "" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the HDFS user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", - "display_name": "HDFS Proxy User Hosts", - "name": "hdfs_proxy_user_hosts_list", - "value": "*" - }, - { - "desc": "The minimal block replication.", - "display_name": "Minimal Block Replication", - "name": "dfs_replication_min", - "value": "1" - }, - { - "desc": "The maximal block replication.", - "display_name": "Maximal Block Replication", - "name": "dfs_replication_max", - "value": "512" - }, - { - "desc": "The service monitor will use these permissions to create the directory and files to test if the hdfs service is healthy. Permissions are specified using the 10-character unix-symbolic format e.g. '-rwxr-xr-x'.", - "display_name": "HDFS Health Canary Directory Permissions", - "name": "firehose_hdfs_canary_directory_permissions", - "value": "-rwxrwxrwx" - }, - { - "desc": "Enable authorization", - "display_name": "Hadoop Secure Authorization", - "name": "hadoop_security_authorization", - "value": "false" - }, - { - "desc": "The Key Management Server used by HDFS. This must be set in order to use encryption for data at rest.", - "display_name": "KMS Service", - "name": "kms_service", - "value": null - }, - { - "desc": "The attribute of the group object that identifies the group name. The default will usually be appropriate for all LDAP systems.", - "display_name": "Hadoop User Group Mapping LDAP Group Name Attribute", - "name": "hadoop_group_mapping_ldap_group_name_attr", - "value": "cn" - }, - { - "desc": "Enables DataNode support for the experimental DistributedFileSystem.getFileVBlockStorageLocations API. Applicable to CDH 4.1 and onwards.", - "display_name": "Enable HDFS Block Metadata API", - "name": "dfs_datanode_hdfs_blocks_metadata_enabled", - "value": "true" - }, - { - "desc": "The tolerance window that will be used in HDFS service tests that depend on detection of the active NameNode.", - "display_name": "Active NameNode Detection Window", - "name": "hdfs_active_namenode_detecton_window", - "value": "3" - }, - { - "desc": "Default block replication. The number of replications to make when the file is created. The default value is used if a replication number is not specified.", - "display_name": "Replication Factor", - "name": "dfs_replication", - "value": "3" - }, - { - "desc": "Comma-delimited list of groups that you want to allow the HTTP user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'. This is used by WebHCat.", - "display_name": "HTTP Proxy User Groups", - "name": "HTTP_proxy_user_groups_list", - "value": "*" - }, - { - "desc": "The name of the system group shared by all the core Hadoop services.", - "display_name": "Shared Hadoop Group Name", - "name": "hdfs_hadoop_group_name", - "value": "hadoop" - }, - { - "desc": "The amount of time to wait for HDFS filesystem image transfer from NameNode to complete.", - "display_name": "FsImage Transfer Timeout", - "name": "dfs_image_transfer_timeout", - "value": "60000" - }, - { - "desc": "ACLs (Access Control Lists) enhance the existing HDFS permission model to support controlling file access for arbitrary combinations of users and groups instead of a single owner, single group, and all other users. When ACLs are disabled, the NameNode rejects all attempts to set an ACL.", - "display_name": "Enable Access Control Lists", - "name": "dfs_namenode_acls_enabled", - "value": "false" - }, - { - "desc": "An additional filter to use when searching for LDAP users. The default will usually be appropriate for Active Directory installations. If connecting to a generic LDAP server, ''sAMAccountName'' will likely be replaced with ''uid''. {0} is a special string used to denote where the username fits into the filter.", - "display_name": "Hadoop User Group Mapping LDAP User Search Filter", - "name": "hadoop_group_mapping_ldap_user_filter", - "value": "(&(objectClass=user)(sAMAccountName={0}))" - }, - { - "desc": "List of fencing methods to use for service fencing. shell(./cloudera_manager_agent_fencer.py) is a fencing mechanism designed to use the Cloudera Manager agent. The sshfence method uses SSH. If using custom fencers (that may communicate with shared store, power units, or network switches), use the shell to invoke them.", - "display_name": "HDFS High Availability Fencing Methods", - "name": "dfs_ha_fencing_methods", - "value": "shell(./cloudera_manager_agent_fencer.py)" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the environment of HDFS replication jobs.", - "display_name": "HDFS Replication Advanced Configuration Snippet (Safety Valve)", - "name": "hdfs_replication_env_safety_valve", - "value": null - }, - { - "desc": "The default block size in bytes for new HDFS files. Note that this value is also used as the HBase Region Server HLog block size.", - "display_name": "HDFS Block Size", - "name": "dfs_block_size", - "value": "134217728" - }, - { - "desc": "Path on the DataNode's local file system to a UNIX domain socket used for communication between the DataNode and local HDFS clients. This socket is used for Short Circuit Reads. Only the HDFS System User and \"root\" should have write access to the parent directory and all of its ancestors. This property is supported in CDH 4.2 or later deployments.", - "display_name": "UNIX Domain Socket path", - "name": "dfs_domain_socket_path", - "value": "/var/run/hdfs-sockets/dn" - }, - { - "desc": "Algorithm to encrypt data transfer between DataNodes and clients, and among DataNodes. 3des is more cryptographically secure, but rc4 is substantially faster.", - "display_name": "Data Transfer Encryption Algorithm", - "name": "dfs_encrypt_data_transfer_algorithm", - "value": "rc4" - }, - { - "desc": "The health check thresholds of the number of under-replicated blocks. Specified as a percentage of the total number of blocks.", - "display_name": "Under-replicated Block Monitoring Thresholds", - "name": "hdfs_under_replicated_blocks_thresholds", - "value": "{\"critical\":\"40.0\",\"warning\":\"10.0\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml. Applies to configurations of all roles in this service except client configuration.", - "display_name": "HDFS Service Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "hdfs_service_config_safety_valve", - "value": null - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the HTTP user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'. This is used by WebHCat.", - "display_name": "HTTP Proxy User Hosts", - "name": "HTTP_proxy_user_hosts_list", - "value": "*" - }, - { - "desc": "\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n
\n\n\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n
\n\nEvent trackers are defined in a JSON object like the following:
\n\n\n{\n \"timeToLive\" : [integer],\n \"fields\" : [\n {\n \"type\" : [string],\n \"name\" : [string]\n }\n ]\n}\n\n\n\nWhere:\n
\n\n\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n
\n\n\nThe following is the list of fields that can be used to compare HDFS events:\n
\n\nThe configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "Minimum number of worker threads in the Hive Metastore Server's thread pool", - "display_name": "Min Hive Metastore Server Threads", - "name": "hive_metastore_min_threads", - "value": "200" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "When computing the overall Hive Metastore Server health, consider the host's health.", - "display_name": "Hive Metastore Server Host Health Test", - "name": "hivemetastore_host_health_enabled", - "value": "true" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Hive Metastore Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Hive Metastore Server in Bytes", - "name": "hive_metastore_java_heapsize", - "value": "1073741824" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "hivemetastore_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Port on which Hive Metastore Server will listen for connections.", - "display_name": "Hive Metastore Server Port", - "name": "hive_metastore_port", - "value": "9083" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", - "display_name": "Hive Metastore Server Advanced Configuration Snippet (Safety Valve) for hive-site.xml", - "name": "hive_metastore_config_safety_valve", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Hive Metastore Server logs. Typically used by log4j.", - "display_name": "Hive Metastore Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Enables the health test that the Hive Metastore Server's process state is consistent with the role configuration", - "display_name": "Hive Metastore Server Process Health Test", - "name": "hivemetastore_scm_health_enabled", - "value": "true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "Directory where Hive Metastore Server will place its log files.", - "display_name": "Hive Metastore Server Log Directory", - "name": "hive_log_dir", - "value": "/var/log/hive" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The delegation token store implementation class. Use DBTokenStore for Highly Available Metastore Configuration.", - "display_name": "Hive Metastore Delegation Token Store", - "name": "hive_metastore_delegation_token_store", - "value": "org.apache.hadoop.hive.thrift.MemoryTokenStore" - }, - { - "desc": "The minimum log level for Hive Metastore Server logs", - "display_name": "Hive Metastore Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Hive Metastore Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "hive_metastore_env_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hive-hiveserver2.json b/sahara/plugins/cdh/v5_3_0/resources/hive-hiveserver2.json deleted file mode 100644 index 42aa3a77..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hive-hiveserver2.json +++ /dev/null @@ -1,272 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Operation will be closed when it's not accessed for this duration of time, which can be disabled by setting to zero. With a positive value, it's checked for operations in terminal state only (FINISHED, CANCELED, CLOSED, ERROR). With a negative value, it's checked for all of the operations regardless of state. Time in miliseconds.", - "display_name": "Idle Operation Timeout", - "name": "hiveserver2_idle_operation_timeout", - "value": "0" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Local Directory where Hive stores jars downloaded for remote file systems (HDFS). If not specified, Hive will use a default location.", - "display_name": "Hive Downloaded Resources Directory", - "name": "hiveserver2_downloaded_resources_dir", - "value": null - }, - { - "desc": "HiveServer2 will impersonate the beeline client user when talking to other services such as Mapreduce and Hdfs.", - "display_name": "HiveServer2 Enable Impersonation", - "name": "hiveserver2_enable_impersonation", - "value": "true" - }, - { - "desc": "Minimum number of worker threads in HiveServer2's thread pool", - "display_name": "Min HiveServer2 Threads", - "name": "hiveserver2_min_threads", - "value": "5" - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "Port on which HiveServer2 will listen for connections.", - "display_name": "HiveServer2 Port", - "name": "hs2_thrift_address_port", - "value": "10000" - }, - { - "desc": "Enable optimization that converts common join into map join based on the input file size.", - "display_name": "Enable MapJoin Optimization", - "name": "hiveserver2_enable_mapjoin", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for HiveServer2 logs. Typically used by log4j.", - "display_name": "HiveServer2 Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", - "display_name": "HiveServer2 Advanced Configuration Snippet (Safety Valve) for hive-site.xml", - "name": "hive_hs2_config_safety_valve", - "value": null - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The check interval for session/operation timeout, which can be disabled by setting to zero or a negative value. Time in miliseconds.", - "display_name": "Session Check Interval", - "name": "hiveserver2_session_check_interval", - "value": "0" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "HiveServer2 Environment Advanced Configuration Snippet (Safety Valve)", - "name": "hive_hs2_env_safety_valve", - "value": null - }, - { - "desc": "Directory in HDFS where Hive writes intermediate data between Map Reduce jobs. If not specified, Hive will use a default location.", - "display_name": "Hive HDFS Scratch Directory", - "name": "hiveserver2_exec_scratchdir", - "value": null - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "When enabled, HiveServer2 will log EXPLAIN EXTENDED output for every query at INFO log4j level.", - "display_name": "Enable Explain Logging", - "name": "hiveserver2_enable_explain_output", - "value": "true" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "HiveServer2 Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "When computing the overall HiveServer2 health, consider the host's health.", - "display_name": "HiveServer2 Host Health Test", - "name": "hiveserver2_host_health_enabled", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "Session will be closed when it's not accessed for this duration, which can be disabled by setting to zero or a negative value. Time in miliseconds.", - "display_name": "Idle Session Timeout", - "name": "hiveserver2_idle_session_timeout", - "value": "0" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for HiveServer2 logs. Typically used by log4j.", - "display_name": "HiveServer2 Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "Local Directory where Hive stores jars and data when performing a map join optimization. If not specified, Hive will use a default location.", - "display_name": "Hive Local Scratch Directory", - "name": "hiveserver2_exec_local_scratchdir", - "value": null - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Enables the health test that the HiveServer2's process state is consistent with the role configuration", - "display_name": "HiveServer2 Process Health Test", - "name": "hiveserver2_scm_health_enabled", - "value": "true" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of HiveServer2 in Bytes", - "name": "hiveserver2_java_heapsize", - "value": "268435456" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "Directory where HiveServer2 will place its log files.", - "display_name": "HiveServer2 Log Directory", - "name": "hive_log_dir", - "value": "/var/log/hive" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "Maximum number of worker threads in HiveServer2's thread pool", - "display_name": "Max HiveServer2 Threads", - "name": "hiveserver2_max_threads", - "value": "100" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "hiveserver2_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The minimum log level for HiveServer2 logs", - "display_name": "HiveServer2 Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for HiveServer2", - "name": "hiveserver2_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hive-metastore-sentry-safety.xml b/sahara/plugins/cdh/v5_3_0/resources/hive-metastore-sentry-safety.xml deleted file mode 100644 index 8395f76b..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hive-metastore-sentry-safety.xml +++ /dev/null @@ -1,15 +0,0 @@ -\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n
\n\n\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n
\n\nEvent trackers are defined in a JSON object like the following:
\n\n\n{\n \"timeToLive\" : [integer],\n \"fields\" : [\n {\n \"type\" : [string],\n \"name\" : [string]\n }\n ]\n}\n\n\n\nWhere:\n
\n\n\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n
\n\n\nThe following is the list of fields that can be used to compare Hive events:\n
\n\nEvent filters are defined in a JSON object like the following:
\n\n\n{\n \"defaultAction\" : (\"accept\", \"discard\"),\n \"rules\" : [\n {\n \"action\" : (\"accept\", \"discard\"),\n \"fields\" : [\n {\n \"name\" : \"fieldName\",\n \"match\" : \"regex\"\n }\n ]\n }\n ]\n}\n\n\n\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n
\n\n\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n
\n\n\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n
\n\n\nThe following is the list of fields that can be filtered for Hive events:\n
\n\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "Instead of talking to Hive Metastore Server for Metastore information, Hive clients will talk directly to the Metastore database.", - "display_name": "Bypass Hive Metastore Server", - "name": "hive_bypass_metastore_server", - "value": "false" - }, - { - "desc": "Below this size Hive will use a single threaded copy while above this size Hive will use DistCp.", - "display_name": "Hive Copy Large File Size", - "name": "hive_exec_copyfile_maxsize", - "value": "33554432" - }, - { - "desc": "User for Hive Metastore database", - "display_name": "Hive Metastore Database User", - "name": "hive_metastore_database_user", - "value": "hive" - }, - { - "desc": "Perform DataNucleus validation of metadata during startup. Note: when enabled, Hive will log DataNucleus warnings even though Hive will function normally.", - "display_name": "Hive Metastore Database DataNucleus Metadata Validation", - "name": "hive_metastore_database_datanucleus_metadata_validation", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for navigator.client.properties.", - "display_name": "Hive Client Advanced Configuration Snippet (Safety Valve) for navigator.client.properties", - "name": "navigator_client_config_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds of the overall WebHCat Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" WebHCat Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" WebHCat Servers falls below the critical threshold.", - "display_name": "Healthy WebHCat Server Monitoring Thresholds", - "name": "hive_webhcats_healthy_thresholds", - "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into sentry-site.xml. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Hive Service Advanced Configuration Snippet (Safety Valve) for sentry-site.xml", - "name": "hive_server2_sentry_safety_valve", - "value": null - }, - { - "desc": "Size per reducer. If the input size is 10GiB and this is set to 1GiB, Hive will use 10 reducers.", - "display_name": "Hive Bytes Per Reducer", - "name": "hive_bytes_per_reducer", - "value": "1073741824" - }, - { - "desc": "Enable collection of audit events from the service's roles.", - "display_name": "Enable Collection", - "name": "navigator_audit_enabled", - "value": "true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Allows URIs when defining privileges in per-database policy files. Warning: Typically, this configuration should be disabled. Enabling it would allow database policy file owner (which is generally not Hive admin user) to grant load privileges to any directory with read access to Hive admin user, including databases controlled by other database policy files.", - "display_name": "Allow URIs in Database Policy File", - "name": "sentry_allow_uri_db_policyfile", - "value": "false" - }, - { - "desc": "In unsecure mode, setting this property to true will cause the Metastore Server to execute DFS operations using the client's reported user and group permissions. Cloudera Manager will set this for all clients and servers.", - "display_name": "Set User and Group Information", - "name": "hive_set_ugi", - "value": "true" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "MapReduce jobs are run against this service.", - "display_name": "MapReduce Service", - "name": "mapreduce_yarn_service", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hive-site.xml b/sahara/plugins/cdh/v5_3_0/resources/hive-site.xml deleted file mode 100644 index 7e70626f..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hive-site.xml +++ /dev/null @@ -1,61 +0,0 @@ - - - -The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of WebHCat Server in Bytes", - "name": "hive_webhcat_java_heapsize", - "value": "268435456" - }, - { - "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", - "display_name": "WebHCat Server Advanced Configuration Snippet (Safety Valve) for hive-site.xml", - "name": "hive_webhcat_hive_config_safety_valve", - "value": null - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "webhcat_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "WebHCat Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "WebHCat Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "hive_webhcat_env_safety_valve", - "value": null - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for WebHCat Server", - "name": "hive_webhcat_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "Enables the health test that the WebHCat Server's process state is consistent with the role configuration", - "display_name": "WebHCat Server Process Health Test", - "name": "webhcat_scm_health_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into webhcat-site.xml for this role only.", - "display_name": "WebHCat Server Advanced Configuration Snippet (Safety Valve) for webhcat-site.xml", - "name": "hive_webhcat_config_safety_valve", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "When computing the overall WebHCat Server health, consider the host's health.", - "display_name": "WebHCat Server Host Health Test", - "name": "webhcat_host_health_enabled", - "value": "true" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The minimum log level for WebHCat Server logs", - "display_name": "WebHCat Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Directory where WebHCat Server will place its log files.", - "display_name": "WebHCat Server Log Directory", - "name": "hcatalog_log_dir", - "value": "/var/log/hcatalog" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hue-hue_server.json b/sahara/plugins/cdh/v5_3_0/resources/hue-hue_server.json deleted file mode 100644 index ca5c83f8..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hue-hue_server.json +++ /dev/null @@ -1,158 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "hue_server_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Enable HTTPS for the Hue web server.", - "display_name": "Enable HTTPS", - "name": "ssl_enable", - "value": "false" - }, - { - "desc": "Path to the SSL private key on the host running the Hue web server. This file must be readable by the Hue system user. Hue only supports a key without a passphrase.", - "display_name": "Local Path to SSL Private Key", - "name": "ssl_private_key", - "value": null - }, - { - "desc": "Directory where Hue Server will place its log files.", - "display_name": "Hue Server Log Directory", - "name": "hue_server_log_dir", - "value": "/var/log/hue" - }, - { - "desc": "Timeout in seconds for Thrift calls to HiveServer2 and Impala.", - "display_name": "HiveServer2 and Impala Thrift Connection Timeout", - "name": "hs2_conn_timeout", - "value": "120" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Location on HDFS where the jobsub examples and templates are stored.", - "display_name": "Jobsub Examples and Templates Directory", - "name": "hue_server_remote_data_dir", - "value": "/user/hue/jobsub" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "When computing the overall Hue Server health, consider the host's health.", - "display_name": "Hue Server Host Health Test", - "name": "hue_server_host_health_enabled", - "value": "true" - }, - { - "desc": "Port to use to connect to the Hue server.", - "display_name": "Hue HTTP Port", - "name": "hue_http_port", - "value": "8888" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into hue_safety_valve_server.ini for this role only.", - "display_name": "Hue Server Advanced Configuration Snippet (Safety Valve) for hue_safety_valve_server.ini", - "name": "hue_server_hue_safety_valve", - "value": null - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Path to the SSL certificate on the host running the Hue web server. This file must be readable by the Hue system user.", - "display_name": "Local Path to SSL Certificate", - "name": "ssl_certificate", - "value": null - }, - { - "desc": "If enabled, the Hue server binds to the wildcard address (\"0.0.0.0\") for its ports.", - "display_name": "Bind Hue Server to Wildcard Address", - "name": "hue_server_bind_wildcard", - "value": "false" - }, - { - "desc": "Enables the health test that the Hue Server's process state is consistent with the role configuration", - "display_name": "Hue Server Process Health Test", - "name": "hue_server_scm_health_enabled", - "value": "true" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Random string used for secure hashing in the session store.", - "display_name": "Secret Key", - "name": "secret_key", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hue-kt_renewer.json b/sahara/plugins/cdh/v5_3_0/resources/hue-kt_renewer.json deleted file mode 100644 index 06af948e..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hue-kt_renewer.json +++ /dev/null @@ -1,110 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Directory where Kerberos Ticket Renewer will place its log files.", - "display_name": "Kerberos Ticket Renewer Log Directory", - "name": "kt_renewer_log_dir", - "value": "/var/log/hue" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "When computing the overall Kerberos Ticket Renewer health, consider the host's health.", - "display_name": "Kerberos Ticket Renewer Host Health Test", - "name": "kt_renewer_host_health_enabled", - "value": "true" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "Interval in seconds with which Hue's Kerberos ticket will get renewed.", - "display_name": "Hue Keytab Renewal Interval", - "name": "keytab_reinit_frequency", - "value": "3600" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "kt_renewer_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Enables the health test that the Kerberos Ticket Renewer's process state is consistent with the role configuration", - "display_name": "Kerberos Ticket Renewer Process Health Test", - "name": "kt_renewer_scm_health_enabled", - "value": "true" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/hue-service.json b/sahara/plugins/cdh/v5_3_0/resources/hue-service.json deleted file mode 100644 index c2c852ea..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/hue-service.json +++ /dev/null @@ -1,344 +0,0 @@ -[ - { - "desc": "Create users in Hue when they try to login with their LDAP credentials. For use when using LdapBackend for Hue authentication.", - "display_name": "Create LDAP users on login", - "name": "create_users_on_login", - "value": "true" - }, - { - "desc": "Number of threads used by the Hue web server.", - "display_name": "Hue Web Server Threads", - "name": "cherrypy_server_threads", - "value": "10" - }, - { - "desc": "Thrift server to use for HBase app.", - "display_name": "HBase Thrift Server", - "name": "hue_hbase_thrift", - "value": null - }, - { - "desc": "Name of the Oozie service that this Hue service instance depends on", - "display_name": "Oozie Service", - "name": "oozie_service", - "value": null - }, - { - "desc": "The distinguished name to use as a search base for finding users and groups.", - "display_name": "LDAP Search Base", - "name": "base_dn", - "value": null - }, - { - "desc": "Comma-separated list of regular expressions, which match 'host:port' of requested proxy target.", - "display_name": "Whitelist", - "name": "whitelist", - "value": "(localhost|127\\.0\\.0\\.1):(50030|50070|50060|50075)" - }, - { - "desc": "Name of the Sentry service that this Hue service instance depends on", - "display_name": "Sentry Service", - "name": "sentry_service", - "value": null - }, - { - "desc": "Name of the HBase service that this Hue service instance depends on", - "display_name": "HBase Service", - "name": "hbase_service", - "value": null - }, - { - "desc": "Base filter for searching for groups", - "display_name": "LDAP Group Filter", - "name": "group_filter", - "value": null - }, - { - "desc": "Time zone name.", - "display_name": "Time Zone", - "name": "time_zone", - "value": "America/Los_Angeles" - }, - { - "desc": "The health test thresholds of the overall Kerberos Ticket Renewer health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Kerberos Ticket Renewers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Kerberos Ticket Renewers falls below the critical threshold.", - "display_name": "Healthy Kerberos Ticket Renewer Monitoring Thresholds", - "name": "hue_kt_renewers_healthy_thresholds", - "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" - }, - { - "desc": "The user that this service's processes should run as.", - "display_name": "System User", - "name": "process_username", - "value": "hue" - }, - { - "desc": "The health test thresholds of the overall Hue Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Hue Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Hue Servers falls below the critical threshold.", - "display_name": "Healthy Hue Server Monitoring Thresholds", - "name": "hue_hue_servers_healthy_thresholds", - "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The group name attribute in the LDAP schema.", - "display_name": "LDAP Group Name Attribute", - "name": "group_name_attr", - "value": null - }, - { - "desc": "HDFS directory used for storing temporary files.", - "display_name": "HDFS Temporary Directory", - "name": "hdfs_tmp_dir", - "value": "/tmp" - }, - { - "desc": "Name of the Solr service that this Hue service instance depends on", - "display_name": "Solr Service", - "name": "solr_service", - "value": null - }, - { - "desc": "If the database is SQLite3, this is the filename of the database to use, and the directory of this file must be writable by the 'hue' user.", - "display_name": "Hue Database Directory", - "name": "database_dir", - "value": "/var/lib/hue/desktop.db" - }, - { - "desc": "The name of a default group that users will be added to at creation time.", - "display_name": "Default User Group", - "name": "default_user_group", - "value": null - }, - { - "desc": "When you enable anonymous usage data collection Hue tracks anonymised pages and application versions in order to gather information about each application's usage levels. The data collected does not include any hostnames or IDs. Data collection option is available on CDH 4.4 and later deployments.", - "display_name": "Enable Usage Data Collection", - "name": "usage_data_collection_enable", - "value": "true" - }, - { - "desc": "Name of the ZooKeeper service that this Hue service instance depends on", - "display_name": "ZooKeeper Service", - "name": "zookeeper_service", - "value": null - }, - { - "desc": "Distinguished name of the user to bind as -- not necessary if the LDAP server supports anonymous searches.", - "display_name": "LDAP Bind User", - "name": "bind_dn", - "value": null - }, - { - "desc": "URL of LDAP Server", - "display_name": "LDAP URL", - "name": "ldap_url", - "value": null - }, - { - "desc": "Whether to use StartTLS (as opposed to ldaps) to communicate securely with the LDAP server. This is only effective when the LDAP certificate is specified.", - "display_name": "Use StartTLS", - "name": "use_start_tls", - "value": "true" - }, - { - "desc": "Name of the Sqoop service that this Hue service instance depends on", - "display_name": "Sqoop Service", - "name": "sqoop_service", - "value": null - }, - { - "desc": "Active Directory Domain", - "display_name": "NT Domain", - "name": "nt_domain", - "value": null - }, - { - "desc": "Class that defines extra accessor methods for user objects.", - "display_name": "User Augmentor", - "name": "user_augmentor", - "value": "desktop.auth.backend.DefaultUserAugmentor" - }, - { - "desc": "File where the database gets dumped to or loaded from.", - "display_name": "Database Dump File", - "name": "database_dump_file", - "value": "/tmp/hue_database_dump.json" - }, - { - "desc": "LDAP Username Pattern for use with non-Active Directory LDAP implementations. Must contain the special '<username>' string for replacement during authentication.", - "display_name": "LDAP Username Pattern", - "name": "ldap_username_pattern", - "value": null - }, - { - "desc": "The password of the bind user.", - "display_name": "LDAP Bind Password", - "name": "bind_password", - "value": null - }, - { - "desc": "The group that this service's processes should run as.", - "display_name": "System Group", - "name": "process_groupname", - "value": "hue" - }, - { - "desc": "In debug mode, Django displays a detailed traceback when an exception occurs. Note that the debugging information may contain sensitive data. Note also that Django remembers every SQL query it executes while in debug mode, which will rapidly consume memory.", - "display_name": "Enable Django Debug Mode", - "name": "django_debug_enable", - "value": "false" - }, - { - "desc": "LDAP certificate for authentication over TLS", - "display_name": "LDAP Certificate", - "name": "ldap_cert", - "value": null - }, - { - "desc": "Type of database used for Hue", - "display_name": "Hue Database Type", - "name": "database_type", - "value": "sqlite3" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "When computing the overall HUE health, consider Beeswax Server's health", - "display_name": "Beeswax Server Role Health Test", - "name": "hue_beeswax_server_health_enabled", - "value": "true" - }, - { - "desc": "HTTPFS role or Namenode (if webhdfs is enabled) that hue can use to communicate with HDFS.", - "display_name": "HDFS Web Interface Role", - "name": "hue_webhdfs", - "value": null - }, - { - "desc": "Mode of authenticating login credentials. Select desktop.auth.backend.LdapBackend to use LDAP to authenticate login credentials. LDAP requires you to also set the LDAP URL, NT Domain, and optionally LDAP certificate if you are using secure LDAP. Select desktop.auth.backend.PamBackend to use PAM to authenticate login credentials.", - "display_name": "Authentication Backend", - "name": "auth_backend", - "value": "desktop.auth.backend.AllowFirstUserDjangoBackend" - }, - { - "desc": "The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "The username to use to log into the Hue database. Not necessary for SQLite3.", - "display_name": "Hue Database Username", - "name": "database_user", - "value": "hue" - }, - { - "desc": "Enable debug output in HTTP Internal Server Error (status 500) responses. Note that the debugging information may contain sensitive data. If Enable Django Debug Mode is set, this is automatically enabled.", - "display_name": "Enable Debugging of Internal Server Error Responses", - "name": "http_500_debug_enable", - "value": "false" - }, - { - "desc": "Comma-separated list of regular expressions, which match any prefix of 'host:port/path' of requested proxy target. This does not support matching GET parameters.", - "display_name": "Blacklist", - "name": "blacklist", - "value": "()" - }, - { - "desc": "For advanced use only, a string to be inserted into sentry-site.xml. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Hue Service Advanced Configuration Snippet (Safety Valve) for sentry-site.xml", - "name": "hue_sentry_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Hue Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "hue_service_env_safety_valve", - "value": null - }, - { - "desc": "Name of the Impala service that this Hue service instance depends on", - "display_name": "Impala Service", - "name": "impala_service", - "value": null - }, - { - "desc": "Name of Hue database.", - "display_name": "Hue Database Name", - "name": "database_name", - "value": "hue" - }, - { - "desc": "The attribute of the group object which identifies the members of the group.", - "display_name": "LDAP Group Membership Attribute", - "name": "group_member_attr", - "value": null - }, - { - "desc": "Name of the Hive service that this Hue service instance depends on", - "display_name": "Hive Service", - "name": "hive_service", - "value": null - }, - { - "desc": "The username attribute in the LDAP schema.", - "display_name": "LDAP Username Attribute", - "name": "user_name_attr", - "value": null - }, - { - "desc": "Default encoding for site data.", - "display_name": "Default Site Encoding", - "name": "default_site_encoding", - "value": "utf" - }, - { - "desc": "Search Bind Authentication connects to the LDAP server using credentials provided in the 'bind_dn' and 'bind_password' configurations. If these configurations are not set, then an anonymous search is performed.", - "display_name": "Use Search Bind Authentication", - "name": "search_bind_authentication", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into hue_safety_valve.ini. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Hue Service Advanced Configuration Snippet (Safety Valve) for hue_safety_valve.ini", - "name": "hue_service_safety_valve", - "value": null - }, - { - "desc": "Port on host where the Hue database is running. Not necessary for SQLite3.", - "display_name": "Hue Database Port", - "name": "database_port", - "value": "3306" - }, - { - "desc": "Password for Hue database. Not necessary for SQLite3.", - "display_name": "Hue Database Password", - "name": "database_password", - "value": "" - }, - { - "desc": "Base filter for searching for users.", - "display_name": "LDAP User Filter", - "name": "user_filter", - "value": null - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "Name of host where the Hue database is running. Not necessary for SQLite3.", - "display_name": "Hue Database Hostname", - "name": "database_host", - "value": "localhost" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/impala-catalogserver.json b/sahara/plugins/cdh/v5_3_0/resources/impala-catalogserver.json deleted file mode 100644 index 559956d1..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/impala-catalogserver.json +++ /dev/null @@ -1,236 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", - "display_name": "Health Check Startup Tolerance", - "name": "catalogserver_startup_tolerance", - "value": "5" - }, - { - "desc": "Port where Catalog Server debug web server runs.", - "display_name": "Catalog Server HTTP Server Port", - "name": "catalogserver_webserver_port", - "value": "25020" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "catalogserver_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "catalogserver_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Port where Catalog Server is exported.", - "display_name": "Catalog Server Service Port", - "name": "catalog_service_port", - "value": "26000" - }, - { - "desc": "Buffer log messages logged at this level or lower (-1 means don't buffer; 0 means buffer INFO only; 1 means buffer WARNING only, ...)", - "display_name": "Catalog Server Log Buffer Level", - "name": "logbuflevel", - "value": "0" - }, - { - "desc": "If true, loads catalog metadata in the background. If false, metadata is loaded lazily (on access). Only effective in CDH 5+ and Impala 1.2.4+.", - "display_name": "Load Catalog in Background", - "name": "load_catalog_in_background", - "value": "true" - }, - { - "desc": "The location of the debug web server's SSL certificate file, in .pem format. If empty, webserver SSL support is not enabled.", - "display_name": "SSL Certificate File Location", - "name": "webserver_certificate_file", - "value": null - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "catalogserver_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "The amount of time to wait for the Catalog Server to fully start up and connect to the StateStore before enforcing the connectivity check.", - "display_name": "Catalog Server Connectivity Tolerance at Startup", - "name": "catalogserver_connectivity_tolerance", - "value": "180" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Timeout for requests to the Hive Metastore Server from Catalog Server. Consider increasing this if you have tables with a lot of metadata and see timeout errors.", - "display_name": "Catalog Server Hive Metastore Connection Timeout", - "name": "hive_metastore_timeout", - "value": "3600" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Username for Catalog Server web server authentication.", - "display_name": "Catalog Server Web Server Username", - "name": "webserver_htpassword_user", - "value": null - }, - { - "desc": "Password for Catalog Server web server authentication.", - "display_name": "Catalog Server Web Server User Password", - "name": "webserver_htpassword_password", - "value": null - }, - { - "desc": "Enables the health test that the Impala Catalog Server's process state is consistent with the role configuration", - "display_name": "Impala Catalog Server Process Health Test", - "name": "catalogserver_scm_health_enabled", - "value": "true" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be added (verbatim) to Catalog Server command line flags. Key names should begin with a hyphen(-). For example: -log_filename=foo.log", - "display_name": "Catalog Server Command Line Argument Advanced Configuration Snippet (Safety Valve)", - "name": "catalogd_cmd_args_safety_valve", - "value": null - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "Catalog Server HDFS Advanced Configuration Snippet (Safety Valve)", - "name": "catalogd_hdfs_site_conf_safety_valve", - "value": null - }, - { - "desc": "Directory where Catalog Server will place its log files.", - "display_name": "Catalog Server Log Directory", - "name": "log_dir", - "value": "/var/log/catalogd" - }, - { - "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", - "display_name": "Catalog Server HBase Advanced Configuration Snippet (Safety Valve)", - "name": "catalogd_hbase_conf_safety_valve", - "value": null - }, - { - "desc": "Enable/Disable Catalog Server web server. This web server contains useful information about Catalog Server daemon.", - "display_name": "Enable Catalog Server Web Server", - "name": "catalogd_enable_webserver", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Enables the health test that verifies the Catalog Server is connected to the StateStore", - "display_name": "Catalog Server Connectivity Health Test", - "name": "catalogserver_connectivity_health_enabled", - "value": "true" - }, - { - "desc": "When computing the overall Impala Catalog Server health, consider the host's health.", - "display_name": "Impala Catalog Server Host Health Test", - "name": "catalogserver_host_health_enabled", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Impala Catalog Server logs. Typically used by log4j.", - "display_name": "Impala Catalog Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", - "display_name": "Catalog Server Hive Advanced Configuration Snippet (Safety Valve)", - "name": "catalogd_hive_conf_safety_valve", - "value": null - }, - { - "desc": "Verbose logging level for the GLog logger. These messages are always logged at 'INFO' log level, so this setting has no effect if Logging Threshold is set to 'WARN' or above.", - "display_name": "Catalog Server Verbose Log Level", - "name": "log_verbose_level", - "value": "1" - }, - { - "desc": "The health test thresholds on the resident size of the process.", - "display_name": "Resident Set Size Thresholds", - "name": "process_resident_set_size_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "The minimum log level for Impala Catalog Server logs", - "display_name": "Impala Catalog Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/impala-impalad.json b/sahara/plugins/cdh/v5_3_0/resources/impala-impalad.json deleted file mode 100644 index ff95924f..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/impala-impalad.json +++ /dev/null @@ -1,386 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Configures the health check thresholds for monitoring free space on the filesystems that contain Impala's scratch directories. Specified as a percentage of the capacity on the filesystem. This setting is not used if a Scratch Directories Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Impala Scratch Directories Free Space Monitoring Percentage Thresholds", - "name": "impalad_scratch_directories_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "User-defined function (UDF) libraries are copied from HDFS into this local directory.", - "display_name": "Local UDF Library Dir", - "name": "local_library_dir", - "value": "/var/lib/impala/udfs" - }, - { - "desc": "Encrypt and verify the integrity of all data spilled to disk as part of a query. This feature is only supported for Impala 2.0+ and CDH 5.2+ (which includes Impala 2.0).", - "display_name": "Disk Spill Encryption", - "name": "disk_spill_encryption", - "value": "false" - }, - { - "desc": "Maximum number of query results a client may request to be cached on a per-query basis to support restarting fetches. This option guards against unreasonably large result caches requested by clients. Requests exceeding this maximum will be rejected.", - "display_name": "Result Cache Maximum Size", - "name": "impalad_result_cache_max_size", - "value": "100000" - }, - { - "desc": "Number of seconds to wait between attempts during Llama registration.", - "display_name": "Llama Registration Wait Seconds", - "name": "llama_registration_wait_secs", - "value": "3" - }, - { - "desc": "The timeout used by the Cloudera Manager Agent's query monitor when communicating with the Impala Daemon web server, specified in seconds.", - "display_name": "Query Monitoring Timeout", - "name": "executing_queries_timeout_seconds", - "value": "5.0" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Directories where Impala Daemon will write data such as spilling information to disk to free up memory. This can potentially be large amounts of data.", - "display_name": "Impala Daemon Scratch Directories", - "name": "scratch_dirs", - "value": null - }, - { - "desc": "Buffer log messages logged at this level or lower (-1 means don't buffer; 0 means buffer INFO only; 1 means buffer WARNING only, ...)", - "display_name": "Impala Daemon Log Buffer Level", - "name": "logbuflevel", - "value": "0" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "impalad_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be added (verbatim) to Impala Daemon command-line flags. Key names should begin with a hyphen(-). For example: -log_filename=foo.log", - "display_name": "Impala Daemon Command Line Argument Advanced Configuration Snippet (Safety Valve)", - "name": "impalad_cmd_args_safety_valve", - "value": null - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "Enable or disable the Impala Daemon web server. This web server contains useful information about Impala Daemon.", - "display_name": "Enable Impala Daemon Web Server", - "name": "impalad_enable_webserver", - "value": "true" - }, - { - "desc": "The location on disk of the certificate, in .pem format, used to confirm the authenticity of the LDAP server certificate. If not set, Impala by default trusts all certificates supplied by the LDAP server.", - "display_name": "LDAP Server Certificate", - "name": "impalad_ldap_ca_certificate", - "value": null - }, - { - "desc": "Port on which HiveServer2 client requests are served by Impala Daemons.", - "display_name": "Impala Daemon HiveServer2 Port", - "name": "hs2_port", - "value": "21050" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Username for Impala Daemon webserver authentication.", - "display_name": "Impala Daemon Web Server Username", - "name": "webserver_htpassword_user", - "value": null - }, - { - "desc": "Enables audit event generation by Impala daemons. The audit log file will be placed in the directory specified by 'Impala Daemon Audit Log Directory' parameter.", - "display_name": "Enable Impala Audit Event Generation", - "name": "enable_audit_event_log", - "value": "false" - }, - { - "desc": "Maximum number of seconds that Impala attempts to register or re-register with Llama. If registration is unsuccessful, Impala cancels the action with an error, which could result in an impalad startup failure or a cancelled query. A setting of -1 seconds means try indefinitely.", - "display_name": "Llama Registration Timeout Seconds", - "name": "llama_registration_timeout_secs", - "value": "30" - }, - { - "desc": "When computing the overall Impala Daemon health, consider the host's health.", - "display_name": "Impala Daemon Host Health Test", - "name": "impalad_host_health_enabled", - "value": "true" - }, - { - "desc": "The directory in which Impala Daemon audit event log files are written. If \"Impala Audit Event Generation\" property is enabled, Impala will generate its audit logs in this directory.", - "display_name": "Impala Daemon Audit Log Directory", - "name": "audit_event_log_dir", - "value": "/var/log/impalad/audit" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "An XML string to use verbatim for the contents of fair-scheduler.xml for Impala Daemons. This configuration only has effect on Impala versions 1.3 or greater.", - "display_name": "Impala Daemon Fair Scheduler Advanced Configuration", - "name": "impalad_fair_scheduler_safety_valve", - "value": null - }, - { - "desc": "A list of key-value pairs of additional query options to pass to the Impala Daemon command line, separated by ','.", - "display_name": "Impala Daemon Query Options Advanced Configuration Snippet (Safety Valve)", - "name": "default_query_options", - "value": "" - }, - { - "desc": "Port on which Beeswax client requests are served by Impala Daemons.", - "display_name": "Impala Daemon Beeswax Port", - "name": "beeswax_port", - "value": "21000" - }, - { - "desc": "Port where StateStoreSubscriberService is running.", - "display_name": "StateStoreSubscriber Service Port", - "name": "state_store_subscriber_port", - "value": "23000" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Impala Daemon Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "Enables the health check that determines if the Impala Daemon is ready to process queries.", - "display_name": "Impala Daemon Ready Status Health Check", - "name": "impalad_ready_status_check_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "An XML snippet to append to llama-site.xml for Impala Daemons. This configuration only has effect on Impala versions 1.3 or greater.", - "display_name": "Impala Daemon Llama Site Advanced Configuration", - "name": "impala_llama_site_conf_safety_valve", - "value": null - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "Address of the load balancer used for Impala Daemons. Should be specified in host:port format.", - "display_name": "Impala Daemons Load Balancer", - "name": "impalad_load_balancer", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Maximum number of times a request to reserve, expand, or release resources is attempted until the request is cancelled.", - "display_name": "Llama Maximum Request Attempts", - "name": "llama_max_request_attempts", - "value": "5" - }, - { - "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", - "display_name": "Impala Daemon HBase Advanced Configuration Snippet (Safety Valve)", - "name": "impalad_hbase_conf_safety_valve", - "value": null - }, - { - "desc": "The maximum size, in megabytes, per log file for Impala Daemon logs. Typically used by log4j.", - "display_name": "Impala Daemon Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "impalad_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "The polling period of the Impala query monitor in the Cloudera Manager Agent, specified in seconds. If set to zero, query monitoring is disabled.", - "display_name": "Query Monitoring Period", - "name": "query_monitoring_period_seconds", - "value": "1.0" - }, - { - "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", - "display_name": "Impala Daemon Hive Advanced Configuration Snippet (Safety Valve)", - "name": "impala_hive_conf_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Directory where Impala Daemon will place its log files.", - "display_name": "Impala Daemon Log Directory", - "name": "log_dir", - "value": "/var/log/impalad" - }, - { - "desc": "The amount of time at Impala Daemon startup allowed for the Impala Daemon to start accepting new queries for processing.", - "display_name": "Impala Daemon Ready Status Startup Tolerance", - "name": "impalad_ready_status_check_startup_tolerance", - "value": "180" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Enables the health test that verifies the Impala Daemon is connected to the StateStore", - "display_name": "Impala Daemon Connectivity Health Test", - "name": "impalad_connectivity_health_enabled", - "value": "true" - }, - { - "desc": "Port where Impala debug web server runs.", - "display_name": "Impala Daemon HTTP Server Port", - "name": "impalad_webserver_port", - "value": "25000" - }, - { - "desc": "Verbose logging level for the GLog logger. These messages are always logged at 'INFO' log level, so this setting has no effect if Logging Threshold is set to 'WARN' or above. ", - "display_name": "Impala Daemon Verbose Log Level", - "name": "log_verbose_level", - "value": "1" - }, - { - "desc": "The health test thresholds on the resident size of the process.", - "display_name": "Resident Set Size Thresholds", - "name": "process_resident_set_size_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Enables the health test that the Impala Daemon's process state is consistent with the role configuration", - "display_name": "Impala Daemon Process Health Test", - "name": "impalad_scm_health_enabled", - "value": "true" - }, - { - "desc": "The maximum size (in queries) of the Impala Daemon audit event log file before a new one is created.", - "display_name": "Impala Daemon Maximum Audit Log File Size", - "name": "max_audit_event_log_file_size", - "value": "5000" - }, - { - "desc": "Timeout for requests to the Hive Metastore Server from Impala. Consider increasing this if you have tables with a lot of metadata and see timeout errors.", - "display_name": "Impala Daemon Hive Metastore Connection Timeout", - "name": "hive_metastore_timeout", - "value": "3600" - }, - { - "desc": "Memory limit in bytes for Impala Daemon, enforced by the daemon itself. If reached, queries running on the Impala Daemon may be killed. Leave it blank to let Impala pick its own limit. Use a value of -1 B to specify no limit.", - "display_name": "Impala Daemon Memory Limit", - "name": "impalad_memory_limit", - "value": null - }, - { - "desc": "Password for Impala Daemon webserver authentication.", - "display_name": "Impala Daemon Web Server User Password", - "name": "webserver_htpassword_password", - "value": null - }, - { - "desc": "The amount of time to wait for the Impala Daemon to fully start up and connect to the StateStore before enforcing the connectivity check.", - "display_name": "Impala Daemon Connectivity Tolerance at Startup", - "name": "impalad_connectivity_tolerance", - "value": "180" - }, - { - "desc": "Port where Llama notification callback should be started", - "display_name": "Llama Callback Port", - "name": "llama_callback_port", - "value": "28000" - }, - { - "desc": "The location of the debug webserver's SSL certificate file, in .pem format. If empty, webserver SSL support is not enabled.", - "display_name": "SSL Certificate File Location", - "name": "webserver_certificate_file", - "value": null - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "impalad_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystems that contain Impala's scratch directories.", - "display_name": "Impala Scratch Directories Free Space Monitoring Absolute Thresholds", - "name": "impalad_scratch_directories_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "Impala Daemon HDFS Advanced Configuration Snippet (Safety Valve)", - "name": "impala_hdfs_site_conf_safety_valve", - "value": null - }, - { - "desc": "Port on which ImpalaBackendService is exported.", - "display_name": "Impala Daemon Backend Port", - "name": "be_port", - "value": "22000" - }, - { - "desc": "The minimum log level for Impala Daemon logs", - "display_name": "Impala Daemon Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/impala-llama.json b/sahara/plugins/cdh/v5_3_0/resources/impala-llama.json deleted file mode 100644 index 20a43af1..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/impala-llama.json +++ /dev/null @@ -1,338 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Directory where Llama will place its log files.", - "display_name": "Llama Log Directory", - "name": "llama_log_dir", - "value": "/var/log/impala-llama" - }, - { - "desc": "Whether Llama should cache allocated resources on release.", - "display_name": "Enable Resource Caching", - "name": "llama_am_cache_enabled", - "value": "true" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "The group that the Llama processes should run as.", - "display_name": "Llama System Group", - "name": "process_groupname", - "value": "llama" - }, - { - "desc": "Queues Llama ApplicationMaster should connect to at start up.", - "display_name": "Core Queues", - "name": "llama_am_core_queues", - "value": "" - }, - { - "desc": "Maximum amount of time the backed off reservations will be in 'backed off' state. The actual amount time is a random value between the minimum and the maximum.", - "display_name": "Anti-Deadlock Maximum Delay", - "name": "llama_am_gang_anti_deadlock_max_delay_ms", - "value": "30000" - }, - { - "desc": "For advanced use only, a string to be inserted into llama-site.xml for this role only.", - "display_name": "Impala Llama ApplicationMaster Advanced Configuration Snippet (Safety Valve) for llama-site.xml", - "name": "llama_config_valve", - "value": null - }, - { - "desc": "ACL for Impala ApplicationMaster clients. The ACL is a comma-separated list of user and group names. The user and group list is separated by a blank. For e.g. \"alice,bob users,wheel\". A special value of \"*\" means all users are allowed. These take effect only if security is enabled.", - "display_name": "Client ACLs", - "name": "llama_am_server_thrift_client_acl", - "value": "*" - }, - { - "desc": "When computing the overall Impala Llama ApplicationMaster health, consider the host's health.", - "display_name": "Impala Llama ApplicationMaster Host Health Test", - "name": "llama_host_health_enabled", - "value": "true" - }, - { - "desc": "Port on which the Llama ApplicationMaster listens to HTTP requests.", - "display_name": "Llama HTTP Port", - "name": "llama_http_port", - "value": "15001" - }, - { - "desc": "Minimum number of threads used by the Llama ApplicationMaster auxiliary service uses for serving client requests.", - "display_name": "Thrift Server Minimum Threads", - "name": "llama_am_server_thrift_server_min_threads", - "value": "10" - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "Enter an XML string that will be inserted verbatim into the Fair Scheduler allocations file. Overrides the configuration set using the Pools configuration UI. This configuration only has effect on Impala versions 1.3 or greater.", - "display_name": "Fair Scheduler XML Advanced Configuration Snippet (Safety Valve)", - "name": "llama_fair_scheduler_safety_valve", - "value": null - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Client notification retry interval, in milliseconds.", - "display_name": "Client Notification Retry Interval", - "name": "llama_am_server_thrift_client_notifier_retry_interval_ms", - "value": "5000" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Timeout policy for resources being cached.", - "display_name": "Resource Caching Idle Timeout", - "name": "llama_am_cache_eviction_timeout_policy_idle_timeout_ms", - "value": "30000" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Impala Llama ApplicationMaster in Bytes", - "name": "llama_java_heapsize", - "value": "268435456" - }, - { - "desc": "Interval of time without any new allocation that will trigger the Impala ApplicationMaster anti-deadlock logic.", - "display_name": "Anti-Deadlock No Allocation Limit Interval", - "name": "llama_am_gang_anti_deadlock_no_allocation_limit_ms", - "value": "30000" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Maximum number of threads used by the Llama ApplicationMaster auxiliary service uses for serving client requests.", - "display_name": "Thrift Server Maximum Threads", - "name": "llama_am_server_thrift_server_max_threads", - "value": "50" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "Time in milliseconds after which Llama will discard its AM for a queue that has been empty of reservations. Does not apply to queues specified with the Core Queues property.", - "display_name": "Queue Expiration Age", - "name": "llama_am_queue_expire_ms", - "value": "300000" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Impala Llama ApplicationMaster Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "Minimum amount of time the backed off reservations will be in 'backed off' state. The actual amount time is a random value between the minimum and the maximum.", - "display_name": "Anti-Deadlock Minimum Delay", - "name": "llama_am_gang_anti_deadlock_min_delay_ms", - "value": "10000" - }, - { - "desc": "Port on which the Llama ApplicationMaster listens to administrative requests on its administrative Thrift interface.", - "display_name": "Llama Thrift Admin Port", - "name": "llama_am_server_thrift_admin_address", - "value": "15002" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "llama_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "If enabled, the Impala Llama ApplicationMaster binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind Impala Llama ApplicationMaster to Wildcard Address", - "name": "llama_bind_wildcard", - "value": "false" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Impala Llama ApplicationMaster logs. Typically used by log4j.", - "display_name": "Impala Llama ApplicationMaster Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Llama Server", - "name": "llama_java_opts", - "value": "" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Llama ApplicationMaster heartbeat interval, in milliseconds. On each heartbeat the ApplicationMaster submits new reservations to YARN ResourceManager and gets updates from it.", - "display_name": "AM Heartbeat Interval", - "name": "llama_am_server_thrift_client_notifier_heartbeat_ms", - "value": "5000" - }, - { - "desc": "The user that the Llama process should run as.", - "display_name": "Llama System User", - "name": "process_username", - "value": "llama" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Percentage of resources that will be backed off by the Impala ApplicationMaster anti-deadlock logic. Random reservations will be backed off until the percentage of backed off resources reaches this percentage.", - "display_name": "Anti-Deadlock Backoff Percentage", - "name": "llama_am_gang_anti_deadlock_backoff_percent", - "value": "30" - }, - { - "desc": "Maximum number of retries for a client notification. After the maximum number of client notification retries has been reached without success the client is considered lost and all its reservations are released. A successful client notification resets the retries count.", - "display_name": "Maximum Client Notification Retries", - "name": "llama_am_server_thrift_client_notifier_max_retries", - "value": "5" - }, - { - "desc": "Port on which the Llama ApplicationMaster serves its Thrift interface.", - "display_name": "Llama Thrift Port", - "name": "llama_port", - "value": "15000" - }, - { - "desc": "Whether to break resource requests into smaller requests of standard size before the resource cache. The sizes are taken from Yarn settings Container Memory Increment and Container Virtual CPU Cores Increment.", - "display_name": "Enable Resource Cache Normalization", - "name": "llama_am_resource_normalizing_enabled", - "value": "true" - }, - { - "desc": "The maximum number of rolled log files to keep for Impala Llama ApplicationMaster logs. Typically used by log4j.", - "display_name": "Impala Llama ApplicationMaster Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "Socket time, in milliseconds, used Llama ApplicationMaster auxiliary service for all its server and client Thrift connections.", - "display_name": "Thrift Transport Timeout", - "name": "llama_am_server_thrift_transport_timeout_ms", - "value": "60000" - }, - { - "desc": "Enables the health test that the Impala Llama ApplicationMaster's process state is consistent with the role configuration", - "display_name": "Impala Llama ApplicationMaster Process Health Test", - "name": "llama_scm_health_enabled", - "value": "true" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "ACL for Impala ApplicationMaster admins. The ACL is a comma-separated list of user and group names. The user and group list is separated by a blank. For e.g. \"alice,bob users,wheel\". A special value of \"*\" means all users are allowed. These take effect only if security is enabled.", - "display_name": "Administrative Interface ACLs", - "name": "llama_am_server_thrift_admin_acl", - "value": "*" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "The minimum log level for Impala Llama ApplicationMaster logs", - "display_name": "Impala Llama ApplicationMaster Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/impala-service.json b/sahara/plugins/cdh/v5_3_0/resources/impala-service.json deleted file mode 100644 index d1f07fae..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/impala-service.json +++ /dev/null @@ -1,320 +0,0 @@ -[ - { - "desc": "The health test thresholds for assignment locality. Specified as a percentage of total assignments.", - "display_name": "Assignment Locality Ratio Thresholds", - "name": "impala_assignment_locality_thresholds", - "value": "{\"critical\":\"5.0\",\"warning\":\"80.0\"}" - }, - { - "desc": "Name of the HBase service that this Impala service instance depends on", - "display_name": "HBase Service", - "name": "hbase_service", - "value": null - }, - { - "desc": "The user that this Impala's processes should run as (except Llama, which has its own user).", - "display_name": "Impala System User (except Llama)", - "name": "process_username", - "value": "impala" - }, - { - "desc": "JSON representation of all the configurations that the Fair Scheduler can take on across all schedules. Typically edited using the Pools configuration UI. This configuration only has effect on Impala versions 1.3 or greater.", - "display_name": "Fair Scheduler Allocations", - "name": "impala_scheduled_allocations", - "value": "{\"defaultMinSharePreemptionTimeout\":null,\"defaultQueueSchedulingPolicy\":null,\"fairSharePreemptionTimeout\":null,\"queueMaxAppsDefault\":null,\"queuePlacementRules\":null,\"queues\":[{\"aclAdministerApps\":null,\"aclSubmitApps\":null,\"minSharePreemptionTimeout\":null,\"name\":\"root\",\"queues\":[{\"aclAdministerApps\":null,\"aclSubmitApps\":null,\"minSharePreemptionTimeout\":null,\"name\":\"default\",\"queues\":[],\"schedulablePropertiesList\":[{\"impalaMaxMemory\":null,\"impalaMaxQueuedQueries\":null,\"impalaMaxRunningQueries\":null,\"maxResources\":null,\"maxRunningApps\":null,\"minResources\":null,\"scheduleName\":\"default\",\"weight\":null}],\"schedulingPolicy\":null}],\"schedulablePropertiesList\":[{\"impalaMaxMemory\":null,\"impalaMaxQueuedQueries\":null,\"impalaMaxRunningQueries\":null,\"maxResources\":null,\"maxRunningApps\":null,\"minResources\":null,\"scheduleName\":\"default\",\"weight\":null}],\"schedulingPolicy\":null}],\"userMaxAppsDefault\":null,\"users\":[]}" - }, - { - "desc": "Name of the Sentry service that this Impala service instance depends on. If selected, Impala uses this Sentry service to look up authorization privileges. Before enabling Sentry, read the requirements and configuration steps in Setting Up The Sentry Service.", - "display_name": "Sentry Service", - "name": "sentry_service", - "value": null - }, - { - "desc": "\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n
\n\n\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n
\n\nEvent trackers are defined in a JSON object like the following:
\n\n\n{\n \"timeToLive\" : [integer],\n \"fields\" : [\n {\n \"type\" : [string],\n \"name\" : [string]\n }\n ]\n}\n\n\n\nWhere:\n
\n\n\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n
\n\n\nThe following is the list of fields that can be used to compare Impala events:\n
\n\nEvent filters are defined in a JSON object like the following:
\n\n\n{\n \"defaultAction\" : (\"accept\", \"discard\"),\n \"rules\" : [\n {\n \"action\" : (\"accept\", \"discard\"),\n \"fields\" : [\n {\n \"name\" : \"fieldName\",\n \"match\" : \"regex\"\n }\n ]\n }\n ]\n}\n\n\n\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n
\n\n\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n
\n\n\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n
\n\n\nThe following is the list of fields that can be filtered for Impala events:\n
\n\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "If true, attempts to establish a TLS (Transport Layer Security) connection with the ldap server. Only supported in Impala 1.4 or CDH 5.1 or higher. Not required when using an LDAP URI with prefix: ldaps://.", - "display_name": "Enable LDAP TLS", - "name": "enable_ldap_tls", - "value": "false" - }, - { - "desc": "Controls which queries admin users can see in the queries list view", - "display_name": "Admin Users Query List Visibility Settings", - "name": "admin_query_list_settings", - "value": "ALL" - }, - { - "desc": "Enable support for encrypted client-server communication using Secure Socket Layer (SSL) for Impala client services. This is only applicable to non-Kerberos environments.", - "display_name": "Enable SSL for Impala Client Services", - "name": "client_services_ssl_enabled", - "value": "false" - }, - { - "desc": "The minimum number of assignments that must occur during the test time period before the threshold values will be checked. Until this number of assignments have been observed in the test time period the health test will be disabled.", - "display_name": "Assignment Locality Minimum Assignments", - "name": "impala_assignment_locality_minimum", - "value": "10" - }, - { - "desc": "When computing the overall IMPALA health, consider Impala StateStore's health", - "display_name": "Impala StateStore Role Health Test", - "name": "impala_statestore_health_enabled", - "value": "true" - }, - { - "desc": "Name of YARN service to use for resource management integration between Impala and YARN. This service dependency and the existence of a Llama role is required for using said integration.", - "display_name": "YARN Service for Resource Management", - "name": "yarn_service", - "value": null - }, - { - "desc": "Local path to the SSL server certificate file.", - "display_name": "SSL Server Certificate", - "name": "ssl_server_certificate", - "value": null - }, - { - "desc": "Timeout in milliseconds for all HBase RPCs made by Impala. Overrides configuration in HBase service.", - "display_name": "HBase RPC Timeout", - "name": "hbase_rpc_timeout", - "value": "3000" - }, - { - "desc": "Name of the Hive service that this Impala service instance depends on", - "display_name": "Hive Service", - "name": "hive_service", - "value": null - }, - { - "desc": "Enable collection of audit events from the service's roles.", - "display_name": "Enable Collection", - "name": "navigator_audit_enabled", - "value": "true" - }, - { - "desc": "A list specifying the rules to run to determine which Fair Scheduler configuration to use. Typically edited using the Rules configuration UI. This configuration only has effect on Impala versions 1.3 or greater.", - "display_name": "Fair Scheduler Configuration Rules", - "name": "impala_schedule_rules", - "value": "[]" - }, - { - "desc": "If true, LDAP-based authentication for users is enabled.", - "display_name": "Enable LDAP Authentication", - "name": "enable_ldap_auth", - "value": "false" - }, - { - "desc": "When computing the overall IMPALA health, consider Impala Catalog Server's health", - "display_name": "Impala Catalog Server Role Health Test", - "name": "impala_catalogserver_health_enabled", - "value": "true" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "Configures the maximum number of concurrently running queries for admission control when using a single pool. -1 indicates no limit and 0 indicates all incoming requests will be rejected. Ignored when Dynamic Resource Pools for Admission Control is enabled.", - "display_name": "Single Pool Max Running Queries", - "name": "admission_control_single_pool_max_requests", - "value": "200" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/impala-statestore.json b/sahara/plugins/cdh/v5_3_0/resources/impala-statestore.json deleted file mode 100644 index 1f1c2a38..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/impala-statestore.json +++ /dev/null @@ -1,206 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "statestore_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", - "display_name": "Health Check Startup Tolerance", - "name": "statestore_startup_tolerance", - "value": "5" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be added (verbatim) to StateStore command line flags.", - "display_name": "Statestore Command Line Argument Advanced Configuration Snippet (Safety Valve)", - "name": "statestore_cmd_args_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "statestore_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Enables the health test that the Impala StateStore's process state is consistent with the role configuration", - "display_name": "Impala StateStore Process Health Test", - "name": "statestore_scm_health_enabled", - "value": "true" - }, - { - "desc": "Port where StateStoreService is exported.", - "display_name": "StateStore Service Port", - "name": "state_store_port", - "value": "24000" - }, - { - "desc": "When computing the overall Impala StateStore health, consider the host's health.", - "display_name": "Impala StateStore Host Health Test", - "name": "statestore_host_health_enabled", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Username for Statestore webserver authentication.", - "display_name": "Statestore Web Server Username", - "name": "webserver_htpassword_user", - "value": null - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Maximum number of tasks allowed to be pending at the thread manager underlying the State Store Thrift server (0 allows infinitely many pending tasks)", - "display_name": "Maximum StateStore Pending Tasks", - "name": "state_store_pending_task_count_max", - "value": "0" - }, - { - "desc": "Directory where StateStore will place its log files.", - "display_name": "StateStore Log Directory", - "name": "log_dir", - "value": "/var/log/statestore" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Port where StateStore debug web server runs.", - "display_name": "StateStore HTTP Server Port", - "name": "statestore_webserver_port", - "value": "25010" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Impala StateStore logs. Typically used by log4j.", - "display_name": "Impala StateStore Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "Number of worker threads for the thread manager underlying the State Store Thrift server.", - "display_name": "StateStore Worker Threads", - "name": "state_store_num_server_worker_threads", - "value": "4" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Verbose logging level for the GLog logger. These messages are always logged at 'INFO' log level, so this setting has no effect if Logging Threshold is set to 'WARN' or above. ", - "display_name": "StateStore Verbose Log Level", - "name": "log_verbose_level", - "value": "1" - }, - { - "desc": "The health test thresholds on the resident size of the process.", - "display_name": "Resident Set Size Thresholds", - "name": "process_resident_set_size_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Buffer log messages logged at this level or lower (-1 means don't buffer; 0 means buffer INFO only; 1 means buffer WARNING only, ...)", - "display_name": "StateStore Log Buffer Level", - "name": "logbuflevel", - "value": "0" - }, - { - "desc": "The location of the debug webserver's SSL certificate file, in .pem format. If empty, webserver SSL support is not enabled.", - "display_name": "SSL Certificate File Location", - "name": "webserver_certificate_file", - "value": null - }, - { - "desc": "Enable/Disable StateStore web server. This web server contains useful information about StateStore daemon.", - "display_name": "Enable StateStore Web Server", - "name": "statestore_enable_webserver", - "value": "true" - }, - { - "desc": "Password for Statestore webserver authentication.", - "display_name": "Statestore Web Server User Password", - "name": "webserver_htpassword_password", - "value": null - }, - { - "desc": "The minimum log level for Impala StateStore logs", - "display_name": "Impala StateStore Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "statestore_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/ks_indexer-hbase_indexer.json b/sahara/plugins/cdh/v5_3_0/resources/ks_indexer-hbase_indexer.json deleted file mode 100644 index 4abb98d2..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/ks_indexer-hbase_indexer.json +++ /dev/null @@ -1,206 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "hbase_indexer_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for Lily HBase Indexer logs. Typically used by log4j.", - "display_name": "Lily HBase Indexer Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The group that the HBase Indexer process should run as.", - "display_name": "System Group", - "name": "hbase_indexer_process_groupname", - "value": "hbase" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Lily HBase Indexer", - "name": "hbase_indexer_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "For advanced use only, a string to be inserted into hbase-indexer-site.xml for this role only.", - "display_name": "Lily HBase Indexer Advanced Configuration Snippet (Safety Valve) for hbase-indexer-site.xml", - "name": "hbase_indexer_config_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Lily HBase Indexer Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Lily HBase Indexer logs. Typically used by log4j.", - "display_name": "Lily HBase Indexer Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Directory where HBase Indexer will place its log files.", - "display_name": "HBase Indexer Log Directory", - "name": "hbase_indexer_log_dir", - "value": "/var/log/hbase-solr" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "When computing the overall Lily HBase Indexer health, consider the host's health.", - "display_name": "Lily HBase Indexer Host Health Test", - "name": "hbase_indexer_host_health_enabled", - "value": "true" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Enables the health test that the Lily HBase Indexer's process state is consistent with the role configuration", - "display_name": "Lily HBase Indexer Process Health Test", - "name": "hbase_indexer_scm_health_enabled", - "value": "true" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The minimum log level for Lily HBase Indexer logs", - "display_name": "Lily HBase Indexer Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "The user that the HBase Indexer process should run as.", - "display_name": "System User", - "name": "hbase_indexer_process_username", - "value": "hbase" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Lily HBase Indexer in Bytes", - "name": "hbase_indexer_java_heapsize", - "value": "1073741824" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/ks_indexer-service.json b/sahara/plugins/cdh/v5_3_0/resources/ks_indexer-service.json deleted file mode 100644 index ff96caa5..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/ks_indexer-service.json +++ /dev/null @@ -1,68 +0,0 @@ -[ - { - "desc": "Name of the Solr service that this Key-Value Store Indexer service instance depends on", - "display_name": "Solr Service", - "name": "solr_service", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Name of the HBase service that this Key-Value Store Indexer service instance depends on", - "display_name": "HBase Service", - "name": "hbase_service", - "value": null - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Key-Value Indexer Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "ks_indexer_env_safety_valve", - "value": null - }, - { - "desc": "Text that goes verbatim into custom-mimetypes.xml file used by HBase Indexers.", - "display_name": "Custom Mime-types File", - "name": "custom_mimetypes_file", - "value": "\n\n\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/mapred-site.xml b/sahara/plugins/cdh/v5_3_0/resources/mapred-site.xml deleted file mode 100644 index 5228ed17..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/mapred-site.xml +++ /dev/null @@ -1,157 +0,0 @@ - - - -The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Oozie Server", - "name": "oozie_java_opts", - "value": "" - }, - { - "desc": "The admin port Oozie server runs.", - "display_name": "Oozie Admin Port", - "name": "oozie_admin_port", - "value": "11001" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing the moving average of extra time the pause monitor spent paused.", - "display_name": "Pause Duration Monitoring Period", - "name": "oozie_server_pause_duration_window", - "value": "5" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "If true, enables the Oozie Server web console. ExtJS 2.2 zip archive must be extracted to /var/lib/oozie on the same host as the Oozie Server.", - "display_name": "Enable Oozie Server Web Console", - "name": "oozie_web_console", - "value": "false" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "oozie_server_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Username for connecting to the database used by Oozie Server. Does not apply if you are using Derby as the database type.", - "display_name": "Oozie Server Database User", - "name": "oozie_database_user", - "value": "sa" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "oozie_server_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds of the weighted average size of the Oozie Server callable queue over a recent period. See also Oozie Server Callable Queue Monitoring Period.", - "display_name": "Oozie Server Callable Queue Monitoring Threshold", - "name": "oozie_server_callable_queue_threshold", - "value": "{\"critical\":\"95.0\",\"warning\":\"80.0\"}" - }, - { - "desc": "Directory where the Oozie Server will place its data. Only applicable when using Derby as the database type.", - "display_name": "Oozie Server Data Directory", - "name": "oozie_data_dir", - "value": "/var/lib/oozie/data" - }, - { - "desc": "Password for the keystore.", - "display_name": "Oozie SSL Keystore Password", - "name": "oozie_https_keystore_password", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "oozie_server_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Oozie Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Location of the keystore file on the local file system.", - "display_name": "Oozie SSL Keystore File", - "name": "oozie_https_keystore_file", - "value": "/var/lib/oozie/.keystore" - }, - { - "desc": "The from address to be used for mailing all emails for Oozie email action", - "display_name": "Oozie Email Action From Address", - "name": "oozie_email_from_address", - "value": "oozie@localhost" - }, - { - "desc": "Comma-separated list of SchemaService workflow extension schemas for additional action types.", - "display_name": "Oozie SchemaService Workflow Extension Schemas", - "name": "oozie_workflow_extension_schemas", - "value": "hive-action-0.2.xsd,sqoop-action-0.2.xsd,email-action-0.1.xsd,distcp-action-0.1.xsd,shell-action-0.1.xsd,ssh-action-0.1.xsd,distcp-action-0.2.xsd,hive-action-0.3.xsd,hive-action-0.4.xsd,hive-action-0.5.xsd,oozie-sla-0.1.xsd,oozie-sla-0.2.xsd,sqoop-action-0.3.xsd,sqoop-action-0.4.xsd,shell-action-0.2.xsd,shell-action-0.3.xsd,ssh-action-0.2.xsd,hive2-action-0.1.xsd" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The SMTP server port to use for Oozie email action", - "display_name": "Oozie Email Action SMTP Port", - "name": "oozie_email_smtp_prt", - "value": "25" - }, - { - "desc": "Port of Oozie Server", - "display_name": "Oozie HTTP Port", - "name": "oozie_http_port", - "value": "11000" - }, - { - "desc": "The maximum number of rolled log files to keep for Oozie Server logs. Typically used by log4j.", - "display_name": "Oozie Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "720" - }, - { - "desc": "Workflow Status metrics collection interval.", - "display_name": "Workflow Status Metrics Collection Interval", - "name": "oozie_job_metric_collection_interval", - "value": "1" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Name of the database used by Oozie Server.", - "display_name": "Oozie Server Database Name", - "name": "oozie_database_name", - "value": "oozie" - }, - { - "desc": "Directory where Oozie Server will place its log files.", - "display_name": "Oozie Server Log Directory", - "name": "oozie_log_dir", - "value": "/var/log/oozie" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Oozie Server in Bytes", - "name": "oozie_java_heapsize", - "value": "1073741824" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Whether to use the Codehale based metrics for instrumentation. Enabling this disables the 'instrumentation' REST end-point and enables the 'metrics' REST end-point (<hostname:port>/v2/admin/metrics).", - "display_name": "Enable The Metrics Instrumentation Service", - "name": "oozie_use_metric_instrumentation", - "value": "true" - }, - { - "desc": "Comma-separated list of ActionService executor extension classes. Only action types with associated executors can be used in workflows.", - "display_name": "Oozie ActionService Executor Extension Classes", - "name": "oozie_executor_extension_classes", - "value": "org.apache.oozie.action.hadoop.HiveActionExecutor,org.apache.oozie.action.hadoop.SqoopActionExecutor,org.apache.oozie.action.email.EmailActionExecutor,org.apache.oozie.action.hadoop.ShellActionExecutor,org.apache.oozie.action.hadoop.DistcpActionExecutor,org.apache.oozie.action.hadoop.Hive2ActionExecutor" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Type of the database used by Oozie Server.", - "display_name": "Oozie Server Database Type", - "name": "oozie_database_type", - "value": "derby" - }, - { - "desc": "Maximum callable queue size", - "display_name": "Maximum Callable Queue Size", - "name": "oozie_service_callablequeueservice_queue_size", - "value": "10000" - }, - { - "desc": "Port of the Oozie Server while using SSL.", - "display_name": "Oozie HTTPS Port", - "name": "oozie_https_port", - "value": "11443" - }, - { - "desc": "For advanced use only, a string to be inserted into oozie-site.xml for this role only.", - "display_name": "Oozie Server Advanced Configuration Snippet (Safety Valve) for oozie-site.xml", - "name": "oozie_config_safety_valve", - "value": null - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "The SMTP server host to use for Oozie email action", - "display_name": "Oozie Email Action SMTP Host", - "name": "oozie_email_smtp_host", - "value": "localhost" - }, - { - "desc": "The minimum log level for Oozie Server logs", - "display_name": "Oozie Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Hostname of the database used by Oozie Server. If the port is non-default for your database type, use host:port notation. Does not apply if you are using Derby as the database type.", - "display_name": "Oozie Server Database Host", - "name": "oozie_database_host", - "value": "localhost" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/oozie-service.json b/sahara/plugins/cdh/v5_3_0/resources/oozie-service.json deleted file mode 100644 index 9219003b..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/oozie-service.json +++ /dev/null @@ -1,128 +0,0 @@ -[ - { - "desc": "Namespace used by this Oozie service in ZooKeeper when High Availability is enabled.", - "display_name": "ZooKeeper Namespace", - "name": "oozie_zookeeper_namespace", - "value": "oozie" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "List of event listeners used by the Oozie service. Listeners needed for JMS or SLA integration are automatically emitted if they are enabled.", - "display_name": "Oozie Event Listeners", - "name": "oozie_event_listeners", - "value": "" - }, - { - "desc": "Enable SSL for Oozie. Note:This is supported only from CDH 4.3 onwards.", - "display_name": "Use SSL", - "name": "oozie_use_ssl", - "value": "false" - }, - { - "desc": "URL of the JMS Broker used by the Oozie service in JMS integration is enabled.", - "display_name": "JMS Broker", - "name": "oozie_jms_broker", - "value": "tcp://localhost:61616" - }, - { - "desc": "Use ACLs on Znode while a secure ZooKeeper is used for Oozie High Availability. Note: This config is not emitted if ZooKeeper is not secure.", - "display_name": "Use ACLs on Znode", - "name": "oozie_zk_secure", - "value": "true" - }, - { - "desc": "Whether to configure Oozie properties needed for JMS integration", - "display_name": "Enable JMS Integration", - "name": "oozie_use_jms", - "value": "false" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Oozie Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "oozie_env_safety_valve", - "value": null - }, - { - "desc": "Name of the ZooKeeper service that this Oozie service instance depends on", - "display_name": "ZooKeeper Service", - "name": "zookeeper_service", - "value": null - }, - { - "desc": "Name of the Hive service that this Oozie service instance depends on. This is used to configure Oozie HCat integration.", - "display_name": "Hive Service", - "name": "hive_service", - "value": null - }, - { - "desc": "A list of credential class mappings for CredentialsProvider.", - "display_name": "Oozie Credential Classes", - "name": "oozie_credential_classes", - "value": "hcat=org.apache.oozie.action.hadoop.HCatCredentials,hbase=org.apache.oozie.action.hadoop.HbaseCredentials,hive2=org.apache.oozie.action.hadoop.Hive2Credentials" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Address of the load balancer used if Oozie HA is enabled. Should be specified in host:port format.", - "display_name": "Oozie Load Balancer", - "name": "oozie_load_balancer", - "value": null - }, - { - "desc": "The group that this service's processes should run as.", - "display_name": "System Group", - "name": "process_groupname", - "value": "oozie" - }, - { - "desc": "Whether to configure Oozie properties needed for SLA integration", - "display_name": "Enable SLA Integration", - "name": "oozie_use_sla", - "value": "false" - }, - { - "desc": "Coordinator Job Lookup trigger command is scheduled at this interval (in seconds).", - "display_name": "Coordinator Job Lookup Interval", - "name": "oozie_service_coord_lookup_interval", - "value": "300" - }, - { - "desc": "The user that this service's processes should run as.", - "display_name": "System User", - "name": "process_username", - "value": "oozie" - }, - { - "desc": "The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "The health test thresholds of the overall Oozie Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Oozie Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Oozie Servers falls below the critical threshold.", - "display_name": "Healthy Oozie Server Monitoring Thresholds", - "name": "oozie_servers_healthy_thresholds", - "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" - }, - { - "desc": "Service to run MapReduce jobs against", - "display_name": "MapReduce Service", - "name": "mapreduce_yarn_service", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/priority-one-confs.json b/sahara/plugins/cdh/v5_3_0/resources/priority-one-confs.json deleted file mode 100644 index f7c77953..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/priority-one-confs.json +++ /dev/null @@ -1,45 +0,0 @@ -[ - "dfs_block_size", - "dfs_umaskmode", - "dfs_webhdfs_enabled", - "dfs_permissions", - "dfs_replication", - "io_compression_codecs", - "io_sort_mb", - "dfs_datanode_du_reserved", - "dfs_datanode_failed_volumes_tolerated", - "dfs_name_dir_restore", - "fs_trash_interval", - "dfs_safemode_min_datanodes", - "dfs_safemode_extension", - "dfs_access_time_precision", - "yarn_acl_enable", - "yarn_admin_acl", - "yarn_log_aggregation_enable", - "yarn_log_aggregation_retain_seconds", - "mapreduce_jobhistory_max_age_ms", - "mapreduce_jobhistory_cleaner_interval", - "mapreduce_reduce_memory_mb", - "mapreduce_reduce_java_opts", - "mapreduce_map_memory_mb", - "mapreduce_map_java_opts", - "yarn_nodemanager_container_manager_thread_count", - "yarn_nodemanager_delete_thread_count", - "yarn_nodemanager_heartbeat_interval_ms", - "yarn_nodemanager_localizer_cache_cleanup_interval_ms", - "yarn_nodemanager_localizer_client_thread_count", - "yarn_nodemanager_localizer_cache_target_size_mb", - "yarn_nodemanager_localizer_fetch_thread_count", - "yarn_nodemanager_log_retain_seconds", - "yarn_nodemanager_resource_memory_mb", - "yarn_resourcemanager_client_thread_count", - "yarn_resourcemanager_scheduler_client_thread_count", - "yarn_resourcemanager_admin_client_thread_count", - "yarn_resourcemanager_amliveliness_monitor_interval_ms", - "yarn_am_liveness_monitor_expiry_interval_ms", - "yarn_resourcemanager_am_max_retries", - "yarn_scheduler_minimum_allocation_mb", - "yarn_scheduler_maximum_allocation_mb", - "yarn_app_mapreduce_am_command_opts", - "yarn_app_mapreduce_am_resource_mb" -] diff --git a/sahara/plugins/cdh/v5_3_0/resources/sentry-impala-client-safety.xml b/sahara/plugins/cdh/v5_3_0/resources/sentry-impala-client-safety.xml deleted file mode 100644 index e986433a..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/sentry-impala-client-safety.xml +++ /dev/null @@ -1,16 +0,0 @@ -The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Sentry Server", - "name": "sentry_server_java_opts", - "value": "" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Directory where Sentry Server will place its log files.", - "display_name": "Sentry Server Log Directory", - "name": "sentry_server_log_dir", - "value": "/var/log/sentry" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Sentry Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Sentry Server in Bytes", - "name": "sentry_server_java_heapsize", - "value": "1073741824" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "RPC port number of Sentry Server.", - "display_name": "Sentry Server RPC Port", - "name": "sentry_service_server_rpc_port", - "value": "8038" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "sentry_server_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Sentry Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "sentry_env_safety_valve", - "value": null - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The minimum log level for Sentry Server logs", - "display_name": "Sentry Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Enables the health test that the Sentry Server's process state is consistent with the role configuration", - "display_name": "Sentry Server Process Health Test", - "name": "sentry_server_scm_health_enabled", - "value": "true" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/sentry-service.json b/sahara/plugins/cdh/v5_3_0/resources/sentry-service.json deleted file mode 100644 index 58ede5d0..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/sentry-service.json +++ /dev/null @@ -1,140 +0,0 @@ -[ - { - "desc": "Password for Sentry Server database.", - "display_name": "Sentry Server Database Password", - "name": "sentry_server_database_password", - "value": "" - }, - { - "desc": "\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n
\n\n\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n
\n\nEvent trackers are defined in a JSON object like the following:
\n\n\n{\n \"timeToLive\" : [integer],\n \"fields\" : [\n {\n \"type\" : [string],\n \"name\" : [string]\n }\n ]\n}\n\n\n\nWhere:\n
\n\n\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n
\n\n\nThe following is the list of fields that can be used to compare Sentry events:\n
\n\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "When computing the overall SENTRY health, consider Sentry Server's health", - "display_name": "Sentry Server Role Health Test", - "name": "sentry_sentry_server_health_enabled", - "value": "true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Port number of Sentry Server database.", - "display_name": "Sentry Server Database Port", - "name": "sentry_server_database_port", - "value": "3306" - }, - { - "desc": "If an end user is in one of these admin groups, that user has administrative privileges on the Sentry Server.", - "display_name": "Admin Groups", - "name": "sentry_service_admin_group", - "value": "hive,impala,hue" - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for navigator.client.properties.", - "display_name": "Sentry Client Advanced Configuration Snippet (Safety Valve) for navigator.client.properties", - "name": "navigator_client_config_safety_valve", - "value": null - }, - { - "desc": "Enable collection of audit events from the service's roles.", - "display_name": "Enable Collection", - "name": "navigator_audit_enabled", - "value": "true" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/solr-gateway.json b/sahara/plugins/cdh/v5_3_0/resources/solr-gateway.json deleted file mode 100644 index 54cbc7ea..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/solr-gateway.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The directory where the client configs will be deployed", - "display_name": "Deploy Directory", - "name": "client_config_root_dir", - "value": "/etc/solr" - }, - { - "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", - "display_name": "Alternatives Priority", - "name": "client_config_priority", - "value": "90" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/solr-service.json b/sahara/plugins/cdh/v5_3_0/resources/solr-service.json deleted file mode 100644 index ec66d5b7..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/solr-service.json +++ /dev/null @@ -1,110 +0,0 @@ -[ - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The user that this service's processes should run as.", - "display_name": "System User", - "name": "process_username", - "value": "solr" - }, - { - "desc": "The group that this service's processes should run as.", - "display_name": "System Group", - "name": "process_groupname", - "value": "solr" - }, - { - "desc": "The class to use in Sentry authorization for user to group mapping. Sentry authorization may be configured to use either Hadoop groups or local groups defined in the policy file. When configured with Hadoop groups, Sentry will ask the HDFS Namenode for group mapping for a given user to determine authorization access.", - "display_name": "Sentry User to Group Mapping Class", - "name": "solr_sentry_provider", - "value": "org.apache.sentry.provider.file.HadoopGroupResourceAuthorizationProvider" - }, - { - "desc": "Name of the ZooKeeper service that this Search service instance depends on", - "display_name": "ZooKeeper Service", - "name": "zookeeper_service", - "value": null - }, - { - "desc": "Choose the authentication mechanism used by Solr.", - "display_name": "Solr Secure Authentication", - "name": "solr_security_authentication", - "value": "simple" - }, - { - "desc": "ZooKeeper znode used to store information about this Solr service.", - "display_name": "ZooKeeper Znode", - "name": "zookeeper_znode", - "value": "/solr" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Solr Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "solr_env_safety_valve", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "HDFS directory used for storage by this Solr service.", - "display_name": "HDFS Data Directory", - "name": "hdfs_data_dir", - "value": "/solr" - }, - { - "desc": "If Solr does not respond on its web URL within this time interval, the Catalina process is killed.", - "display_name": "Solrd Watchdog Timeout", - "name": "solrd_watchdog_timeout", - "value": "30" - }, - { - "desc": "Use Sentry to enable role-based, fine-grained authorization. Sentry is supported only on Search 1.1 or later and CDH 5 or later deployments and requires authentication to be turned on for Solr..", - "display_name": "Enable Sentry Authorization", - "name": "solr_sentry_enabled", - "value": "false" - }, - { - "desc": "HDFS path to the global policy file for Sentry authorization. This should be a relative path (and not a full HDFS URL). The global policy file must be in Sentry policy file format.", - "display_name": "Sentry Global Policy File", - "name": "sentry_solr_provider_resource", - "value": "/user/solr/sentry/sentry-provider.ini" - }, - { - "desc": "For advanced use only, a string to be inserted into sentry-site.xml. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Solr Service Advanced Configuration Snippet (Safety Valve) for sentry-site.xml", - "name": "solr_sentry_safety_valve", - "value": null - }, - { - "desc": "The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "Name of the HDFS service that this Search service instance depends on", - "display_name": "HDFS Service", - "name": "hdfs_service", - "value": null - }, - { - "desc": "The health test thresholds of the overall Solr Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Solr Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Solr Servers falls below the critical threshold.", - "display_name": "Healthy Solr Server Monitoring Thresholds", - "name": "solr_solr_servers_healthy_thresholds", - "value": "{\"critical\":\"90.0\",\"warning\":\"95.0\"}" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/solr-solr_server.json b/sahara/plugins/cdh/v5_3_0/resources/solr-solr_server.json deleted file mode 100644 index f7b1686a..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/solr-solr_server.json +++ /dev/null @@ -1,272 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Enable caching of HDFS blocks in Solr. There is one block cache per Solr collection. configured to use off-heap memory, Maximum Off-Heap Memory must be set high enough to account for all block caches.", - "display_name": "HDFS Block Cache", - "name": "solr_hdfs_blockcache_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Number of blocks per cache slab. The size of the cache is 8 KB (the block size) times the number of blocks per slab times the number of slabs.", - "display_name": "HDFS Block Cache Blocks per Slab", - "name": "solr_hdfs_blockcache_blocksperbank", - "value": "16384" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "solr_server_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Solr Server in Bytes", - "name": "solr_java_heapsize", - "value": "1073741824" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for Solr Server logs. Typically used by log4j.", - "display_name": "Solr Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Enables the health test that the Solr Server's process state is consistent with the role configuration", - "display_name": "Solr Server Process Health Test", - "name": "solr_server_scm_health_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds on the duration of the Solr Server API request.", - "display_name": "Solr Server API Liveness Request Duration", - "name": "solr_core_status_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "When computing the overall Solr Server health, consider the host's health.", - "display_name": "Solr Server Host Health Test", - "name": "solr_server_host_health_enabled", - "value": "true" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Solr Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather status of Solr Cores from the Solr Server with a simple API request.", - "display_name": "Solr Server API Liveness", - "name": "solr_core_status_collection_health_enabled", - "value": "true" - }, - { - "desc": "Use off-heap memory when caching HDFS blocks in Solr.", - "display_name": "HDFS Block Cache Off-Heap Memory", - "name": "solr_hdfs_blockcache_direct_memory_allocation", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "solr_server_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Solr Server logs. Typically used by log4j.", - "display_name": "Solr Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "Admin port of the Solr Server.", - "display_name": "Solr Admin Port", - "name": "solr_admin_port", - "value": "8984" - }, - { - "desc": "Directory on local file system where Solr Server keeps the configurations for collections.", - "display_name": "Solr Data Directory", - "name": "solr_data_dir", - "value": "/var/lib/solr" - }, - { - "desc": "Maximum amount of off-heap memory in bytes that may be allocated by the Java process. Passed to Java -XX:MaxDirectMemorySize. If unset, defaults to the size of the heap.", - "display_name": "Java Direct Memory Size of Solr Server in Bytes", - "name": "solr_java_direct_memory_size", - "value": "1073741824" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "solr_server_gc_duration_window", - "value": "5" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "solr_server_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "Number of slabs per block cache. The size of the cache is 8 KB (the block size) times the number of blocks per slab times the number of slabs.", - "display_name": "HDFS Block Cache Number of Slabs", - "name": "solr_hdfs_blockcache_slab_count", - "value": "1" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "Directory where Solr Server will place its log files.", - "display_name": "Solr Server Log Directory", - "name": "solr_log_dir", - "value": "/var/log/solr" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "solr_server_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Solr Server", - "name": "solr_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "The minimum log level for Solr Server logs", - "display_name": "Solr Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "HTTP port of Solr Server.", - "display_name": "Solr HTTP Port", - "name": "solr_http_port", - "value": "8983" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/spark-gateway.json b/sahara/plugins/cdh/v5_3_0/resources/spark-gateway.json deleted file mode 100644 index 801ba204..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/spark-gateway.json +++ /dev/null @@ -1,44 +0,0 @@ -[ - { - "desc": "For advanced use only, a string to be inserted into the client configuration for spark-conf/log4j.properties.", - "display_name": "Spark Client Advanced Configuration Snippet (Safety Valve) for spark-conf/log4j.properties", - "name": "spark-conf/log4j.properties_client_config_safety_valve", - "value": null - }, - { - "desc": "The directory where the client configs will be deployed", - "display_name": "Deploy Directory", - "name": "client_config_root_dir", - "value": "/etc/spark" - }, - { - "desc": "Write Spark application history logs to HDFS.", - "display_name": "Enable History", - "name": "spark_history_enabled", - "value": "true" - }, - { - "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", - "display_name": "Alternatives Priority", - "name": "client_config_priority", - "value": "51" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for spark-conf/spark-defaults.conf.", - "display_name": "Spark Client Advanced Configuration Snippet (Safety Valve) for spark-conf/spark-defaults.conf", - "name": "spark-conf/spark-defaults.conf_client_config_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for spark-conf/spark-env.sh.", - "display_name": "Spark Client Advanced Configuration Snippet (Safety Valve) for spark-conf/spark-env.sh", - "name": "spark-conf/spark-env.sh_client_config_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/spark-service.json b/sahara/plugins/cdh/v5_3_0/resources/spark-service.json deleted file mode 100644 index 526b6201..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/spark-service.json +++ /dev/null @@ -1,62 +0,0 @@ -[ - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The location of the Spark jar in HDFS", - "display_name": "Spark Jar Location (HDFS)", - "name": "spark_jar_hdfs_path", - "value": "/user/spark/share/lib/spark-assembly.jar" - }, - { - "desc": "The group that this service's processes should run as.", - "display_name": "System Group", - "name": "process_groupname", - "value": "spark" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Spark Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "SPARK_ON_YARN_service_env_safety_valve", - "value": null - }, - { - "desc": "The location of Spark application history logs in HDFS. Changing this value will not move existing logs to the new location.", - "display_name": "Spark History Location (HDFS)", - "name": "spark_history_log_dir", - "value": "/user/spark/applicationHistory" - }, - { - "desc": "Name of the YARN (MR2 Included) service that this Spark service instance depends on", - "display_name": "YARN (MR2 Included) Service", - "name": "yarn_service", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The user that this service's processes should run as.", - "display_name": "System User", - "name": "process_username", - "value": "spark" - }, - { - "desc": "The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/spark-spark_yarn_history_server.json b/sahara/plugins/cdh/v5_3_0/resources/spark-spark_yarn_history_server.json deleted file mode 100644 index f757a795..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/spark-spark_yarn_history_server.json +++ /dev/null @@ -1,152 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "History Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "SPARK_YARN_HISTORY_SERVER_role_env_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into config/spark-env.sh for this role only.", - "display_name": "History Server Advanced Configuration Snippet (Safety Valve) for config/spark-env.sh", - "name": "config/spark-env.sh_role_safety_valve", - "value": null - }, - { - "desc": "The port of the history server WebUI", - "display_name": "History Server WebUI Port", - "name": "history_server_web_port", - "value": "18088" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for History Server logs. Typically used by log4j.", - "display_name": "History Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "spark_yarn_history_server_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The log directory for log files of the role History Server.", - "display_name": "History Server Log Directory", - "name": "log_dir", - "value": "/var/log/spark" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "History Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "Maximum size for the Java process heap memory. Passed to Java -Xmx. Measured in bytes.", - "display_name": "Java Heap Size of History Server in Bytes", - "name": "history_server_max_heapsize", - "value": "268435456" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "When computing the overall History Server health, consider the host's health.", - "display_name": "History Server Host Health Test", - "name": "spark_yarn_history_server_host_health_enabled", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The maximum size, in megabytes, per log file for History Server logs. Typically used by log4j.", - "display_name": "History Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Enables the health test that the History Server's process state is consistent with the role configuration", - "display_name": "History Server Process Health Test", - "name": "spark_yarn_history_server_scm_health_enabled", - "value": "true" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "The minimum log level for History Server logs", - "display_name": "History Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/sqoop-service.json b/sahara/plugins/cdh/v5_3_0/resources/sqoop-service.json deleted file mode 100644 index 8076dc2b..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/sqoop-service.json +++ /dev/null @@ -1,56 +0,0 @@ -[ - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "When computing the overall SQOOP health, consider Sqoop 2 Server's health", - "display_name": "Sqoop 2 Server Role Health Test", - "name": "sqoop_sqoop_server_health_enabled", - "value": "true" - }, - { - "desc": "The group that this service's processes should run as.", - "display_name": "System Group", - "name": "process_groupname", - "value": "sqoop" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Sqoop 2 Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "sqoop_env_safety_valve", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The user that this service's processes should run as.", - "display_name": "System User", - "name": "process_username", - "value": "sqoop2" - }, - { - "desc": "The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "MapReduce jobs are run against this service.", - "display_name": "MapReduce Service", - "name": "mapreduce_yarn_service", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/sqoop-sqoop_server.json b/sahara/plugins/cdh/v5_3_0/resources/sqoop-sqoop_server.json deleted file mode 100644 index bf391413..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/sqoop-sqoop_server.json +++ /dev/null @@ -1,218 +0,0 @@ -[ - { - "desc": "Directory where Sqoop 2 Server will place its log files.", - "display_name": "Sqoop 2 Server Log Directory", - "name": "sqoop_log_dir", - "value": "/var/log/sqoop2" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Admin port of Sqoop 2 Server. (Note: This configuration only applies from CDH 4.3 onwards.)", - "display_name": "Sqoop 2 Admin Port", - "name": "sqoop_admin_port", - "value": "8005" - }, - { - "desc": "Maximum number of clients allowed to connect to the Sqoop 2 Server.", - "display_name": "Maximum Client Connections", - "name": "max_client_connections", - "value": "10" - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "For advanced use only, a string to be inserted into sqoop.properties for this role only.", - "display_name": "Sqoop 2 Server Advanced Configuration Snippet (Safety Valve) for sqoop.properties", - "name": "sqoop_config_safety_valve", - "value": null - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for Sqoop 2 Server logs. Typically used by log4j.", - "display_name": "Sqoop 2 Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "HTTP port of Sqoop 2 Server. (Note: This configuration only applies from CDH 4.3 onwards.)", - "display_name": "Sqoop 2 HTTP Port", - "name": "sqoop_http_port", - "value": "12000" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "sqoop_server_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Sqoop 2 Server in Bytes", - "name": "sqoop_java_heapsize", - "value": "1073741824" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Sqoop 2 Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Sqoop 2 Server logs. Typically used by log4j.", - "display_name": "Sqoop 2 Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Directory where the Sqoop 2 Server will place its metastore data.", - "display_name": "Sqoop 2 Server Metastore Directory", - "name": "sqoop_metastore_data_dir", - "value": "/var/lib/sqoop2" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Sqoop 2 Server", - "name": "sqoop_java_opts", - "value": "" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "When computing the overall Sqoop 2 Server health, consider the host's health.", - "display_name": "Sqoop 2 Server Host Health Test", - "name": "sqoop_server_host_health_enabled", - "value": "true" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Enables the health test that the Sqoop 2 Server's process state is consistent with the role configuration", - "display_name": "Sqoop 2 Server Process Health Test", - "name": "sqoop_server_scm_health_enabled", - "value": "true" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The minimum log level for Sqoop 2 Server logs", - "display_name": "Sqoop 2 Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/yarn-gateway.json b/sahara/plugins/cdh/v5_3_0/resources/yarn-gateway.json deleted file mode 100644 index a7e4cae8..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/yarn-gateway.json +++ /dev/null @@ -1,326 +0,0 @@ -[ - { - "desc": "For MapReduce job outputs that are compressed, specify the compression codec to use. Will be part of generated client configuration.", - "display_name": "Compression Codec of MapReduce Job Output", - "name": "mapred_output_compression_codec", - "value": "org.apache.hadoop.io.compress.DefaultCodec" - }, - { - "desc": "The default number of parallel transfers run by reduce during the copy (shuffle) phase. This number should be between sqrt(nodes*number_of_map_slots_per_node) and nodes*number_of_map_slots_per_node/2. Will be part of generated client configuration.", - "display_name": "Default Number of Parallel Transfers During Shuffle", - "name": "mapred_reduce_parallel_copies", - "value": "10" - }, - { - "desc": "Whether Map tasks should attempt to use the optimized native implementation of the map-side output collector. This can improve performance of many jobs which are shuffle-intensive, though is experimental in CDH 5.2.", - "display_name": "Enable Optimized Map-side Output Collector", - "name": "mapreduce_enable_native_map_output_collector", - "value": "false" - }, - { - "desc": "The directory where the client configs will be deployed", - "display_name": "Deploy Directory", - "name": "client_config_root_dir", - "value": "/etc/hadoop" - }, - { - "desc": "The number of streams to merge at the same time while sorting files. That is, the number of sort heads to use during the merge sort on the reducer side. This determines the number of open file handles. Merging more files in parallel reduces merge sort iterations and improves run time by eliminating disk I/O. Note that merging more files in parallel uses more memory. If 'io.sort.factor' is set too high or the maximum JVM heap is set too low, excessive garbage collection will occur. The Hadoop default is 10, but Cloudera recommends a higher value. Will be part of generated client configuration.", - "display_name": "I/O Sort Factor", - "name": "io_sort_factor", - "value": "64" - }, - { - "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", - "display_name": "Alternatives Priority", - "name": "client_config_priority", - "value": "92" - }, - { - "desc": "Location to store the job history files of running jobs. This is a path on the host where the JobTracker is running.", - "display_name": "Running Job History Location", - "name": "hadoop_job_history_dir", - "value": "/var/log/hadoop-mapreduce/history" - }, - { - "desc": "The number of virtual CPU cores allocated for each map task of a job. This parameter has no effect prior to CDH 4.4.", - "display_name": "Map Task CPU Virtual Cores", - "name": "mapreduce_map_cpu_vcores", - "value": "1" - }, - { - "desc": "If enabled, multiple instances of some reduce tasks may be executed in parallel.", - "display_name": "Reduce Tasks Speculative Execution", - "name": "mapred_reduce_tasks_speculative_execution", - "value": "false" - }, - { - "desc": "The application framework to run jobs with. If not set, jobs will be run with the local job runner.", - "display_name": "Application Framework", - "name": "mapreduce_framework_name", - "value": "yarn" - }, - { - "desc": "Base sleep time between failover attempts. Used only if RM HA is enabled.", - "display_name": "Client Failover Sleep Base Time", - "name": "client_failover_sleep_base", - "value": "100" - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for yarn-site.xml.", - "display_name": "YARN Client Advanced Configuration Snippet (Safety Valve) for yarn-site.xml", - "name": "yarn_client_config_safety_valve", - "value": null - }, - { - "desc": "Size of buffer for read and write operations of SequenceFiles.", - "display_name": "SequenceFile I/O Buffer Size", - "name": "io_file_buffer_size", - "value": "65536" - }, - { - "desc": "Fraction of the number of map tasks in the job which should be completed before reduce tasks are scheduled for the job.", - "display_name": "Number of Map Tasks to Complete Before Reduce Tasks", - "name": "mapred_reduce_slowstart_completed_maps", - "value": "0.8" - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for mapred-site.xml.", - "display_name": "MapReduce Client Advanced Configuration Snippet (Safety Valve) for mapred-site.xml", - "name": "mapreduce_client_config_safety_valve", - "value": null - }, - { - "desc": "The maximum heap size, in bytes, of the Java MapReduce ApplicationMaster. This number will be formatted and concatenated with 'ApplicationMaster Java Opts Base' to pass to Hadoop.", - "display_name": "ApplicationMaster Java Maximum Heap Size", - "name": "yarn_app_mapreduce_am_max_heap", - "value": "825955249" - }, - { - "desc": "The replication level for submitted job files.", - "display_name": "Mapreduce Submit Replication", - "name": "mapred_submit_replication", - "value": "10" - }, - { - "desc": "The total amount of memory buffer, in megabytes, to use while sorting files. Note that this memory comes out of the user JVM heap size (meaning total user JVM heap - this amount of memory = total user usable heap space. Note that Cloudera's default differs from Hadoop's default; Cloudera uses a bigger buffer by default because modern machines often have more RAM. The smallest value across all TaskTrackers will be part of generated client configuration.", - "display_name": "I/O Sort Memory Buffer (MiB)", - "name": "io_sort_mb", - "value": "256" - }, - { - "desc": "Whether to enable ubertask optimization, which runs \"sufficiently small\" jobs sequentially within a single JVM. \"Small\" is defined by the mapreduce.job.ubertask.maxmaps, mapreduce.job.ubertask.maxreduces, and mapreduce.job.ubertask.maxbytes settings.", - "display_name": "Enable Ubertask Optimization", - "name": "mapreduce_job_ubertask_enabled", - "value": "false" - }, - { - "desc": "Java command line arguments passed to the MapReduce ApplicationMaster.", - "display_name": "ApplicationMaster Java Opts Base", - "name": "yarn_app_mapreduce_am_command_opts", - "value": "-Djava.net.preferIPv4Stack=true" - }, - { - "desc": "The amount of physical memory, in MiB, allocated for each reduce task of a job. This parameter has no effect prior to CDH 4.4.", - "display_name": "Reduce Task Memory", - "name": "mapreduce_reduce_memory_mb", - "value": "1024" - }, - { - "desc": "The maximum permissible size of the split metainfo file. The JobTracker won't attempt to read split metainfo files bigger than the configured value. No limits if set to -1.", - "display_name": "JobTracker MetaInfo Maxsize", - "name": "mapreduce_jobtracker_split_metainfo_maxsize", - "value": "10000000" - }, - { - "desc": "Compression level for the codec used to compress MapReduce outputs. Default compression is a balance between speed and compression ratio.", - "display_name": "Compression Level of Codecs", - "name": "zlib_compress_level", - "value": "DEFAULT_COMPRESSION" - }, - { - "desc": "A shared directory for temporary files.", - "display_name": "Shared Temp Directories", - "name": "mapreduce_cluster_temp_dir", - "value": null - }, - { - "desc": "These are Java command line arguments. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Client Java Configuration Options", - "name": "mapreduce_client_java_opts", - "value": "-Djava.net.preferIPv4Stack=true" - }, - { - "desc": "The physical memory requirement, in MiB, for the ApplicationMaster.", - "display_name": "ApplicationMaster Memory", - "name": "yarn_app_mapreduce_am_resource_mb", - "value": "1024" - }, - { - "desc": "The maximum Java heap size, in bytes, of the reduce processes. This number will be formatted and concatenated with 'Reduce Task Java Opts Base' to pass to Hadoop.", - "display_name": "Reduce Task Maximum Heap Size", - "name": "mapreduce_reduce_java_opts_max_heap", - "value": "825955249" - }, - { - "desc": "Maximum size in bytes for the Java process heap memory. Passed to Java -Xmx.", - "display_name": "Client Java Heap Size in Bytes", - "name": "mapreduce_client_java_heapsize", - "value": "825955249" - }, - { - "desc": "The number of milliseconds before a task will be terminated if it neither reads an input, writes an output, nor updates its status string.", - "display_name": "Mapreduce Task Timeout", - "name": "mapred_task_timeout", - "value": "600000" - }, - { - "desc": "The virtual CPU cores requirement, for the ApplicationMaster. This parameter has no effect prior to CDH 4.4.", - "display_name": "ApplicationMaster Virtual CPU Cores", - "name": "yarn_app_mapreduce_am_resource_cpu_vcores", - "value": "1" - }, - { - "desc": "The amount of physical memory, in MiB, allocated for each map task of a job.", - "display_name": "Map Task Memory", - "name": "mapreduce_map_memory_mb", - "value": "1024" - }, - { - "desc": "Threshold for number of reduces, beyond which a job is considered too big for ubertask optimization. Note: As of CDH 5, MR2 does not support more than one reduce in an ubertask. (Zero is valid.)", - "display_name": "Ubertask Maximum Reduces", - "name": "mapreduce_job_ubertask_maxreduces", - "value": "1" - }, - { - "desc": "For MapReduce map outputs that are compressed, specify the compression codec to use. Will be part of generated client configuration.", - "display_name": "Compression Codec of MapReduce Map Output", - "name": "mapred_map_output_compression_codec", - "value": "org.apache.hadoop.io.compress.SnappyCodec" - }, - { - "desc": "Classpaths to include for MapReduce applications.", - "display_name": "MR Application Classpath", - "name": "mapreduce_application_classpath", - "value": "$HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$MR2_CLASSPATH" - }, - { - "desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.", - "display_name": "Enable Log Event Capture", - "name": "catch_events", - "value": "true" - }, - { - "desc": "Maximum sleep time between failover attempts. Used only if RM HA is enabled.", - "display_name": "Client Failover Sleep Max Time", - "name": "client_failover_sleep_max", - "value": "2000" - }, - { - "desc": "Limit on the number of counters allowed per job.", - "display_name": "Job Counters Limit", - "name": "mapreduce_job_counters_limit", - "value": "120" - }, - { - "desc": "Compress the output of MapReduce jobs. Will be part of generated client configuration.", - "display_name": "Compress MapReduce Job Output", - "name": "mapred_output_compress", - "value": "false" - }, - { - "desc": "For MapReduce job outputs that are compressed as SequenceFiles, you can select one of these compression type options: NONE, RECORD or BLOCK. Cloudera recommends BLOCK. Will be part of generated client configuration.", - "display_name": "Compression Type of MapReduce Job Output", - "name": "mapred_output_compression_type", - "value": "BLOCK" - }, - { - "desc": "Java opts for the map processes. The following symbol, if present, will be interpolated: @taskid@ is replaced by current TaskID. Any other occurrences of '@' will go unchanged. For example, to enable verbose gc logging to a file named for the taskid in /tmp pass a value of: \"-verbose:gc -Xloggc:/tmp/@taskid@.gc\". The configuration variable 'Map Task Memory' can be used to control the maximum memory of the map processes.", - "display_name": "Map Task Java Opts Base", - "name": "mapreduce_map_java_opts", - "value": "-Djava.net.preferIPv4Stack=true" - }, - { - "desc": "Additional execution environment entries for map and reduce task processes.", - "display_name": "MR Application Environment", - "name": "mapreduce_admin_user_env", - "value": "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH" - }, - { - "desc": "The soft limit in either the buffer or record collection buffers. When this limit is reached, a thread will begin to spill the contents to disk in the background. Note that this does not imply any chunking of data to the spill. A value less than 0.5 is not recommended. The syntax is in decimal units; the default is 80% and is formatted 0.8. Will be part of generated client configuration.", - "display_name": "I/O Sort Spill Percent", - "name": "io_sort_spill_percent", - "value": "0.8" - }, - { - "desc": "The default number of reduce tasks per job. Will be part of generated client configuration.", - "display_name": "Default Number of Reduce Tasks per Job", - "name": "mapred_reduce_tasks", - "value": "1" - }, - { - "desc": "Maximum allowed connections for the shuffle. Set to 0 (zero) to indicate no limit on the number of connections.", - "display_name": "Max Shuffle Connections", - "name": "mapreduce_shuffle_max_connections", - "value": "80" - }, - { - "desc": "The maximum Java heap size, in bytes, of the map processes. This number will be formatted and concatenated with 'Map Task Java Opts Base' to pass to Hadoop.", - "display_name": "Map Task Maximum Heap Size", - "name": "mapreduce_map_java_opts_max_heap", - "value": "825955249" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the client configuration for hadoop-env.sh", - "display_name": "Gateway Client Environment Advanced Configuration Snippet for hadoop-env.sh (Safety Valve)", - "name": "mapreduce_client_env_safety_valve", - "value": null - }, - { - "desc": "The number of virtual CPU cores for each reduce task of a job.", - "display_name": "Reduce Task CPU Virtual Cores", - "name": "mapreduce_reduce_cpu_vcores", - "value": "1" - }, - { - "desc": "Environment variables for the MapReduce ApplicationMaster. These settings can be overridden in the Application Master User Environment (yarn.app.mapreduce.am.env).", - "display_name": "ApplicationMaster Environment", - "name": "yarn_app_mapreduce_am_admin_user_env", - "value": "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH" - }, - { - "desc": "Java opts for the reduce processes. The following symbol, if present, will be interpolated: @taskid@ is replaced by current TaskID. Any other occurrences of '@' will go unchanged. For example, to enable verbose gc logging to a file named for the taskid in /tmp pass a value of: \"-verbose:gc -Xloggc:/tmp/@taskid@.gc\". The configuration variable 'Reduce Task Memory' can be used to control the maximum memory of the reduce processes.", - "display_name": "Reduce Task Java Opts Base", - "name": "mapreduce_reduce_java_opts", - "value": "-Djava.net.preferIPv4Stack=true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "If enabled, uses compression on the map outputs before they are sent across the network. Will be part of generated client configuration.", - "display_name": "Use Compression on Map Outputs", - "name": "mapred_compress_map_output", - "value": "true" - }, - { - "desc": "Threshold for number of maps, beyond which a job is considered too big for ubertask optimization.", - "display_name": "Ubertask Maximum Maps", - "name": "mapreduce_job_ubertask_maxmaps", - "value": "9" - }, - { - "desc": "If enabled, multiple instances of some map tasks may be executed in parallel.", - "display_name": "Map Tasks Speculative Execution", - "name": "mapred_map_tasks_speculative_execution", - "value": "false" - }, - { - "desc": "Threshold for number of input bytes, beyond which a job is considered too big for ubertask optimization. If no value is specified, dfs.block.size is used as a default.", - "display_name": "Ubertask Maximum Job Size", - "name": "mapreduce_job_ubertask_maxbytes", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/yarn-jobhistory.json b/sahara/plugins/cdh/v5_3_0/resources/yarn-jobhistory.json deleted file mode 100644 index fe8639c5..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/yarn-jobhistory.json +++ /dev/null @@ -1,290 +0,0 @@ -[ - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of JobHistory Server in Bytes", - "name": "mr2_jobhistory_java_heapsize", - "value": "1073741824" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "jobhistory_gc_duration_window", - "value": "5" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "The root HDFS directory of the staging area for users' MR2 jobs; for example /user. The staging directories are always named after the user.", - "display_name": "MapReduce ApplicationMaster Staging Root Directory", - "name": "yarn_app_mapreduce_am_staging_dir", - "value": "/user" - }, - { - "desc": "Enables the health test that the JobHistory Server's process state is consistent with the role configuration", - "display_name": "JobHistory Server Process Health Test", - "name": "jobhistory_scm_health_enabled", - "value": "true" - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "When computing the overall JobHistory Server health, consider the host's health.", - "display_name": "JobHistory Server Host Health Test", - "name": "jobhistory_host_health_enabled", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for JobHistory Server logs. Typically used by log4j.", - "display_name": "JobHistory Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Directory where JobHistory Server will place its log files.", - "display_name": "JobHistory Server Log Directory", - "name": "mr2_jobhistory_log_dir", - "value": "/var/log/hadoop-mapreduce" - }, - { - "desc": "Time interval for history cleaner to check for files to delete. Files are only deleted if they are older than mapreduce.jobhistory.max-age-ms.", - "display_name": "Job History Files Cleaner Interval", - "name": "mapreduce_jobhistory_cleaner_interval", - "value": "86400000" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "jobhistory_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for JobHistory Server", - "name": "mr2_jobhistory_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "For advanced use only, a string to be inserted into yarn-site.xml for this role only.", - "display_name": "JobHistory Server Advanced Configuration Snippet (Safety Valve) for yarn-site.xml", - "name": "jobhistory_config_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "JobHistory Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The port of the MapReduce JobHistory Server administrative interface. Together with the host name of the JobHistory role forms the address.", - "display_name": "MapReduce JobHistory Server Admin Interface Port", - "name": "mapreduce_jobhistory_admin_address", - "value": "10033" - }, - { - "desc": "The port of the MapReduce JobHistory Server. Together with the host name of the JobHistory role forms the address. ", - "display_name": "MapReduce JobHistory Server Port", - "name": "mapreduce_jobhistory_address", - "value": "10020" - }, - { - "desc": "The group that the JobHistory Server process should run as.", - "display_name": "System Group", - "name": "history_process_groupname", - "value": "hadoop" - }, - { - "desc": "Job history files older than this time duration will deleted when the history cleaner runs.", - "display_name": "Job History Files Maximum Age", - "name": "mapreduce_jobhistory_max_age_ms", - "value": "604800000" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for JobHistory Server logs. Typically used by log4j.", - "display_name": "JobHistory Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "jobhistory_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The HTTP port of the MapReduce JobHistory Server web application. Together with the host name of the JobHistory role forms the address.", - "display_name": "MapReduce JobHistory Web Application HTTP Port", - "name": "mapreduce_jobhistory_webapp_address", - "value": "19888" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "jobhistory_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The minimum log level for JobHistory Server logs", - "display_name": "JobHistory Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "For advanced use only, a string to be inserted into mapred-site.xml for this role only.", - "display_name": "JobHistory Server Advanced Configuration Snippet (Safety Valve) for mapred-site.xml", - "name": "jobhistory_mapred_safety_valve", - "value": null - }, - { - "desc": "If enabled, the JobHistory Server binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind JobHistory Server to Wildcard Address", - "name": "yarn_jobhistory_bind_wildcard", - "value": "false" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The user that the JobHistory Server process should run as.", - "display_name": "System User", - "name": "history_process_username", - "value": "mapred" - }, - { - "desc": "The HTTPS port of the MapReduce JobHistory Server web application. Together with the host name of the JobHistory role forms the address.", - "display_name": "MapReduce JobHistory Web Application HTTPS Port (SSL)", - "name": "mapreduce_jobhistory_webapp_https_address", - "value": "19890" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "jobhistory_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/yarn-nodemanager.json b/sahara/plugins/cdh/v5_3_0/resources/yarn-nodemanager.json deleted file mode 100644 index 0a99a1a4..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/yarn-nodemanager.json +++ /dev/null @@ -1,428 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into yarn-site.xml for this role only.", - "display_name": "NodeManager Advanced Configuration Snippet (Safety Valve) for yarn-site.xml", - "name": "nodemanager_config_safety_valve", - "value": null - }, - { - "desc": "The HTTP Port of the NodeManager web application.", - "display_name": "NodeManager Web Application HTTP Port", - "name": "nodemanager_webserver_port", - "value": "8042" - }, - { - "desc": "If enabled, adds 'org.apache.hadoop.mapred.ShuffleHandler' to the NodeManager auxiliary services. This is required for MapReduce applications.", - "display_name": "Enable Shuffle Auxiliary Service", - "name": "mapreduce_aux_service", - "value": "true" - }, - { - "desc": "Heartbeat interval to ResourceManager", - "display_name": "Heartbeat Interval", - "name": "yarn_nodemanager_heartbeat_interval_ms", - "value": "1000" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Enables the health test that the NodeManager's process state is consistent with the role configuration", - "display_name": "NodeManager Process Health Test", - "name": "nodemanager_scm_health_enabled", - "value": "true" - }, - { - "desc": "The amount of time to wait for the NodeManager to fully start up and connect to the ResourceManager before enforcing the connectivity check.", - "display_name": "NodeManager Connectivity Tolerance at Startup", - "name": "nodemanager_connectivity_tolerance_seconds", - "value": "180" - }, - { - "desc": "List of directories on the local filesystem where a NodeManager stores intermediate data files.", - "display_name": "NodeManager Local Directory List", - "name": "yarn_nodemanager_local_dirs", - "value": null - }, - { - "desc": "Enables the health check that verifies the NodeManager is connected to the ResourceManager", - "display_name": "NodeManager Connectivity Health Check", - "name": "nodemanager_connectivity_health_enabled", - "value": "true" - }, - { - "desc": "Number of threads to handle localization requests.", - "display_name": "Localizer Client Thread Count", - "name": "yarn_nodemanager_localizer_client_thread_count", - "value": "5" - }, - { - "desc": "The local filesystem directory in which the NodeManager will store state when recovery is enabled (as it is by default).", - "display_name": "NodeManager Recovery Directory", - "name": "yarn_nodemanager_recovery_dir", - "value": "/var/lib/hadoop-yarn/yarn-nm-recovery" - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "Number of threads to use for localization fetching.", - "display_name": "Localizer Fetch Thread Count", - "name": "yarn_nodemanager_localizer_fetch_thread_count", - "value": "4" - }, - { - "desc": "Target size of localizer cache in MB, per local directory.", - "display_name": "Localizer Cache Target Size", - "name": "yarn_nodemanager_localizer_cache_target_size_mb", - "value": "10240" - }, - { - "desc": "The minimum Linux user ID allowed. Used to prevent other super users.", - "display_name": "Minimum User ID", - "name": "container_executor_min_user_id", - "value": "1000" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for NodeManager", - "name": "node_manager_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "List of users explicitly whitelisted to be allowed to run containers. Users with IDs lower than the \"Minimum User Id\" setting may be whitelisted by using this setting.", - "display_name": "Allowed System Users", - "name": "container_executor_allowed_system_users", - "value": "nobody,impala,hive,llama" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of NodeManager in Bytes", - "name": "node_manager_java_heapsize", - "value": "1073741824" - }, - { - "desc": "Directory where NodeManager will place its log files.", - "display_name": "NodeManager Log Directory", - "name": "node_manager_log_dir", - "value": "/var/log/hadoop-yarn" - }, - { - "desc": "The system group that owns the container-executor binary. This does not need to be changed unless the ownership of the binary is explicitly changed.", - "display_name": "Container Executor Group", - "name": "container_executor_group", - "value": "yarn" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "nodemanager_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "nodemanager_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The address of the NodeManager IPC.", - "display_name": "NodeManager IPC Address", - "name": "yarn_nodemanager_address", - "value": "8041" - }, - { - "desc": "Environment variables that should be forwarded from the NodeManager's environment to the container's.", - "display_name": "Containers Environment Variable", - "name": "yarn_nodemanager_admin_env", - "value": "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "nodemanager_gc_duration_window", - "value": "5" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "NodeManager Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "HDFS directory where application logs are stored when an application completes.", - "display_name": "Remote App Log Directory", - "name": "yarn_nodemanager_remote_app_log_dir", - "value": "/tmp/logs" - }, - { - "desc": "Time in seconds to retain user logs. Only applicable if log aggregation is disabled.", - "display_name": "Log Retain Duration", - "name": "yarn_nodemanager_log_retain_seconds", - "value": "10800" - }, - { - "desc": "Number of threads used in cleanup.", - "display_name": "Cleanup Thread Count", - "name": "yarn_nodemanager_delete_thread_count", - "value": "4" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "Number of threads container manager uses.", - "display_name": "Container Manager Thread Count", - "name": "yarn_nodemanager_container_manager_thread_count", - "value": "20" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Address where the localizer IPC is.", - "display_name": "Localizer Port", - "name": "yarn_nodemanager_localizer_address", - "value": "8040" - }, - { - "desc": "Address where the localizer IPC is.", - "display_name": "Localizer Cache Cleanup Interval", - "name": "yarn_nodemanager_localizer_cache_cleanup_interval_ms", - "value": "600000" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "Absolute path to the script which is periodically run by the node health monitoring service to determine if the node is healthy or not. If the value of this key is empty or the file does not exist in the location configured here, the node health monitoring service is not started.", - "display_name": "Healthchecker Script Path", - "name": "mapred_healthchecker_script_path", - "value": "" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The HTTPS port of the NodeManager web application.", - "display_name": "NodeManager Web Application HTTPS Port (SSL)", - "name": "nodemanager_webserver_https_port", - "value": "8044" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When computing the overall NodeManager health, consider the host's health.", - "display_name": "NodeManager Host Health Test", - "name": "nodemanager_host_health_enabled", - "value": "true" - }, - { - "desc": "List of directories on the local filesystem where a NodeManager stores container log files.", - "display_name": "NodeManager Container Log Directories", - "name": "yarn_nodemanager_log_dirs", - "value": "/var/log/hadoop-yarn/container" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Path (rooted in the cgroups hierarchy on the machine) where to place YARN-managed cgroups.", - "display_name": "CGroups Hierarchy", - "name": "linux_container_executor_cgroups_hierarchy", - "value": "/hadoop-yarn" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "nodemanager_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "For advanced use only, a string to be inserted into mapred-site.xml for this role only.", - "display_name": "NodeManager Advanced Configuration Snippet (Safety Valve) for mapred-site.xml", - "name": "nodemanager_mapred_safety_valve", - "value": null - }, - { - "desc": "The maximum size, in megabytes, per log file for NodeManager logs. Typically used by log4j.", - "display_name": "NodeManager Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The remote log dir will be created at {yarn.nodemanager.remote-app-log-dir}/${user}/{thisParam}", - "display_name": "Remote App Log Directory Suffix", - "name": "yarn_nodemanager_remote_app_log_dir_suffix", - "value": "logs" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "The maximum number of rolled log files to keep for NodeManager logs. Typically used by log4j.", - "display_name": "NodeManager Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "Amount of physical memory, in MiB, that can be allocated for containers.", - "display_name": "Container Memory", - "name": "yarn_nodemanager_resource_memory_mb", - "value": "8192" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "nodemanager_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Number of virtual CPU cores that can be allocated for containers. This parameter has no effect prior to CDH 4.4.", - "display_name": "Container Virtual CPU Cores", - "name": "yarn_nodemanager_resource_cpu_vcores", - "value": "8" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "Environment variables that containers may override rather than use NodeManager's default.", - "display_name": "Containers Environment Variables Whitelist ", - "name": "yarn_nodemanager_env_whitelist", - "value": "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "Comma-separated list of arguments which are to be passed to node health script when it is being launched.", - "display_name": "Healthchecker Script Arguments", - "name": "mapred_healthchecker_script_args", - "value": "" - }, - { - "desc": "List of users banned from running containers.", - "display_name": "Banned System Users", - "name": "container_executor_banned_users", - "value": "hdfs,yarn,mapred,bin" - }, - { - "desc": "The minimum log level for NodeManager logs", - "display_name": "NodeManager Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Number of seconds after an application finishes before the NodeManager's DeletionService will delete the application's localized file and log directory. To diagnose YARN application problems, set this property's value large enough (for example, to 600 = 10 minutes) to permit examination of these directories.", - "display_name": "Localized Dir Deletion Delay", - "name": "yarn_nodemanager_delete_debug_delay_sec", - "value": "0" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/yarn-resourcemanager.json b/sahara/plugins/cdh/v5_3_0/resources/yarn-resourcemanager.json deleted file mode 100644 index e160b373..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/yarn-resourcemanager.json +++ /dev/null @@ -1,476 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "resourcemanager_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "An XML string that will be inserted verbatim into the Fair Scheduler allocations file. For CDH5, overrides the configuration set using the Pools configuration UI. For CDH4, this is the only way to configure the Fair Scheduler for YARN.", - "display_name": "Fair Scheduler XML Advanced Configuration Snippet (Safety Valve)", - "name": "resourcemanager_fair_scheduler_configuration", - "value": null - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into nodes_allow.txt for this role only.", - "display_name": "ResourceManager Advanced Configuration Snippet (Safety Valve) for nodes_allow.txt", - "name": "rm_hosts_allow_safety_valve", - "value": null - }, - { - "desc": "The maximum number of application attempts. This is a global setting for all ApplicationMasters. Each ApplicationMaster can specify its individual maximum through the API, but if the individual maximum is more than the global maximum, the ResourceManager will override it.", - "display_name": "ApplicationMaster Maximum Attempts", - "name": "yarn_resourcemanager_am_max_retries", - "value": "2" - }, - { - "desc": "For advanced use only, a string to be inserted into nodes_exclude.txt for this role only.", - "display_name": "ResourceManager Advanced Configuration Snippet (Safety Valve) for nodes_exclude.txt", - "name": "rm_hosts_exclude_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "resourcemanager_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The maximum number of completed applications that the ResourceManager keeps.", - "display_name": "Max Completed Applications", - "name": "yarn_resourcemanager_max_completed_applications", - "value": "10000" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "For advanced use only, a string to be inserted into yarn-site.xml for this role only.", - "display_name": "ResourceManager Advanced Configuration Snippet (Safety Valve) for yarn-site.xml", - "name": "resourcemanager_config_safety_valve", - "value": null - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for ResourceManager", - "name": "resource_manager_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "Enables the health test that the ResourceManager's process state is consistent with the role configuration", - "display_name": "ResourceManager Process Health Test", - "name": "resourcemanager_scm_health_enabled", - "value": "true" - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "Directory where ResourceManager will place its log files.", - "display_name": "ResourceManager Log Directory", - "name": "resource_manager_log_dir", - "value": "/var/log/hadoop-yarn" - }, - { - "desc": "The smallest amount of physical memory, in MiB, that can be requested for a container. If using the Capacity or FIFO scheduler (or any scheduler, prior to CDH 5), memory requests will be rounded up to the nearest multiple of this number.", - "display_name": "Container Memory Minimum", - "name": "yarn_scheduler_minimum_allocation_mb", - "value": "1024" - }, - { - "desc": "The number of threads used to handle applications manager requests.", - "display_name": "Client Thread Count", - "name": "yarn_resourcemanager_client_thread_count", - "value": "50" - }, - { - "desc": "The periodic interval that the ResourceManager will check whether containers are still alive.", - "display_name": "Container Monitor Interval", - "name": "yarn_resourcemanager_container_liveness_monitor_interval_ms", - "value": "600000" - }, - { - "desc": "The HTTP port of the ResourceManager web application.", - "display_name": "ResourceManager Web Application HTTP Port", - "name": "resourcemanager_webserver_port", - "value": "8088" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The expiry interval to wait until a NodeManager is considered dead.", - "display_name": "NodeManager Monitor Expiry", - "name": "yarn_nm_liveness_monitor_expiry_interval_ms", - "value": "600000" - }, - { - "desc": "The largest amount of physical memory, in MiB, that can be requested for a container.", - "display_name": "Container Memory Maximum", - "name": "yarn_scheduler_maximum_allocation_mb", - "value": "65536" - }, - { - "desc": "The largest number of virtual CPU cores that can be requested for a container. This parameter has no effect prior to CDH 4.4.", - "display_name": "Container Virtual CPU Cores Maximum", - "name": "yarn_scheduler_maximum_allocation_vcores", - "value": "32" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "resourcemanager_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "Number of threads to handle resource tracker calls.", - "display_name": "Resource Tracker Thread Count", - "name": "yarn_resourcemanager_resource_tracker_client_thread_count", - "value": "50" - }, - { - "desc": "If using the Fair Scheduler, virtual core requests will be rounded up to the nearest multiple of this number. This parameter has no effect prior to CDH 5.", - "display_name": "Container Virtual CPU Cores Increment", - "name": "yarn_scheduler_increment_allocation_vcores", - "value": "1" - }, - { - "desc": "The address of the scheduler interface in the ResourceManager.", - "display_name": "Scheduler Address", - "name": "yarn_resourcemanager_scheduler_address", - "value": "8030" - }, - { - "desc": "For advanced use only, a string to be inserted into mapred-site.xml for this role only.", - "display_name": "ResourceManager Advanced Configuration Snippet (Safety Valve) for mapred-site.xml", - "name": "resourcemanager_mapred_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The HTTPS port of the ResourceManager web application.", - "display_name": "ResourceManager Web Application HTTPS Port (SSL)", - "name": "resourcemanager_webserver_https_port", - "value": "8090" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "resourcemanager_gc_duration_window", - "value": "5" - }, - { - "desc": "For applications that request containers on particular racks, the number of scheduling opportunities since the last container assignment to wait before accepting a placement on another rack. Expressed as a float between 0 and 1, which, as a fraction of the cluster size, is the number of scheduling opportunities to pass up. If not set, this means don't pass up any scheduling opportunities. Requires Fair Scheduler continuous scheduling to be disabled. If continuous scheduling is enabled, yarn.scheduler.fair.locality-delay-rack-ms should be used instead.", - "display_name": "Fair Scheduler Rack Locality Threshold", - "name": "resourcemanager_fair_scheduler_locality_threshold_rack", - "value": null - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "For applications that request containers on particular racks, the minimum time in milliseconds the Fair Scheduler waits before accepting a placement on another rack. Requires Fair Scheduler continuous scheduling to be enabled. If continuous scheduling is disabled, yarn.scheduler.fair.locality.threshold.rack should be used instead.", - "display_name": "Fair Scheduler Rack Locality Delay", - "name": "yarn_scheduler_fair_locality_delay_rack_ms", - "value": "4000" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The address of the applications manager interface in the ResourceManager.", - "display_name": "ResourceManager Address", - "name": "yarn_resourcemanager_address", - "value": "8032" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "ResourceManager Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "Enables multiple Fair Scheduler container assignments in one heartbeat, which improves cluster throughput when there are many small tasks to run.", - "display_name": "Fair Scheduler Assign Multiple Tasks", - "name": "resourcemanager_fair_scheduler_assign_multiple", - "value": "false" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "resourcemanager_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "When enabled, any applications that were running on the cluster when the ResourceManager died will be recovered when the ResourceManager next starts. Note: If RM-HA is enabled, then this configuration is always enabled.", - "display_name": "Enable ResourceManager Recovery", - "name": "yarn_resourcemanager_recovery_enabled", - "value": "false" - }, - { - "desc": "The periodic interval that the ResourceManager will check whether ApplicationMasters is still alive.", - "display_name": "ApplicationMaster Monitor Interval", - "name": "yarn_resourcemanager_amliveliness_monitor_interval_ms", - "value": "1000" - }, - { - "desc": "For applications that request containers on particular nodes, the minimum time in milliseconds the Fair Scheduler waits before accepting a placement on another node. Requires Fair Scheduler continuous scheduling to be enabled. If continuous scheduling is disabled, yarn.scheduler.fair.locality.threshold.node should be used instead.", - "display_name": "Fair Scheduler Node Locality Delay", - "name": "yarn_scheduler_fair_locality_delay_node_ms", - "value": "2000" - }, - { - "desc": "Number of threads used to handle the ResourceManager admin interface.", - "display_name": "Admin Client Thread Count", - "name": "yarn_resourcemanager_admin_client_thread_count", - "value": "1" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of ResourceManager in Bytes", - "name": "resource_manager_java_heapsize", - "value": "1073741824" - }, - { - "desc": "The maximum size, in megabytes, per log file for ResourceManager logs. Typically used by log4j.", - "display_name": "ResourceManager Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When enabled, if a pool's minimum share is not met for some period of time, the Fair Scheduler preempts applications in other pools. Preemption guarantees that production applications are not starved while also allowing the cluster to be used for experimental and research applications. To minimize wasted computation, the Fair Scheduler preempts the most recently launched applications.", - "display_name": "Fair Scheduler Preemption", - "name": "resourcemanager_fair_scheduler_preemption", - "value": "false" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The smallest number of virtual CPU cores that can be requested for a container. If using the Capacity or FIFO scheduler (or any scheduler, prior to CDH 5), virtual core requests will be rounded up to the nearest multiple of this number. This parameter has no effect prior to CDH 4.4.", - "display_name": "Container Virtual CPU Cores Minimum", - "name": "yarn_scheduler_minimum_allocation_vcores", - "value": "1" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "If using the Fair Scheduler, memory requests will be rounded up to the nearest multiple of this number. This parameter has no effect prior to CDH 5.", - "display_name": "Container Memory Increment", - "name": "yarn_scheduler_increment_allocation_mb", - "value": "512" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "If enabled, the ResourceManager binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind ResourceManager to Wildcard Address", - "name": "yarn_rm_bind_wildcard", - "value": "false" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Enable continuous scheduling in the Fair Scheduler. When enabled, scheduling decisions are decoupled from NodeManager heartbeats, leading to faster resource allocations.", - "display_name": "Enable Fair Scheduler Continuous Scheduling", - "name": "yarn_scheduler_fair_continuous_scheduling_enabled", - "value": "true" - }, - { - "desc": "The directory in which stacks logs will be placed. If not set, stacks will be logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "When computing the overall ResourceManager health, consider the host's health.", - "display_name": "ResourceManager Host Health Test", - "name": "resourcemanager_host_health_enabled", - "value": "true" - }, - { - "desc": "The number of threads used to handle requests through the scheduler interface.", - "display_name": "Scheduler Thread Count", - "name": "yarn_resourcemanager_scheduler_client_thread_count", - "value": "50" - }, - { - "desc": "The maximum number of rolled log files to keep for ResourceManager logs. Typically used by log4j.", - "display_name": "ResourceManager Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "When enabled, the Fair Scheduler will assign shares to individual apps based on their size, rather than providing an equal share to all apps regardless of size.", - "display_name": "Fair Scheduler Size-Based Weight", - "name": "resourcemanager_fair_scheduler_size_based_weight", - "value": "false" - }, - { - "desc": "For applications that request containers on particular nodes, the number of scheduling opportunities since the last container assignment to wait before accepting a placement on another node. Expressed as a float between 0 and 1, which, as a fraction of the cluster size, is the number of scheduling opportunities to pass up. If not set, this means don't pass up any scheduling opportunities. Requires Fair Scheduler continuous scheduling to be disabled. If continuous scheduling is enabled, yarn.scheduler.fair.locality-delay-node-ms should be used instead.", - "display_name": "Fair Scheduler Node Locality Threshold", - "name": "resourcemanager_fair_scheduler_locality_threshold_node", - "value": null - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it doesn't exist. However, if this directory already exists, role user must have write access to this directory. If this directory is shared amongst multiple roles, it should have 1777 permissions. Note that the heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks will be collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "When set to true, the Fair Scheduler uses the username as the default pool name, in the event that a pool name is not specified. When set to false, all applications are run in a shared pool, called default.", - "display_name": "Fair Scheduler User As Default Queue", - "name": "resourcemanager_fair_scheduler_user_as_default_queue", - "value": "true" - }, - { - "desc": "The address of the resource tracker interface in the ResourceManager.", - "display_name": "Resource Tracker Address", - "name": "yarn_resourcemanager_resource_tracker_address", - "value": "8031" - }, - { - "desc": "The expiry interval to wait until an ApplicationMaster is considered dead.", - "display_name": "ApplicationMaster Monitor Expiry", - "name": "yarn_am_liveness_monitor_expiry_interval_ms", - "value": "600000" - }, - { - "desc": "The class to use as the resource scheduler. FairScheduler is only supported in CDH 4.2.1 and later.", - "display_name": "Scheduler Class", - "name": "yarn_resourcemanager_scheduler_class", - "value": "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler" - }, - { - "desc": "The minimum log level for ResourceManager logs", - "display_name": "ResourceManager Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", - "display_name": "Health Check Startup Tolerance", - "name": "resourcemanager_startup_tolerance_minutes", - "value": "5" - }, - { - "desc": "The address of the admin interface in the ResourceManager.", - "display_name": "Administration Address", - "name": "yarn_resourcemanager_admin_address", - "value": "8033" - }, - { - "desc": "The periodic interval that the ResourceManager will check whether NodeManagers are still alive.", - "display_name": "NodeManager Monitor Interval", - "name": "yarn_resourcemanager_nm_liveness_monitor_interval_ms", - "value": "1000" - }, - { - "desc": "Enter an XML string that represents the Capacity Scheduler configuration.", - "display_name": "Capacity Scheduler Configuration", - "name": "resourcemanager_capacity_scheduler_configuration", - "value": "\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "Controls which applications an admin user can see in the applications list view", - "display_name": "Admin Users Applications List Visibility Settings", - "name": "admin_application_list_settings", - "value": "ALL" - }, - { - "desc": "Whether YARN uses the Linux Container Executor both in secure (Kerberos) and insecure (not Kerberos) environments. Cgroups enforcement only works when the Linux Container Executor is used.", - "display_name": "Always Use Linux Container Executor", - "name": "yarn_service_lce_always", - "value": "false" - }, - { - "desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.", - "display_name": "Enable Log Event Capture", - "name": "catch_events", - "value": "true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Comma-delimited list of groups that you want to allow the Llama (AM for Impala) user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", - "display_name": "Llama Proxy User Groups", - "name": "llama_proxy_user_groups_list", - "value": "*" - }, - { - "desc": "Path to the keystore file containing the server certificate and private key used for encrypted shuffle and encrypted web UIs. Applies to configurations of all daemon roles of this service.", - "display_name": "Hadoop SSL Server Keystore File Location", - "name": "ssl_server_keystore_location", - "value": null - }, - { - "desc": "Controls the aggregate metrics generated for YARN applications. The structure is a JSON list of the attributes to aggregate and the entities to aggregate to. For example, if the attributeName is 'maps_completed' and the aggregationTargets is ['USER'] then the Service Monitor will create the metric 'yarn_application_maps_completed_rate' and, every ten minutes, will record the total maps completed for each user across all their YARN applications. By default it will also record the number of applications submitted ('apps_submitted_rate') for both users and pool. For a full list of the supported attributes see the YARN search page. Note that the valid aggregation targets are USER, YARN_POOL, and YARN (the service), and that these aggregate metrics can be viewed on both the reports and charts search pages.", - "display_name": "YARN Application Aggregates", - "name": "yarn_application_aggregates", - "value": "[\n {\n \"attributeName\": \"maps_total\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"reduces_total\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"cpu_milliseconds\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"mb_millis_maps\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"mb_millis_reduces\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"vcores_millis_maps\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"vcores_millis_reduces\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"file_bytes_read\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"file_bytes_written\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"hdfs_bytes_read\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"hdfs_bytes_written\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"cm_cpu_milliseconds\",\n \"aggregationTargets\": [\"USER\"]\n }\n]\n" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the Llama (AM for Impala) user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", - "display_name": "Llama Proxy User Hosts", - "name": "llama_proxy_user_hosts_list", - "value": "*" - }, - { - "desc": "UNIX user that containers run as when Linux-container-executor is used in nonsecure mode.", - "display_name": "UNIX User for Nonsecure Mode with Linux Container Executor", - "name": "yarn_nodemanager_linux_container_executor_nonsecure_mode_local_user", - "value": "nobody" - }, - { - "desc": "For advanced use only, a string to be inserted into mapred-site.xml. Applies to configurations of all roles in this service except client configuration.", - "display_name": "YARN Service MapReduce Advanced Configuration Snippet (Safety Valve)", - "name": "yarn_service_mapred_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "YARN Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "yarn_service_env_safety_valve", - "value": null - }, - { - "desc": "Whether to enable log aggregation", - "display_name": "Enable Log Aggregation", - "name": "yarn_log_aggregation_enable", - "value": "true" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/resources/zookeeper-server.json b/sahara/plugins/cdh/v5_3_0/resources/zookeeper-server.json deleted file mode 100644 index 51ff04e8..00000000 --- a/sahara/plugins/cdh/v5_3_0/resources/zookeeper-server.json +++ /dev/null @@ -1,386 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "The port to monitor for inter-server communication", - "display_name": "Quorum Port", - "name": "quorumPort", - "value": "3181" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "The port to monitor for leadership election", - "display_name": "Election Port", - "name": "electionPort", - "value": "4181" - }, - { - "desc": "Specifies the name of the user that has read-only privileges when using password file based authentication for JMX access. JMX authentication must be enabled for this setting to take effect.", - "display_name": "Name of User with Read-Only access to the JMX Agent", - "name": "jmx_passwd_file_readonly_user", - "value": "monitorRole" - }, - { - "desc": "The health check thresholds of the weighted average size of the ZooKeeper Server connection count over a recent period. See ZooKeeper Server Connection Count Monitoring Period.", - "display_name": "ZooKeeper Server Connection Count Thresholds", - "name": "zookeeper_server_connection_count_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The disk location that ZooKeeper will use to store its transaction logs.", - "display_name": "Transaction Log Directory", - "name": "dataLogDir", - "value": "/var/lib/zookeeper" - }, - { - "desc": "The port used by the ZooKeeper Server's RMI registry. This is required to enable JMX access through RMI which is required for Cloudera Manager ZooKeeper monitoring. This is added as \"-Dcom.sun.management.jmxremote.port\" to the ZooKeeper Server's JVM command line.", - "display_name": "JMX Remote Port", - "name": "server_jmx_agent_port", - "value": "9010" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "zookeeper_server_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "The tolerance window that will be used in the detection of a ZooKeeper server's membership in a quorum. Specified in minutes.", - "display_name": "Quorum Membership Detection Window", - "name": "zookeeper_server_quorum_membership_detection_window", - "value": "3" - }, - { - "desc": "Enables the health test that the Server's process state is consistent with the role configuration", - "display_name": "Server Process Health Test", - "name": "zookeeper_server_scm_health_enabled", - "value": "true" - }, - { - "desc": "The address (IPv4, IPv6, or hostname) to monitor for client connections. This is the address that clients attempt to connect to. This setting is optional, because by default, ZooKeeper binds in such a way that any connection to the client port for any address/interface/NIC on the server will be accepted.", - "display_name": "Client Port Address", - "name": "clientPortAddress", - "value": null - }, - { - "desc": "When computing the overall Server health, consider the host's health.", - "display_name": "Server Host Health Test", - "name": "zookeeper_server_host_health_enabled", - "value": "true" - }, - { - "desc": "The method that will be used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "Specifies the password of the user that has read-only privileges when using password file based authentication for JMX access. JMX authentication must be enabled for this setting to take effect.", - "display_name": "Password of User with Read-Only Access to the JMX agent", - "name": "jmx_passwd_file_readonly_user_password", - "value": "MONITOR" - }, - { - "desc": "The health check thresholds of the weighted average size of the ZooKeeper Server outstanding requests queue over a recent period. See ZooKeeper Server Outstanding Requests Monitoring Period.", - "display_name": "ZooKeeper Server Outstanding Requests Thresholds", - "name": "zookeeper_server_outstanding_requests_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The period to review when computing the moving average of the outstanding requests queue size. Specified in minutes.", - "display_name": "ZooKeeper Server Outstanding Requests Monitoring Period", - "name": "zookeeper_server_outstanding_requests_window", - "value": "3" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of ZooKeeper Server in Bytes", - "name": "zookeeper_server_java_heapsize", - "value": "1073741824" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger configured for a DataNode that fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "The minimum session timeout, in milliseconds, that the ZooKeeper Server will allow the client to negotiate", - "display_name": "Minimum Session Timeout", - "name": "minSessionTimeout", - "value": "4000" - }, - { - "desc": "Specifies the name of the user that has read-write privileges when using password file based authentication for JMX access. JMX authentication must be enabled for this setting to take effect.", - "display_name": "Name of User with Read-Write Access to the JMX Agent", - "name": "jmx_passwd_file_readwrite_user", - "value": "controlRole" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health check thresholds for monitoring of free space on the filesystem that contains the ZooKeeper server's data directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Data Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Data Directory Free Space Monitoring Percentage Thresholds", - "name": "zookeeper_server_data_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The disk location that ZooKeeper will use to store its database snapshots.", - "display_name": "Data Directory", - "name": "dataDir", - "value": "/var/lib/zookeeper" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health check thresholds for monitoring of free space on the filesystem that contains the ZooKeeper Server's data directory.", - "display_name": "Data Directory Free Space Monitoring Absolute Thresholds", - "name": "zookeeper_server_data_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The amount of stacks data that will be retained. After the retention limit is reached, the oldest data will be deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "Enables authentication when interacting with the JMX agent on the ZooKeeper server.", - "display_name": "Enable Authenticated Communication with the JMX Agent", - "name": "enable_jmx_authentication", - "value": "false" - }, - { - "desc": "The period to review when computing the moving average of the connection count. Specified in minutes.", - "display_name": "ZooKeeper Server Connection Count Monitoring Period", - "name": "zookeeper_server_connection_count_window", - "value": "3" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The maximum session timeout, in milliseconds, that the ZooKeeper Server will allow the client to negotiate", - "display_name": "Maximum Session Timeout", - "name": "maxSessionTimeout", - "value": "40000" - }, - { - "desc": "For advanced use only, a string to be inserted into zoo.cfg for this role only.", - "display_name": "Server Advanced Configuration Snippet (Safety Valve) for zoo.cfg", - "name": "zookeeper_config_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The maximum number of concurrent connections (at the socket level) that a single client, identified by the IP address, may make to a single member of the ZooKeeper ensemble. This setting is used to prevent certain classes of DoS attacks, including file descriptor exhaustion. To remove the limit on concurrent connections, set this value to 0.", - "display_name": "Maximum Client Connections", - "name": "maxClientCnxns", - "value": "60" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Zookeeper Server", - "name": "zk_server_java_opts", - "value": "" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The port used by the ZooKeeper Server's RMI server to handle JMX RMI requests. This is added as \"-Dcom.sun.management.jmxremote.rmi.port=The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - the name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - a tsquery expression representing the trigger. streamThreshold (optional) - the maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - by default set to 'true'. If set to 'false' the trigger will not be evaluated.For example, here is a JSON formatted trigger that fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]Consult the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and as a result backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "The length of time, in milliseconds, of a single tick, which is the basic time unit used by ZooKeeper. A tick is used to regulate heartbeats and timeouts.", - "display_name": "Tick Time", - "name": "tickTime", - "value": "2000" - }, - { - "desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.", - "display_name": "Enable Log Event Capture", - "name": "catch_events", - "value": "true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The health test thresholds of the overall Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Servers falls below the critical threshold.", - "display_name": "Healthy Server Monitoring Thresholds", - "name": "zookeeper_servers_healthy_thresholds", - "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "Whether the leader accepts client connections.", - "display_name": "Leader Serves", - "name": "leaderServes", - "value": "yes" - }, - { - "desc": "Automatically create data directories at startup, if they do not exist. Enabling this configuration should be used with care as it will suppress any errors in setup of data directories.", - "display_name": "Enable auto-creation of data directories", - "name": "zookeeper_datadir_autocreate", - "value": "false" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_3_0/validation.py b/sahara/plugins/cdh/v5_3_0/validation.py deleted file mode 100644 index 8f715a5d..00000000 --- a/sahara/plugins/cdh/v5_3_0/validation.py +++ /dev/null @@ -1,158 +0,0 @@ -# Copyright (c) 2014 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.i18n import _ -from sahara.plugins.cdh.v5_3_0 import plugin_utils as pu -from sahara.plugins.cdh import validation -from sahara.plugins import exceptions as ex -from sahara.plugins import utils as u - - -class ValidatorV530(validation.Validator): - PU = pu.PluginUtilsV530() - - @classmethod - def validate_cluster_creating(cls, cluster): - super(ValidatorV530, cls).validate_cluster_creating(cluster) - cls._flume_validation(cluster) - cls._sentry_validation(cluster) - cls._solr_validation(cluster) - cls._sqoop_validation(cluster) - cls._hbase_indexer_validation(cluster) - cls._impala_validation(cluster) - - @classmethod - def _flume_validation(cls, cluster): - a_count = cls._get_inst_count(cluster, 'FLUME_AGENT') - dn_count = cls._get_inst_count(cluster, 'HDFS_DATANODE') - - if a_count >= 1: - if dn_count < 1: - raise ex.RequiredServiceMissingException( - 'HDFS_DATANODE', required_by='FLUME_AGENT') - - @classmethod - def _sentry_validation(cls, cluster): - - snt_count = cls._get_inst_count(cluster, 'SENTRY_SERVER') - dn_count = cls._get_inst_count(cluster, 'HDFS_DATANODE') - zk_count = cls._get_inst_count(cluster, 'ZOOKEEPER_SERVER') - - if snt_count > 1: - raise ex.InvalidComponentCountException( - 'SENTRY_SERVER', _('0 or 1'), snt_count) - if snt_count == 1: - if dn_count < 1: - raise ex.RequiredServiceMissingException( - 'HDFS_DATANODE', required_by='SENTRY_SERVER') - if zk_count < 1: - raise ex.RequiredServiceMissingException( - 'ZOOKEEPER', required_by='SENTRY_SERVER') - - @classmethod - def _solr_validation(cls, cluster): - slr_count = cls._get_inst_count(cluster, 'SOLR_SERVER') - zk_count = cls._get_inst_count(cluster, 'ZOOKEEPER_SERVER') - dn_count = cls._get_inst_count(cluster, 'HDFS_DATANODE') - - if slr_count >= 1: - if dn_count < 1: - raise ex.RequiredServiceMissingException( - 'HDFS_DATANODE', required_by='SOLR_SERVER') - if zk_count < 1: - raise ex.RequiredServiceMissingException( - 'ZOOKEEPER', required_by='SOLR_SERVER') - - @classmethod - def _sqoop_validation(cls, cluster): - - s2s_count = cls._get_inst_count(cluster, 'SQOOP_SERVER') - dn_count = cls._get_inst_count(cluster, 'HDFS_DATANODE') - hs_count = cls._get_inst_count(cluster, 'YARN_JOBHISTORY') - nm_count = cls._get_inst_count(cluster, 'YARN_NODEMANAGER') - - if s2s_count > 1: - raise ex.InvalidComponentCountException( - 'SQOOP_SERVER', _('0 or 1'), s2s_count) - if s2s_count == 1: - if dn_count < 1: - raise ex.RequiredServiceMissingException( - 'HDFS_DATANODE', required_by='SQOOP_SERVER') - if nm_count < 1: - raise ex.RequiredServiceMissingException( - 'YARN_NODEMANAGER', required_by='SQOOP_SERVER') - if hs_count != 1: - raise ex.RequiredServiceMissingException( - 'YARN_JOBHISTORY', required_by='SQOOP_SERVER') - - @classmethod - def _hbase_indexer_validation(cls, cluster): - - lhbi_count = cls._get_inst_count(cluster, 'HBASE_INDEXER') - zk_count = cls._get_inst_count(cluster, 'ZOOKEEPER_SERVER') - dn_count = cls._get_inst_count(cluster, 'HDFS_DATANODE') - slr_count = cls._get_inst_count(cluster, 'SOLR_SERVER') - hbm_count = cls._get_inst_count(cluster, 'HBASE_MASTER') - - if lhbi_count >= 1: - if dn_count < 1: - raise ex.RequiredServiceMissingException( - 'HDFS_DATANODE', required_by='HBASE_INDEXER') - if zk_count < 1: - raise ex.RequiredServiceMissingException( - 'ZOOKEEPER', required_by='HBASE_INDEXER') - if slr_count < 1: - raise ex.RequiredServiceMissingException( - 'SOLR_SERVER', required_by='HBASE_INDEXER') - if hbm_count < 1: - raise ex.RequiredServiceMissingException( - 'HBASE_MASTER', required_by='HBASE_INDEXER') - - @classmethod - def _impala_validation(cls, cluster): - ics_count = cls._get_inst_count(cluster, 'IMPALA_CATALOGSERVER') - iss_count = cls._get_inst_count(cluster, 'IMPALA_STATESTORE') - id_count = cls._get_inst_count(cluster, 'IMPALAD') - dn_count = cls._get_inst_count(cluster, 'HDFS_DATANODE') - hms_count = cls._get_inst_count(cluster, 'HIVE_METASTORE') - - if ics_count > 1: - raise ex.InvalidComponentCountException('IMPALA_CATALOGSERVER', - _('0 or 1'), ics_count) - if iss_count > 1: - raise ex.InvalidComponentCountException('IMPALA_STATESTORE', - _('0 or 1'), iss_count) - if ics_count == 1: - datanode_ng = u.get_node_groups(cluster, "HDFS_DATANODE") - impalad_ng = u.get_node_groups(cluster, "IMPALAD") - datanodes = set(ng.id for ng in datanode_ng) - impalads = set(ng.id for ng in impalad_ng) - - if datanodes != impalads: - raise ex.InvalidClusterTopology( - _("IMPALAD must be installed on every HDFS_DATANODE")) - - if iss_count != 1: - raise ex.RequiredServiceMissingException( - 'IMPALA_STATESTORE', required_by='IMPALA') - if id_count < 1: - raise ex.RequiredServiceMissingException( - 'IMPALAD', required_by='IMPALA') - if dn_count < 1: - raise ex.RequiredServiceMissingException( - 'HDFS_DATANODE', required_by='IMPALA') - if hms_count < 1: - raise ex.RequiredServiceMissingException( - 'HIVE_METASTORE', required_by='IMPALA') diff --git a/sahara/plugins/cdh/v5_3_0/versionhandler.py b/sahara/plugins/cdh/v5_3_0/versionhandler.py deleted file mode 100644 index 49c9acdd..00000000 --- a/sahara/plugins/cdh/v5_3_0/versionhandler.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (c) 2014 Mirantis Inc. -# Copyright (c) 2015 ISPRAS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.plugins.cdh import abstractversionhandler as avm -from sahara.plugins.cdh.v5_3_0 import cloudera_utils -from sahara.plugins.cdh.v5_3_0 import config_helper -from sahara.plugins.cdh.v5_3_0 import deploy -from sahara.plugins.cdh.v5_3_0 import edp_engine -from sahara.plugins.cdh.v5_3_0 import plugin_utils -from sahara.plugins.cdh.v5_3_0 import validation - - -class VersionHandler(avm.BaseVersionHandler): - - def __init__(self): - super(VersionHandler, self).__init__() - self.config_helper = config_helper.ConfigHelperV530() - self.cloudera_utils = cloudera_utils.ClouderaUtilsV530() - self.plugin_utils = plugin_utils.PluginUtilsV530() - self.deploy = deploy - self.edp_engine = edp_engine - self.validation = validation.ValidatorV530() - - def get_node_processes(self): - return { - "CLOUDERA": ['CLOUDERA_MANAGER'], - "HDFS": ['HDFS_NAMENODE', 'HDFS_DATANODE', - 'HDFS_SECONDARYNAMENODE'], - "YARN": ['YARN_RESOURCEMANAGER', 'YARN_NODEMANAGER', - 'YARN_JOBHISTORY'], - "OOZIE": ['OOZIE_SERVER'], - "HIVE": ['HIVE_SERVER2', 'HIVE_METASTORE', 'HIVE_WEBHCAT'], - "HUE": ['HUE_SERVER'], - "SPARK_ON_YARN": ['SPARK_YARN_HISTORY_SERVER'], - "ZOOKEEPER": ['ZOOKEEPER_SERVER'], - "HBASE": ['HBASE_MASTER', 'HBASE_REGIONSERVER'], - "FLUME": ['FLUME_AGENT'], - "IMPALA": ['IMPALA_CATALOGSERVER', 'IMPALA_STATESTORE', 'IMPALAD'], - "KS_INDEXER": ['KEY_VALUE_STORE_INDEXER'], - "SOLR": ['SOLR_SERVER'], - "SQOOP": ['SQOOP_SERVER'], - "SENTRY": ['SENTRY_SERVER'], - "YARN_GATEWAY": [], - "HDFS_GATEWAY": [] - } - - def get_edp_engine(self, cluster, job_type): - oozie_type = self.edp_engine.EdpOozieEngine.get_supported_job_types() - spark_type = self.edp_engine.EdpSparkEngine.get_supported_job_types() - if job_type in oozie_type: - return self.edp_engine.EdpOozieEngine(cluster) - if job_type in spark_type: - return self.edp_engine.EdpSparkEngine(cluster) - return None - - def get_edp_job_types(self): - return (self.edp_engine.EdpOozieEngine.get_supported_job_types() + - self.edp_engine.EdpSparkEngine.get_supported_job_types()) diff --git a/sahara/plugins/cdh/v5_4_0/__init__.py b/sahara/plugins/cdh/v5_4_0/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/sahara/plugins/cdh/v5_4_0/cloudera_utils.py b/sahara/plugins/cdh/v5_4_0/cloudera_utils.py deleted file mode 100644 index 53e77709..00000000 --- a/sahara/plugins/cdh/v5_4_0/cloudera_utils.py +++ /dev/null @@ -1,450 +0,0 @@ -# Copyright (c) 2015 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.i18n import _ -from sahara.plugins.cdh import cloudera_utils as cu -from sahara.plugins.cdh.v5_4_0 import config_helper -from sahara.plugins.cdh.v5_4_0 import plugin_utils as pu -from sahara.plugins.cdh.v5_4_0 import validation -from sahara.swift import swift_helper -from sahara.utils import cluster_progress_ops as cpo -from sahara.utils import configs as s_cfg -from sahara.utils import xmlutils - - -HDFS_SERVICE_TYPE = 'HDFS' -YARN_SERVICE_TYPE = 'YARN' -OOZIE_SERVICE_TYPE = 'OOZIE' -HIVE_SERVICE_TYPE = 'HIVE' -HUE_SERVICE_TYPE = 'HUE' -SPARK_SERVICE_TYPE = 'SPARK_ON_YARN' -ZOOKEEPER_SERVICE_TYPE = 'ZOOKEEPER' -HBASE_SERVICE_TYPE = 'HBASE' -FLUME_SERVICE_TYPE = 'FLUME' -SENTRY_SERVICE_TYPE = 'SENTRY' -SOLR_SERVICE_TYPE = 'SOLR' -SQOOP_SERVICE_TYPE = 'SQOOP' -KS_INDEXER_SERVICE_TYPE = 'KS_INDEXER' -IMPALA_SERVICE_TYPE = 'IMPALA' -KMS_SERVICE_TYPE = 'KMS' - -c_helper = config_helper.ConfigHelperV540() - - -class ClouderaUtilsV540(cu.ClouderaUtils): - FLUME_SERVICE_NAME = 'flume01' - SOLR_SERVICE_NAME = 'solr01' - SQOOP_SERVICE_NAME = 'sqoop01' - KS_INDEXER_SERVICE_NAME = 'ks_indexer01' - IMPALA_SERVICE_NAME = 'impala01' - SENTRY_SERVICE_NAME = 'sentry01' - KMS_SERVICE_NAME = 'kms01' - CM_API_VERSION = 8 - NAME_SERVICE = 'nameservice01' - - def __init__(self): - cu.ClouderaUtils.__init__(self) - self.pu = pu.PluginUtilsV540() - self.validator = validation.ValidatorV540 - - def get_service_by_role(self, role, cluster=None, instance=None): - cm_cluster = None - if cluster: - cm_cluster = self.get_cloudera_cluster(cluster) - elif instance: - cm_cluster = self.get_cloudera_cluster(instance.cluster) - else: - raise ValueError(_("'cluster' or 'instance' argument missed")) - - if role in ['AGENT']: - return cm_cluster.get_service(self.FLUME_SERVICE_NAME) - elif role in ['SENTRY_SERVER']: - return cm_cluster.get_service(self.SENTRY_SERVICE_NAME) - elif role in ['SQOOP_SERVER']: - return cm_cluster.get_service(self.SQOOP_SERVICE_NAME) - elif role in ['SOLR_SERVER']: - return cm_cluster.get_service(self.SOLR_SERVICE_NAME) - elif role in ['HBASE_INDEXER']: - return cm_cluster.get_service(self.KS_INDEXER_SERVICE_NAME) - elif role in ['CATALOGSERVER', 'STATESTORE', 'IMPALAD', 'LLAMA']: - return cm_cluster.get_service(self.IMPALA_SERVICE_NAME) - elif role in ['KMS']: - return cm_cluster.get_service(self.KMS_SERVICE_NAME) - elif role in ['JOURNALNODE']: - return cm_cluster.get_service(self.HDFS_SERVICE_NAME) - elif role in ['YARN_STANDBYRM']: - return cm_cluster.get_service(self.YARN_SERVICE_NAME) - else: - return super(ClouderaUtilsV540, self).get_service_by_role( - role, cluster, instance) - - @cpo.event_wrapper( - True, step=_("First run cluster"), param=('cluster', 1)) - @cu.cloudera_cmd - def first_run(self, cluster): - cm_cluster = self.get_cloudera_cluster(cluster) - yield cm_cluster.first_run() - - @cpo.event_wrapper(True, step=_("Create services"), param=('cluster', 1)) - def create_services(self, cluster): - api = self.get_api_client(cluster) - cm_cluster = api.create_cluster(cluster.name, - fullVersion=cluster.hadoop_version) - - if len(self.pu.get_zookeepers(cluster)) > 0: - cm_cluster.create_service(self.ZOOKEEPER_SERVICE_NAME, - ZOOKEEPER_SERVICE_TYPE) - cm_cluster.create_service(self.HDFS_SERVICE_NAME, HDFS_SERVICE_TYPE) - cm_cluster.create_service(self.YARN_SERVICE_NAME, YARN_SERVICE_TYPE) - cm_cluster.create_service(self.OOZIE_SERVICE_NAME, OOZIE_SERVICE_TYPE) - if self.pu.get_hive_metastore(cluster): - cm_cluster.create_service(self.HIVE_SERVICE_NAME, - HIVE_SERVICE_TYPE) - if self.pu.get_hue(cluster): - cm_cluster.create_service(self.HUE_SERVICE_NAME, HUE_SERVICE_TYPE) - if self.pu.get_spark_historyserver(cluster): - cm_cluster.create_service(self.SPARK_SERVICE_NAME, - SPARK_SERVICE_TYPE) - if self.pu.get_hbase_master(cluster): - cm_cluster.create_service(self.HBASE_SERVICE_NAME, - HBASE_SERVICE_TYPE) - if len(self.pu.get_flumes(cluster)) > 0: - cm_cluster.create_service(self.FLUME_SERVICE_NAME, - FLUME_SERVICE_TYPE) - if self.pu.get_sentry(cluster): - cm_cluster.create_service(self.SENTRY_SERVICE_NAME, - SENTRY_SERVICE_TYPE) - if len(self.pu.get_solrs(cluster)) > 0: - cm_cluster.create_service(self.SOLR_SERVICE_NAME, - SOLR_SERVICE_TYPE) - if self.pu.get_sqoop(cluster): - cm_cluster.create_service(self.SQOOP_SERVICE_NAME, - SQOOP_SERVICE_TYPE) - if len(self.pu.get_hbase_indexers(cluster)) > 0: - cm_cluster.create_service(self.KS_INDEXER_SERVICE_NAME, - KS_INDEXER_SERVICE_TYPE) - if self.pu.get_catalogserver(cluster): - cm_cluster.create_service(self.IMPALA_SERVICE_NAME, - IMPALA_SERVICE_TYPE) - if self.pu.get_kms(cluster): - cm_cluster.create_service(self.KMS_SERVICE_NAME, - KMS_SERVICE_TYPE) - - def await_agents(self, cluster, instances): - self._await_agents(cluster, instances, c_helper.AWAIT_AGENTS_TIMEOUT) - - @cpo.event_wrapper( - True, step=_("Configure services"), param=('cluster', 1)) - def configure_services(self, cluster): - cm_cluster = self.get_cloudera_cluster(cluster) - - if len(self.pu.get_zookeepers(cluster)) > 0: - zookeeper = cm_cluster.get_service(self.ZOOKEEPER_SERVICE_NAME) - zookeeper.update_config(self._get_configs(ZOOKEEPER_SERVICE_TYPE, - cluster=cluster)) - - hdfs = cm_cluster.get_service(self.HDFS_SERVICE_NAME) - hdfs.update_config(self._get_configs(HDFS_SERVICE_TYPE, - cluster=cluster)) - - yarn = cm_cluster.get_service(self.YARN_SERVICE_NAME) - yarn.update_config(self._get_configs(YARN_SERVICE_TYPE, - cluster=cluster)) - - oozie = cm_cluster.get_service(self.OOZIE_SERVICE_NAME) - oozie.update_config(self._get_configs(OOZIE_SERVICE_TYPE, - cluster=cluster)) - - if self.pu.get_hive_metastore(cluster): - hive = cm_cluster.get_service(self.HIVE_SERVICE_NAME) - hive.update_config(self._get_configs(HIVE_SERVICE_TYPE, - cluster=cluster)) - - if self.pu.get_hue(cluster): - hue = cm_cluster.get_service(self.HUE_SERVICE_NAME) - hue.update_config(self._get_configs(HUE_SERVICE_TYPE, - cluster=cluster)) - - if self.pu.get_spark_historyserver(cluster): - spark = cm_cluster.get_service(self.SPARK_SERVICE_NAME) - spark.update_config(self._get_configs(SPARK_SERVICE_TYPE, - cluster=cluster)) - - if self.pu.get_hbase_master(cluster): - hbase = cm_cluster.get_service(self.HBASE_SERVICE_NAME) - hbase.update_config(self._get_configs(HBASE_SERVICE_TYPE, - cluster=cluster)) - - if len(self.pu.get_flumes(cluster)) > 0: - flume = cm_cluster.get_service(self.FLUME_SERVICE_NAME) - flume.update_config(self._get_configs(FLUME_SERVICE_TYPE, - cluster=cluster)) - - if self.pu.get_sentry(cluster): - sentry = cm_cluster.get_service(self.SENTRY_SERVICE_NAME) - sentry.update_config(self._get_configs(SENTRY_SERVICE_TYPE, - cluster=cluster)) - - if len(self.pu.get_solrs(cluster)) > 0: - solr = cm_cluster.get_service(self.SOLR_SERVICE_NAME) - solr.update_config(self._get_configs(SOLR_SERVICE_TYPE, - cluster=cluster)) - - if self.pu.get_sqoop(cluster): - sqoop = cm_cluster.get_service(self.SQOOP_SERVICE_NAME) - sqoop.update_config(self._get_configs(SQOOP_SERVICE_TYPE, - cluster=cluster)) - - if len(self.pu.get_hbase_indexers(cluster)) > 0: - ks_indexer = cm_cluster.get_service(self.KS_INDEXER_SERVICE_NAME) - ks_indexer.update_config( - self._get_configs(KS_INDEXER_SERVICE_TYPE, cluster=cluster)) - - if self.pu.get_catalogserver(cluster): - impala = cm_cluster.get_service(self.IMPALA_SERVICE_NAME) - impala.update_config(self._get_configs(IMPALA_SERVICE_TYPE, - cluster=cluster)) - - if self.pu.get_kms(cluster): - kms = cm_cluster.get_service(self.KMS_SERVICE_NAME) - kms.update_config(self._get_configs(KMS_SERVICE_TYPE, - cluster=cluster)) - - def _get_configs(self, service, cluster=None, instance=None): - def get_hadoop_dirs(mount_points, suffix): - return ','.join([x + suffix for x in mount_points]) - - all_confs = {} - if cluster: - zk_count = self.validator._get_inst_count(cluster, - 'ZOOKEEPER_SERVER') - hbm_count = self.validator._get_inst_count(cluster, 'HBASE_MASTER') - snt_count = self.validator._get_inst_count(cluster, - 'SENTRY_SERVER') - ks_count =\ - self.validator._get_inst_count(cluster, - 'KEY_VALUE_STORE_INDEXER') - kms_count = self.validator._get_inst_count(cluster, 'KMS') - imp_count =\ - self.validator._get_inst_count(cluster, - 'IMPALA_CATALOGSERVER') - hive_count = self.validator._get_inst_count(cluster, - 'HIVE_METASTORE') - slr_count = self.validator._get_inst_count(cluster, 'SOLR_SERVER') - sqp_count = self.validator._get_inst_count(cluster, 'SQOOP_SERVER') - core_site_safety_valve = '' - if self.pu.c_helper.is_swift_enabled(cluster): - configs = swift_helper.get_swift_configs() - confs = {c['name']: c['value'] for c in configs} - core_site_safety_valve = xmlutils.create_elements_xml(confs) - all_confs = { - 'HDFS': { - 'zookeeper_service': - self.ZOOKEEPER_SERVICE_NAME if zk_count else '', - 'dfs_block_local_path_access_user': - 'impala' if imp_count else '', - 'kms_service': self.KMS_SERVICE_NAME if kms_count else '', - 'core_site_safety_valve': core_site_safety_valve - }, - 'HIVE': { - 'mapreduce_yarn_service': self.YARN_SERVICE_NAME, - 'sentry_service': - self.SENTRY_SERVICE_NAME if snt_count else '', - 'zookeeper_service': - self.ZOOKEEPER_SERVICE_NAME if zk_count else '' - }, - 'OOZIE': { - 'mapreduce_yarn_service': self.YARN_SERVICE_NAME, - 'hive_service': - self.HIVE_SERVICE_NAME if hive_count else '', - 'zookeeper_service': - self.ZOOKEEPER_SERVICE_NAME if zk_count else '' - }, - 'YARN': { - 'hdfs_service': self.HDFS_SERVICE_NAME, - 'zookeeper_service': - self.ZOOKEEPER_SERVICE_NAME if zk_count else '' - }, - 'HUE': { - 'hive_service': self.HIVE_SERVICE_NAME, - 'oozie_service': self.OOZIE_SERVICE_NAME, - 'sentry_service': - self.SENTRY_SERVICE_NAME if snt_count else '', - 'solr_service': - self.SOLR_SERVICE_NAME if slr_count else '', - 'zookeeper_service': - self.ZOOKEEPER_SERVICE_NAME if zk_count else '', - 'hbase_service': - self.HBASE_SERVICE_NAME if hbm_count else '', - 'impala_service': - self.IMPALA_SERVICE_NAME if imp_count else '', - 'sqoop_service': - self.SQOOP_SERVICE_NAME if sqp_count else '' - }, - 'SPARK_ON_YARN': { - 'yarn_service': self.YARN_SERVICE_NAME - }, - 'HBASE': { - 'hdfs_service': self.HDFS_SERVICE_NAME, - 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME, - 'hbase_enable_indexing': 'true' if ks_count else 'false', - 'hbase_enable_replication': - 'true' if ks_count else 'false' - }, - 'FLUME': { - 'hdfs_service': self.HDFS_SERVICE_NAME, - 'solr_service': - self.SOLR_SERVICE_NAME if slr_count else '', - 'hbase_service': - self.HBASE_SERVICE_NAME if hbm_count else '' - }, - 'SENTRY': { - 'hdfs_service': self.HDFS_SERVICE_NAME, - 'sentry_server_config_safety_valve': ( - c_helper.SENTRY_IMPALA_CLIENT_SAFETY_VALVE - if imp_count else '') - }, - 'SOLR': { - 'hdfs_service': self.HDFS_SERVICE_NAME, - 'zookeeper_service': self.ZOOKEEPER_SERVICE_NAME - }, - 'SQOOP': { - 'mapreduce_yarn_service': self.YARN_SERVICE_NAME - }, - 'KS_INDEXER': { - 'hbase_service': self.HBASE_SERVICE_NAME, - 'solr_service': self.SOLR_SERVICE_NAME - }, - 'IMPALA': { - 'hdfs_service': self.HDFS_SERVICE_NAME, - 'hbase_service': - self.HBASE_SERVICE_NAME if hbm_count else '', - 'hive_service': self.HIVE_SERVICE_NAME, - 'sentry_service': - self.SENTRY_SERVICE_NAME if snt_count else '', - 'zookeeper_service': - self.ZOOKEEPER_SERVICE_NAME if zk_count else '' - } - } - hive_confs = { - 'HIVE': { - 'hive_metastore_database_type': 'postgresql', - 'hive_metastore_database_host': - self.pu.get_manager(cluster).internal_ip, - 'hive_metastore_database_port': '7432', - 'hive_metastore_database_password': - self.pu.db_helper.get_hive_db_password(cluster) - } - } - hue_confs = { - 'HUE': { - 'hue_webhdfs': self.pu.get_role_name( - self.pu.get_namenode(cluster), 'NAMENODE') - } - } - sentry_confs = { - 'SENTRY': { - 'sentry_server_database_type': 'postgresql', - 'sentry_server_database_host': - self.pu.get_manager(cluster).internal_ip, - 'sentry_server_database_port': '7432', - 'sentry_server_database_password': - self.pu.db_helper.get_sentry_db_password(cluster) - } - } - - all_confs = s_cfg.merge_configs(all_confs, hue_confs) - all_confs = s_cfg.merge_configs(all_confs, hive_confs) - all_confs = s_cfg.merge_configs(all_confs, sentry_confs) - all_confs = s_cfg.merge_configs(all_confs, cluster.cluster_configs) - - if instance: - snt_count = self.validator._get_inst_count(instance.cluster, - 'SENTRY_SERVER') - paths = instance.storage_paths() - - instance_default_confs = { - 'NAMENODE': { - 'dfs_name_dir_list': get_hadoop_dirs(paths, '/fs/nn') - }, - 'SECONDARYNAMENODE': { - 'fs_checkpoint_dir_list': - get_hadoop_dirs(paths, '/fs/snn') - }, - 'DATANODE': { - 'dfs_data_dir_list': get_hadoop_dirs(paths, '/fs/dn'), - 'dfs_datanode_data_dir_perm': 755, - 'dfs_datanode_handler_count': 30 - }, - 'NODEMANAGER': { - 'yarn_nodemanager_local_dirs': - get_hadoop_dirs(paths, '/yarn/local') - }, - 'SERVER': { - 'maxSessionTimeout': 60000 - }, - 'HIVESERVER2': { - 'hiveserver2_enable_impersonation': - 'false' if snt_count else 'true', - 'hive_hs2_config_safety_valve': ( - c_helper.HIVE_SERVER2_SENTRY_SAFETY_VALVE - if snt_count else '') - }, - 'HIVEMETASTORE': { - 'hive_metastore_config_safety_valve': ( - c_helper.HIVE_METASTORE_SENTRY_SAFETY_VALVE - if snt_count else '') - } - } - - ng_user_confs = self.pu.convert_process_configs( - instance.node_group.node_configs) - all_confs = s_cfg.merge_configs(all_confs, ng_user_confs) - all_confs = s_cfg.merge_configs(all_confs, instance_default_confs) - - return all_confs.get(service, {}) - - @cpo.event_wrapper( - True, step=_("Enable NameNode HA"), param=('cluster', 1)) - @cu.cloudera_cmd - def enable_namenode_ha(self, cluster): - standby_nn = self.pu.get_secondarynamenode(cluster) - standby_nn_host_name = standby_nn.fqdn() - jns = self.pu.get_jns(cluster) - jn_list = [] - for index, jn in enumerate(jns): - jn_host_name = jn.fqdn() - jn_list.append({'jnHostId': jn_host_name, - 'jnName': 'JN%i' % index, - 'jnEditsDir': '/dfs/jn' - }) - cm_cluster = self.get_cloudera_cluster(cluster) - hdfs = cm_cluster.get_service(self.HDFS_SERVICE_NAME) - nn = hdfs.get_roles_by_type('NAMENODE')[0] - - yield hdfs.enable_nn_ha(active_name=nn.name, - standby_host_id=standby_nn_host_name, - nameservice=self.NAME_SERVICE, jns=jn_list - ) - - @cpo.event_wrapper( - True, step=_("Enable ResourceManager HA"), param=('cluster', 1)) - @cu.cloudera_cmd - def enable_resourcemanager_ha(self, cluster): - new_rm = self.pu.get_stdb_rm(cluster) - new_rm_host_name = new_rm.fqdn() - cm_cluster = self.get_cloudera_cluster(cluster) - yarn = cm_cluster.get_service(self.YARN_SERVICE_NAME) - yield yarn.enable_rm_ha(new_rm_host_id=new_rm_host_name) diff --git a/sahara/plugins/cdh/v5_4_0/config_helper.py b/sahara/plugins/cdh/v5_4_0/config_helper.py deleted file mode 100644 index 8111e1dd..00000000 --- a/sahara/plugins/cdh/v5_4_0/config_helper.py +++ /dev/null @@ -1,243 +0,0 @@ -# Copyright (c) 2014 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.plugins.cdh import config_helper as c_h -from sahara.plugins import provisioning as p -from sahara.utils import files as f - - -class ConfigHelperV540(c_h.ConfigHelper): - path_to_config = 'plugins/cdh/v5_4_0/resources/' - - CDH5_UBUNTU_REPO = ( - 'deb [arch=amd64] http://archive.cloudera.com/cdh5' - '/ubuntu/precise/amd64/cdh precise-cdh5.4.0 contrib' - '\ndeb-src http://archive.cloudera.com/cdh5/ubuntu' - '/precise/amd64/cdh precise-cdh5.4.0 contrib') - - DEFAULT_CDH5_UBUNTU_REPO_KEY_URL = ( - 'http://archive.cloudera.com/cdh5/ubuntu' - '/precise/amd64/cdh/archive.key') - - CM5_UBUNTU_REPO = ( - 'deb [arch=amd64] http://archive.cloudera.com/cm5' - '/ubuntu/precise/amd64/cm precise-cm5.4.0 contrib' - '\ndeb-src http://archive.cloudera.com/cm5/ubuntu' - '/precise/amd64/cm precise-cm5.4.0 contrib') - - DEFAULT_CM5_UBUNTU_REPO_KEY_URL = ( - 'http://archive.cloudera.com/cm5/ubuntu' - '/precise/amd64/cm/archive.key') - - CDH5_CENTOS_REPO = ( - '[cloudera-cdh5]' - '\nname=Cloudera\'s Distribution for Hadoop, Version 5' - '\nbaseurl=http://archive.cloudera.com/cdh5/redhat/6' - '/x86_64/cdh/5.4.0/' - '\ngpgkey = http://archive.cloudera.com/cdh5/redhat/6' - '/x86_64/cdh/RPM-GPG-KEY-cloudera' - '\ngpgcheck = 1') - - CM5_CENTOS_REPO = ( - '[cloudera-manager]' - '\nname=Cloudera Manager' - '\nbaseurl=http://archive.cloudera.com/cm5/redhat/6' - '/x86_64/cm/5.4.0/' - '\ngpgkey = http://archive.cloudera.com/cm5/redhat/6' - '/x86_64/cm/RPM-GPG-KEY-cloudera' - '\ngpgcheck = 1') - - KEY_TRUSTEE_UBUNTU_REPO_URL = ( - 'http://archive.cloudera.com/navigator-' - 'keytrustee5/ubuntu/precise/amd64/navigator-' - 'keytrustee/cloudera.list') - - DEFAULT_KEY_TRUSTEE_UBUNTU_REPO_KEY_URL = ( - 'http://archive.cloudera.com/' - 'navigator-keytrustee5/ubuntu/' - 'precise/amd64/navigator-keytrustee' - '/archive.key') - - KEY_TRUSTEE_CENTOS_REPO_URL = ( - 'http://archive.cloudera.com/navigator-' - 'keytrustee5/redhat/6/x86_64/navigator-' - 'keytrustee/navigator-keytrustee5.repo') - - DEFAULT_SWIFT_LIB_URL = ( - 'https://repository.cloudera.com/artifactory/repo/org' - '/apache/hadoop/hadoop-openstack/2.6.0-cdh5.4.0' - '/hadoop-openstack-2.6.0-cdh5.4.0.jar') - - HIVE_SERVER2_SENTRY_SAFETY_VALVE = f.get_file_text( - path_to_config + 'hive-server2-sentry-safety.xml') - - HIVE_METASTORE_SENTRY_SAFETY_VALVE = f.get_file_text( - path_to_config + 'hive-metastore-sentry-safety.xml') - - SENTRY_IMPALA_CLIENT_SAFETY_VALVE = f.get_file_text( - path_to_config + 'sentry-impala-client-safety.xml') - - _default_executor_classpath = ":".join( - ['/usr/lib/hadoop/lib/jackson-core-asl-1.8.8.jar', - '/usr/lib/hadoop-mapreduce/hadoop-openstack.jar']) - - EXECUTOR_EXTRA_CLASSPATH = p.Config( - 'Executor extra classpath', 'Spark', 'cluster', priority=2, - default_value=_default_executor_classpath, - description='Value for spark.executor.extraClassPath in ' - 'spark-defaults.conf (default: %s)' - % _default_executor_classpath) - - SWIFT_LIB_URL = p.Config( - 'Hadoop OpenStack library URL', 'general', 'cluster', priority=1, - default_value=DEFAULT_SWIFT_LIB_URL, - description=("Library that adds Swift support to CDH. The file" - " will be downloaded by VMs.")) - - KMS_REPO_URL = p.Config( - 'KMS repo list URL', 'general', 'cluster', priority=1, - default_value="") - - KMS_REPO_KEY_URL = p.Config( - 'KMS repo key URL (for debian-based only)', 'general', - 'cluster', - priority=1, default_value="") - - REQUIRE_ANTI_AFFINITY = p.Config('Require Anti Affinity', - 'general', 'cluster', - config_type='bool', - priority=2, - default_value=True) - - def __init__(self): - super(ConfigHelperV540, self).__init__() - self.priority_one_confs = self._load_json( - self.path_to_config + 'priority-one-confs.json') - self._init_all_ng_plugin_configs() - - def _get_cluster_plugin_configs(self): - confs = super(ConfigHelperV540, self)._get_ng_plugin_configs() - confs += [self.EXECUTOR_EXTRA_CLASSPATH, - self.KMS_REPO_URL, - self.KMS_REPO_KEY_URL, - self.REQUIRE_ANTI_AFFINITY] - - return confs - - def _init_all_ng_plugin_configs(self): - self.hdfs_confs = self._load_and_init_configs( - 'hdfs-service.json', 'HDFS', 'cluster') - self.namenode_confs = self._load_and_init_configs( - 'hdfs-namenode.json', 'NAMENODE', 'node') - self.datanode_confs = self._load_and_init_configs( - 'hdfs-datanode.json', 'DATANODE', 'node') - self.secnamenode_confs = self._load_and_init_configs( - 'hdfs-secondarynamenode.json', 'SECONDARYNAMENODE', 'node') - self.hdfs_gateway_confs = self._load_and_init_configs( - 'hdfs-gateway.json', 'HDFS_GATEWAY', 'node') - self.journalnode_confs = self._load_and_init_configs( - 'hdfs-journalnode.json', 'JOURNALNODE', 'node') - - self.yarn_confs = self._load_and_init_configs( - 'yarn-service.json', 'YARN', 'cluster') - self.resourcemanager_confs = self._load_and_init_configs( - 'yarn-resourcemanager.json', 'RESOURCEMANAGER', 'node') - self.nodemanager_confs = self._load_and_init_configs( - 'yarn-nodemanager.json', 'NODEMANAGER', 'node') - self.jobhistory_confs = self._load_and_init_configs( - 'yarn-jobhistory.json', 'JOBHISTORY', 'node') - self.yarn_gateway_conf = self._load_and_init_configs( - 'yarn-gateway.json', 'YARN_GATEWAY', 'node') - - self.oozie_service_confs = self._load_and_init_configs( - 'oozie-service.json', 'OOZIE', 'cluster') - self.oozie_role_confs = self._load_and_init_configs( - 'oozie-oozie_server.json', 'OOZIE', 'node') - - self.hive_service_confs = self._load_and_init_configs( - 'hive-service.json', 'HIVE', 'cluster') - self.hive_metastore_confs = self._load_and_init_configs( - 'hive-hivemetastore.json', 'HIVEMETASTORE', 'node') - self.hive_hiveserver_confs = self._load_and_init_configs( - 'hive-hiveserver2.json', 'HIVESERVER', 'node') - self.hive_webhcat_confs = self._load_and_init_configs( - 'hive-webhcat.json', 'WEBHCAT', 'node') - - self.hue_service_confs = self._load_and_init_configs( - 'hue-service.json', 'HUE', 'cluster') - self.hue_role_confs = self._load_and_init_configs( - 'hue-hue_server.json', 'HUE', 'node') - - self.spark_service_confs = self._load_and_init_configs( - 'spark-service.json', 'SPARK_ON_YARN', 'cluster') - self.spark_role_confs = self._load_and_init_configs( - 'spark-spark_yarn_history_server.json', 'SPARK_ON_YARN', 'node') - - self.zookeeper_server_confs = self._load_and_init_configs( - 'zookeeper-service.json', 'ZOOKEEPER', 'cluster') - self.zookeeper_service_confs = self._load_and_init_configs( - 'zookeeper-server.json', 'ZOOKEEPER', 'node') - - self.hbase_confs = self._load_and_init_configs( - 'hbase-service.json', 'HBASE', 'cluster') - self.master_confs = self._load_and_init_configs( - 'hbase-master.json', 'MASTER', 'node') - self.regionserver_confs = self._load_and_init_configs( - 'hbase-regionserver.json', 'REGIONSERVER', 'node') - - self.flume_service_confs = self._load_and_init_configs( - 'flume-service.json', 'FLUME', 'cluster') - self.flume_agent_confs = self._load_and_init_configs( - 'flume-agent.json', 'FLUME', 'node') - - self.sentry_service_confs = self._load_and_init_configs( - 'sentry-service.json', 'SENTRY', 'cluster') - self.sentry_server_confs = self._load_and_init_configs( - 'sentry-sentry_server.json', 'SENTRY', 'node') - - self.solr_service_confs = self._load_and_init_configs( - 'solr-service.json', 'SOLR', 'cluster') - self.solr_server_confs = self._load_and_init_configs( - 'solr-solr_server.json', 'SOLR', 'node') - - self.sqoop_service_confs = self._load_and_init_configs( - 'sqoop-service.json', 'SQOOP', 'cluster') - self.sqoop_server_confs = self._load_and_init_configs( - 'sqoop-sqoop_server.json', 'SQOOP', 'node') - - self.ks_indexer_service_confs = self._load_and_init_configs( - 'ks_indexer-service.json', 'KS_INDEXER', 'cluster') - self.ks_indexer_role_confs = self._load_and_init_configs( - 'ks_indexer-hbase_indexer.json', 'KS_INDEXER', 'node') - - self.impala_service_confs = self._load_and_init_configs( - 'impala-service.json', 'IMPALA', 'cluster') - self.impala_catalogserver_confs = self._load_and_init_configs( - 'impala-catalogserver.json', 'CATALOGSERVER', 'node') - self.impala_impalad_confs = self._load_and_init_configs( - 'impala-impalad.json', 'IMPALAD', 'node') - self.impala_statestore_confs = self._load_and_init_configs( - 'impala-statestore.json', 'STATESTORE', 'node') - - self.kms_service_confs = self._load_and_init_configs( - 'kms-service.json', 'KMS', 'cluster') - self.kms_kms_confs = self._load_and_init_configs( - 'kms-kms.json', 'KMS', 'node') - - def get_required_anti_affinity(self, cluster): - return self._get_config_value(cluster, self.REQUIRE_ANTI_AFFINITY) - - def get_kms_key_url(self, cluster): - return self._get_config_value(cluster, self.KMS_REPO_KEY_URL) diff --git a/sahara/plugins/cdh/v5_4_0/db_helper.py b/sahara/plugins/cdh/v5_4_0/db_helper.py deleted file mode 100644 index 8a2511ff..00000000 --- a/sahara/plugins/cdh/v5_4_0/db_helper.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2015 Intel Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.plugins.cdh import db_helper as dh -from sahara.utils import files - - -def get_hive_db_password(cluster): - return dh.get_password_from_db(cluster, dh.HIVE_DB_PASSWORD) - - -def get_sentry_db_password(cluster): - return dh.get_password_from_db(cluster, dh.SENTRY_DB_PASSWORD) - - -def create_hive_database(cluster, remote): - db_password = get_hive_db_password(cluster) - create_db_script = files.get_file_text( - 'plugins/cdh/v5_4_0/resources/create_hive_db.sql') - create_db_script = create_db_script % db_password - dh.remote_execute_db_script(remote, create_db_script) - - -def create_sentry_database(cluster, remote): - db_password = get_sentry_db_password(cluster) - create_db_script = files.get_file_text( - 'plugins/cdh/v5_4_0/resources/create_sentry_db.sql') - create_db_script = create_db_script % db_password - dh.remote_execute_db_script(remote, create_db_script) diff --git a/sahara/plugins/cdh/v5_4_0/deploy.py b/sahara/plugins/cdh/v5_4_0/deploy.py deleted file mode 100644 index e951fefa..00000000 --- a/sahara/plugins/cdh/v5_4_0/deploy.py +++ /dev/null @@ -1,232 +0,0 @@ -# Copyright (c) 2015 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.i18n import _ -from sahara.plugins.cdh import commands as cmd -from sahara.plugins.cdh.v5_4_0 import cloudera_utils as cu -from sahara.plugins import utils as gu -from sahara.service.edp import hdfs_helper as h -from sahara.utils import cluster_progress_ops as cpo - -PACKAGES = [ - 'cloudera-manager-agent', - 'cloudera-manager-daemons', - 'cloudera-manager-server', - 'cloudera-manager-server-db-2', - 'flume-ng', - 'hadoop-hdfs-datanode', - 'hadoop-hdfs-namenode', - 'hadoop-hdfs-secondarynamenode', - 'hadoop-kms' - 'hadoop-mapreduce', - 'hadoop-mapreduce-historyserver', - 'hadoop-yarn-nodemanager', - 'hadoop-yarn-resourcemanager', - 'hbase', - 'hbase-solr', - 'hive-hcatalog', - 'hive-metastore', - 'hive-server2', - 'hive-webhcat-server', - 'hue', - 'impala', - 'impala-server', - 'impala-state-store', - 'impala-catalog', - 'impala-shell', - 'keytrustee-keyprovider', - 'oozie', - 'oracle-j2sdk1.7', - 'sentry', - 'solr-server', - 'solr-doc', - 'search', - 'spark-history-server', - 'sqoop2', - 'unzip', - 'zookeeper' -] - -CU = cu.ClouderaUtilsV540() - - -def configure_cluster(cluster): - instances = gu.get_instances(cluster) - - if not cmd.is_pre_installed_cdh(CU.pu.get_manager(cluster).remote()): - CU.pu.configure_os(instances) - CU.pu.install_packages(instances, PACKAGES) - - CU.pu.start_cloudera_agents(instances) - CU.pu.start_cloudera_manager(cluster) - CU.update_cloudera_password(cluster) - CU.await_agents(cluster, instances) - CU.create_mgmt_service(cluster) - CU.create_services(cluster) - CU.configure_services(cluster) - CU.configure_instances(instances, cluster) - CU.deploy_configs(cluster) - - -@cpo.event_wrapper( - True, step=_("Start roles: NODEMANAGER, DATANODE"), param=('cluster', 0)) -def _start_roles(cluster, instances): - for instance in instances: - if 'HDFS_DATANODE' in instance.node_group.node_processes: - hdfs = CU.get_service_by_role('DATANODE', instance=instance) - CU.start_roles(hdfs, CU.pu.get_role_name(instance, 'DATANODE')) - - if 'YARN_NODEMANAGER' in instance.node_group.node_processes: - yarn = CU.get_service_by_role('NODEMANAGER', instance=instance) - CU.start_roles(yarn, CU.pu.get_role_name(instance, 'NODEMANAGER')) - - -def scale_cluster(cluster, instances): - if not instances: - return - - if not cmd.is_pre_installed_cdh(instances[0].remote()): - CU.pu.configure_os(instances) - CU.pu.install_packages(instances, PACKAGES) - - CU.pu.start_cloudera_agents(instances) - CU.await_agents(cluster, instances) - CU.configure_instances(instances, cluster) - CU.update_configs(instances) - CU.pu.configure_swift(cluster, instances) - _start_roles(cluster, instances) - CU.refresh_datanodes(cluster) - CU.refresh_yarn_nodes(cluster) - CU.restart_stale_services(cluster) - - -def decommission_cluster(cluster, instances): - dns = [] - dns_to_delete = [] - nms = [] - nms_to_delete = [] - for i in instances: - if 'HDFS_DATANODE' in i.node_group.node_processes: - dns.append(CU.pu.get_role_name(i, 'DATANODE')) - dns_to_delete.append( - CU.pu.get_role_name(i, 'HDFS_GATEWAY')) - - if 'YARN_NODEMANAGER' in i.node_group.node_processes: - nms.append(CU.pu.get_role_name(i, 'NODEMANAGER')) - nms_to_delete.append( - CU.pu.get_role_name(i, 'YARN_GATEWAY')) - - if dns: - CU.decommission_nodes( - cluster, 'DATANODE', dns, dns_to_delete) - - if nms: - CU.decommission_nodes( - cluster, 'NODEMANAGER', nms, nms_to_delete) - - CU.delete_instances(cluster, instances) - - CU.refresh_datanodes(cluster) - CU.refresh_yarn_nodes(cluster) - CU.restart_stale_services(cluster) - - -@cpo.event_wrapper(True, step=_("Prepare cluster"), param=('cluster', 0)) -def _prepare_cluster(cluster): - if CU.pu.get_oozie(cluster): - CU.pu.install_extjs(cluster) - - if CU.pu.get_hive_metastore(cluster): - CU.pu.configure_hive(cluster) - - if CU.pu.get_sentry(cluster): - CU.pu.configure_sentry(cluster) - - -@cpo.event_wrapper( - True, step=_("Finish cluster starting"), param=('cluster', 0)) -def _finish_cluster_starting(cluster): - if CU.pu.get_hive_metastore(cluster): - CU.pu.put_hive_hdfs_xml(cluster) - - server = CU.pu.get_hbase_master(cluster) - if CU.pu.c_helper.is_hbase_common_lib_enabled(cluster) and server: - with server.remote() as r: - h.create_hbase_common_lib(r) - - if CU.pu.get_flumes(cluster): - flume = CU.get_service_by_role('AGENT', cluster) - CU.start_service(flume) - - -def start_cluster(cluster): - _prepare_cluster(cluster) - - CU.first_run(cluster) - - CU.pu.configure_swift(cluster) - - if len(CU.pu.get_jns(cluster)) > 0: - CU.enable_namenode_ha(cluster) - # updating configs for NameNode role on needed nodes - CU.update_role_config(CU.pu.get_secondarynamenode(cluster), - 'HDFS_NAMENODE') - - if CU.pu.get_stdb_rm(cluster): - CU.enable_resourcemanager_ha(cluster) - # updating configs for ResourceManager role on needed nodes - CU.update_role_config(CU.pu.get_stdb_rm(cluster), 'YARN_STANDBYRM') - - _finish_cluster_starting(cluster) - - -def get_open_ports(node_group): - ports = [9000] # for CM agent - - ports_map = { - 'CLOUDERA_MANAGER': [7180, 7182, 7183, 7432, 7184, 8084, 8086, 10101, - 9997, 9996, 8087, 9998, 9999, 8085, 9995, 9994], - 'HDFS_NAMENODE': [8020, 8022, 50070, 50470], - 'HDFS_SECONDARYNAMENODE': [50090, 50495], - 'HDFS_DATANODE': [50010, 1004, 50075, 1006, 50020], - 'YARN_RESOURCEMANAGER': [8030, 8031, 8032, 8033, 8088], - 'YARN_STANDBYRM': [8030, 8031, 8032, 8033, 8088], - 'YARN_NODEMANAGER': [8040, 8041, 8042], - 'YARN_JOBHISTORY': [10020, 19888], - 'HIVE_METASTORE': [9083], - 'HIVE_SERVER2': [10000], - 'HUE_SERVER': [8888], - 'OOZIE_SERVER': [11000, 11001], - 'SPARK_YARN_HISTORY_SERVER': [18088], - 'ZOOKEEPER_SERVER': [2181, 3181, 4181, 9010], - 'HBASE_MASTER': [60000], - 'HBASE_REGIONSERVER': [60020], - 'FLUME_AGENT': [41414], - 'SENTRY_SERVER': [8038], - 'SOLR_SERVER': [8983, 8984], - 'SQOOP_SERVER': [8005, 12000], - 'KEY_VALUE_STORE_INDEXER': [], - 'IMPALA_CATALOGSERVER': [25020, 26000], - 'IMPALA_STATESTORE': [25010, 24000], - 'IMPALAD': [21050, 21000, 23000, 25000, 28000, 22000], - 'KMS': [16000, 16001], - 'JOURNALNODE': [8480, 8481, 8485] - } - - for process in node_group.node_processes: - if process in ports_map: - ports.extend(ports_map[process]) - - return ports diff --git a/sahara/plugins/cdh/v5_4_0/edp_engine.py b/sahara/plugins/cdh/v5_4_0/edp_engine.py deleted file mode 100644 index 6dc7676f..00000000 --- a/sahara/plugins/cdh/v5_4_0/edp_engine.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) 2015 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.plugins.cdh import confighints_helper as ch_helper -from sahara.plugins.cdh import edp_engine -from sahara.plugins.cdh.v5_4_0 import cloudera_utils as cu -from sahara.service.edp.oozie import engine as oozie_engine -from sahara.utils import edp - - -class EdpOozieEngine(edp_engine.EdpOozieEngine): - - def __init__(self, cluster): - super(EdpOozieEngine, self).__init__(cluster) - self.cloudera_utils = cu.ClouderaUtilsV540() - - def get_name_node_uri(self, cluster): - if len(self.cloudera_utils.pu.get_jns(cluster)) > 0: - return 'hdfs://%s' % self.cloudera_utils.NAME_SERVICE - else: - namenode_ip = self.cloudera_utils.pu.get_namenode(cluster).fqdn() - return 'hdfs://%s:8020' % namenode_ip - - @staticmethod - def get_possible_job_config(job_type): - if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE): - return {'job_config': ch_helper.get_possible_hive_config_from( - 'plugins/cdh/v5_4_0/resources/hive-site.xml')} - if edp.compare_job_type(job_type, - edp.JOB_TYPE_MAPREDUCE, - edp.JOB_TYPE_MAPREDUCE_STREAMING): - return {'job_config': ch_helper.get_possible_mapreduce_config_from( - 'plugins/cdh/v5_4_0/resources/mapred-site.xml')} - if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG): - return {'job_config': ch_helper.get_possible_pig_config_from( - 'plugins/cdh/v5_4_0/resources/mapred-site.xml')} - return oozie_engine.OozieJobEngine.get_possible_job_config(job_type) - - -class EdpSparkEngine(edp_engine.EdpSparkEngine): - - edp_base_version = "5.4.0" diff --git a/sahara/plugins/cdh/v5_4_0/plugin_utils.py b/sahara/plugins/cdh/v5_4_0/plugin_utils.py deleted file mode 100644 index 7e6e913e..00000000 --- a/sahara/plugins/cdh/v5_4_0/plugin_utils.py +++ /dev/null @@ -1,167 +0,0 @@ -# Copyright (c) 2015 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from oslo_log import log as logging - -from sahara.conductor import resource as res -from sahara.plugins.cdh import commands as cmd -from sahara.plugins.cdh import plugin_utils as pu -from sahara.plugins.cdh.v5_4_0 import config_helper -from sahara.plugins.cdh.v5_4_0 import db_helper -from sahara.plugins import utils as u - - -LOG = logging.getLogger(__name__) - - -class PluginUtilsV540(pu.AbstractPluginUtils): - - def __init__(self): - self.c_helper = config_helper.ConfigHelperV540() - self.db_helper = db_helper - - def get_role_name(self, instance, service): - # NOTE: role name must match regexp "[_A-Za-z][-_A-Za-z0-9]{0,63}" - shortcuts = { - 'AGENT': 'A', - 'ALERTPUBLISHER': 'AP', - 'CATALOGSERVER': 'ICS', - 'DATANODE': 'DN', - 'EVENTSERVER': 'ES', - 'HBASE_INDEXER': 'LHBI', - 'HIVEMETASTORE': 'HVM', - 'HIVESERVER2': 'HVS', - 'HOSTMONITOR': 'HM', - 'IMPALAD': 'ID', - 'JOBHISTORY': 'JS', - 'JOURNALNODE': 'JN', - 'KMS': 'KMS', - 'MASTER': 'M', - 'NAMENODE': 'NN', - 'NODEMANAGER': 'NM', - 'OOZIE_SERVER': 'OS', - 'REGIONSERVER': 'RS', - 'RESOURCEMANAGER': 'RM', - 'SECONDARYNAMENODE': 'SNN', - 'SENTRY_SERVER': 'SNT', - 'SERVER': 'S', - 'SERVICEMONITOR': 'SM', - 'SOLR_SERVER': 'SLR', - 'SPARK_YARN_HISTORY_SERVER': 'SHS', - 'SQOOP_SERVER': 'S2S', - 'STATESTORE': 'ISS', - 'WEBHCAT': 'WHC', - 'HDFS_GATEWAY': 'HG', - 'YARN_GATEWAY': 'YG' - } - return '%s_%s' % (shortcuts.get(service, service), - instance.hostname().replace('-', '_')) - - def get_sentry(self, cluster): - return u.get_instance(cluster, 'SENTRY_SERVER') - - def get_flumes(self, cluster): - return u.get_instances(cluster, 'FLUME_AGENT') - - def get_solrs(self, cluster): - return u.get_instances(cluster, 'SOLR_SERVER') - - def get_sqoop(self, cluster): - return u.get_instance(cluster, 'SQOOP_SERVER') - - def get_hbase_indexers(self, cluster): - return u.get_instances(cluster, 'KEY_VALUE_STORE_INDEXER') - - def get_catalogserver(self, cluster): - return u.get_instance(cluster, 'IMPALA_CATALOGSERVER') - - def get_statestore(self, cluster): - return u.get_instance(cluster, 'IMPALA_STATESTORE') - - def get_impalads(self, cluster): - return u.get_instances(cluster, 'IMPALAD') - - def get_kms(self, cluster): - return u.get_instances(cluster, 'KMS') - - def get_jns(self, cluster): - return u.get_instances(cluster, 'HDFS_JOURNALNODE') - - def get_stdb_rm(self, cluster): - return u.get_instance(cluster, 'YARN_STANDBYRM') - - def convert_process_configs(self, configs): - p_dict = { - "CLOUDERA": ['MANAGER'], - "NAMENODE": ['NAMENODE'], - "DATANODE": ['DATANODE'], - "SECONDARYNAMENODE": ['SECONDARYNAMENODE'], - "RESOURCEMANAGER": ['RESOURCEMANAGER'], - "NODEMANAGER": ['NODEMANAGER'], - "JOBHISTORY": ['JOBHISTORY'], - "OOZIE": ['OOZIE_SERVER'], - "HIVESERVER": ['HIVESERVER2'], - "HIVEMETASTORE": ['HIVEMETASTORE'], - "WEBHCAT": ['WEBHCAT'], - "HUE": ['HUE_SERVER'], - "SPARK_ON_YARN": ['SPARK_YARN_HISTORY_SERVER'], - "ZOOKEEPER": ['SERVER'], - "MASTER": ['MASTER'], - "REGIONSERVER": ['REGIONSERVER'], - "FLUME": ['AGENT'], - "CATALOGSERVER": ['CATALOGSERVER'], - "STATESTORE": ['STATESTORE'], - "IMPALAD": ['IMPALAD'], - "KS_INDEXER": ['HBASE_INDEXER'], - "SENTRY": ['SENTRY_SERVER'], - "SOLR": ['SOLR_SERVER'], - "SQOOP": ['SQOOP_SERVER'], - "KMS": ['KMS'], - 'YARN_GATEWAY': ['YARN_GATEWAY'], - 'HDFS_GATEWAY': ['HDFS_GATEWAY'], - "JOURNALNODE": ['JOURNALNODE'] - } - if isinstance(configs, res.Resource): - configs = configs.to_dict() - for k in configs.keys(): - if k in p_dict.keys(): - item = configs[k] - del configs[k] - newkey = p_dict[k][0] - configs[newkey] = item - return res.Resource(configs) - - def configure_sentry(self, cluster): - manager = self.get_manager(cluster) - with manager.remote() as r: - self.db_helper.create_sentry_database(cluster, r) - - def _configure_repo_from_inst(self, instance): - super(PluginUtilsV540, self)._configure_repo_from_inst(instance) - - cluster = instance.cluster - with instance.remote() as r: - if cmd.is_ubuntu_os(r): - kms_key = ( - self.c_helper.get_kms_key_url(cluster) or - self.c_helper.DEFAULT_KEY_TRUSTEE_UBUNTU_REPO_KEY_URL) - kms_repo_url = self.c_helper.KEY_TRUSTEE_UBUNTU_REPO_URL - cmd.add_ubuntu_repository(r, kms_repo_url, 'kms') - cmd.add_apt_key(r, kms_key) - cmd.update_repository(r) - if cmd.is_centos_os(r): - kms_repo_url = self.c_helper.KEY_TRUSTEE_CENTOS_REPO_URL - cmd.add_centos_repository(r, kms_repo_url, 'kms') - cmd.update_repository(r) diff --git a/sahara/plugins/cdh/v5_4_0/resources/cdh_config.py b/sahara/plugins/cdh/v5_4_0/resources/cdh_config.py deleted file mode 100644 index 6affa23d..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/cdh_config.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright (c) 2015 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from oslo_serialization import jsonutils as json -import six - -from sahara.plugins.cdh.client import api_client - -# -- cm config -- - -cm_address = 'localhost' -cm_port = 7180 -cm_username = 'admin' -cm_password = 'admin' - -hdfs_service_name = 'hdfs01' -yarn_service_name = 'yarn01' -oozie_service_name = 'oozie01' -hive_service_name = 'hive01' -hue_service_name = 'hue01' -spark_service_name = 'spark_on_yarn01' -zookeeper_service_name = 'zookeeper01' -hbase_service_name = 'hbase01' -flume_service_name = 'flume01' -sqoop_service_name = 'sqoop01' -solr_service_name = 'solr01' -ks_indexer_service_name = 'ks_indexer01' -impala_service_name = 'impala01' -sentry_service_name = 'sentry01' - - -def get_cm_api(): - return api_client.ApiResource(cm_address, server_port=cm_port, - username=cm_username, password=cm_password) - - -def get_cluster(api): - return api.get_all_clusters()[0] - - -def process_service(service, service_name): - for role_cfgs in service.get_all_role_config_groups(): - role_cm_cfg = role_cfgs.get_config(view='full') - role_cfg = parse_config(role_cm_cfg) - role_name = role_cfgs.roleType.lower() - write_cfg(role_cfg, '%s-%s.json' % (service_name, role_name)) - - service_cm_cfg = service.get_config(view='full')[0] - service_cfg = parse_config(service_cm_cfg) - write_cfg(service_cfg, '%s-service.json' % service_name) - - -def parse_config(config): - cfg = [] - for name, value in six.iteritems(config): - p = { - 'name': value.name, - 'value': value.default, - 'display_name': value.displayName, - 'desc': value.description - } - cfg.append(p) - - return cfg - - -def write_cfg(cfg, file_name): - to_write = json.dumps(cfg, sort_keys=True, indent=4, - separators=(',', ': ')) - - with open(file_name, 'w') as f: - f.write(to_write) - - -def main(): - client = get_cm_api() - cluster = get_cluster(client) - - hdfs = cluster.get_service(hdfs_service_name) - process_service(hdfs, 'hdfs') - - yarn = cluster.get_service(yarn_service_name) - process_service(yarn, 'yarn') - - oozie = cluster.get_service(oozie_service_name) - process_service(oozie, 'oozie') - - hive = cluster.get_service(hive_service_name) - process_service(hive, 'hive') - - hue = cluster.get_service(hue_service_name) - process_service(hue, 'hue') - - spark = cluster.get_service(spark_service_name) - process_service(spark, 'spark') - - zookeeper = cluster.get_service(zookeeper_service_name) - process_service(zookeeper, 'zookeeper') - - hbase = cluster.get_service(hbase_service_name) - process_service(hbase, 'hbase') - - flume = cluster.get_service(flume_service_name) - process_service(flume, 'flume') - - sqoop = cluster.get_service(sqoop_service_name) - process_service(sqoop, 'sqoop') - - solr = cluster.get_service(solr_service_name) - process_service(solr, 'solr') - - ks_indexer = cluster.get_service(ks_indexer_service_name) - process_service(ks_indexer, 'ks_indexer') - - impala = cluster.get_service(impala_service_name) - process_service(impala, 'impala') - - sentry = cluster.get_service(sentry_service_name) - process_service(sentry, 'sentry') - - -if __name__ == '__main__': - main() diff --git a/sahara/plugins/cdh/v5_4_0/resources/cdh_config.sh b/sahara/plugins/cdh/v5_4_0/resources/cdh_config.sh deleted file mode 100755 index 4cfb7b7a..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/cdh_config.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -tox -evenv -- python $(dirname $0)/cdh_config.py $* diff --git a/sahara/plugins/cdh/v5_4_0/resources/create_hive_db.sql b/sahara/plugins/cdh/v5_4_0/resources/create_hive_db.sql deleted file mode 100644 index 7545eeb7..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/create_hive_db.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE ROLE hive LOGIN PASSWORD '%s'; -CREATE DATABASE metastore OWNER hive encoding 'UTF8'; -GRANT ALL PRIVILEGES ON DATABASE metastore TO hive; -COMMIT; diff --git a/sahara/plugins/cdh/v5_4_0/resources/create_sentry_db.sql b/sahara/plugins/cdh/v5_4_0/resources/create_sentry_db.sql deleted file mode 100644 index 30cb6ab3..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/create_sentry_db.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE ROLE sentry LOGIN PASSWORD '%s'; -CREATE DATABASE sentry OWNER sentry encoding 'UTF8'; -GRANT ALL PRIVILEGES ON DATABASE sentry TO sentry; -COMMIT; diff --git a/sahara/plugins/cdh/v5_4_0/resources/flume-agent.json b/sahara/plugins/cdh/v5_4_0/resources/flume-agent.json deleted file mode 100644 index 2aa05e72..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/flume-agent.json +++ /dev/null @@ -1,284 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Used to select an agent configuration to use from flume.conf. Multiple agents may share the same agent name, in which case they will be assigned the same agent configuration.", - "display_name": "Agent Name", - "name": "agent_name", - "value": "tier1" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Text that goes into morphlines.conf file used by the Flume-NG Solr sink. The text goes verbatim into the config file except that $ZK_HOST is replaced by the ZooKeeper quorum of the Solr service.", - "display_name": "Morphlines File", - "name": "agent_morphlines_conf_file", - "value": "# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Application configuration file in HOCON format (Human-Optimized Config Object Notation). \n# HOCON syntax is defined at http://github.com/typesafehub/config/blob/master/HOCON.md\n# and also used by Akka (http://www.akka.io) and Play (http://www.playframework.org/).\n# For more examples see http://doc.akka.io/docs/akka/2.1.2/general/configuration.html\n\n# morphline.conf example file\n# this is a comment\n\n# Specify server locations in a SOLR_LOCATOR variable; used later in variable substitutions:\nSOLR_LOCATOR : {\n # Name of solr collection\n collection : collection1\n \n # ZooKeeper ensemble\n zkHost : \"$ZK_HOST\"\n \n # Relative or absolute path to a directory containing conf/solrconfig.xml and conf/schema.xml\n # If this path is uncommented it takes precedence over the configuration stored in ZooKeeper. \n # solrHomeDir : \"example/solr/collection1\"\n \n # The maximum number of documents to send to Solr per network batch (throughput knob)\n # batchSize : 100\n}\n\n# Specify an array of one or more morphlines, each of which defines an ETL \n# transformation chain. A morphline consists of one or more (potentially \n# nested) commands. A morphline is a way to consume records (e.g. Flume events, \n# HDFS files or blocks), turn them into a stream of records, and pipe the stream \n# of records through a set of easily configurable transformations on it's way to \n# Solr (or a MapReduceIndexerTool RecordWriter that feeds via a Reducer into Solr).\nmorphlines : [\n {\n # Name used to identify a morphline. E.g. used if there are multiple morphlines in a \n # morphline config file\n id : morphline1 \n \n # Import all morphline commands in these java packages and their subpackages.\n # Other commands that may be present on the classpath are not visible to this morphline.\n importCommands : [\"org.kitesdk.**\", \"org.apache.solr.**\"]\n \n commands : [ \n { \n # Parse Avro container file and emit a record for each avro object\n readAvroContainer {\n # Optionally, require the input record to match one of these MIME types:\n # supportedMimeTypes : [avro/binary]\n \n # Optionally, use a custom Avro schema in JSON format inline:\n # schemaString : \"\"\"$KERBEROS_PRINCIPAL\" and \"$KERBEROS_KEYTAB\", which will be replaced by the principal name and the keytab path respectively.",
- "display_name": "Configuration File",
- "name": "agent_config_file",
- "value": "# Please paste flume.conf here. Example:\n\n# Sources, channels, and sinks are defined per\n# agent name, in this case 'tier1'.\ntier1.sources = source1\ntier1.channels = channel1\ntier1.sinks = sink1\n\n# For each source, channel, and sink, set\n# standard properties.\ntier1.sources.source1.type = netcat\ntier1.sources.source1.bind = 127.0.0.1\ntier1.sources.source1.port = 9999\ntier1.sources.source1.channels = channel1\ntier1.channels.channel1.type = memory\ntier1.sinks.sink1.type = logger\ntier1.sinks.sink1.channel = channel1\n\n# Other properties are specific to each type of\n# source, channel, or sink. In this case, we\n# specify the capacity of the memory channel.\ntier1.channels.channel1.capacity = 100\n"
- },
- {
- "desc": "Whether or not periodic stacks collection is enabled.",
- "display_name": "Stacks Collection Enabled",
- "name": "stacks_collection_enabled",
- "value": "false"
- },
- {
- "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.",
- "display_name": "Automatically Restart Process",
- "name": "process_auto_restart",
- "value": "true"
- },
- {
- "desc": "The maximum number of rolled log files to keep for Agent logs. Typically used by log4j or logback.",
- "display_name": "Agent Maximum Log File Backups",
- "name": "max_log_backup_index",
- "value": "10"
- },
- {
- "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Enables the health test that the Agent's process state is consistent with the role configuration", - "display_name": "Agent Process Health Test", - "name": "flume_agent_scm_health_enabled", - "value": "true" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "flume_agent_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The port on which the Flume web server listens for requests.", - "display_name": "HTTP Port", - "name": "agent_http_port", - "value": "41414" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Agent Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Agent in Bytes", - "name": "agent_java_heapsize", - "value": "1073741824" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Disables import of ZooKeeper configuration from the HBase classpath. This prevents zoo.cfg from overriding hbase-site.xml for Zookeeper quorum information. This option is only supported on CDH 4.4 or later deployments.", - "display_name": "HBase sink prefer hbase-site.xml over Zookeeper config", - "name": "agent_disable_zoo_cfg", - "value": "true" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Agent Environment Advanced Configuration Snippet (Safety Valve)", - "name": "AGENT_role_env_safety_valve", - "value": null - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "Directory where Flume Agent will place its log files.", - "display_name": "Flume Agent Log Directory", - "name": "flume_agent_log_dir", - "value": "/var/log/flume-ng" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Text that goes verbatim into custom-mimetypes.xml file used by the Flume-NG Solr sink.", - "display_name": "Custom Mime-types File", - "name": "agent_custom_mimetypes_file", - "value": "\n\n\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "Name of the HDFS service that this Flume service instance depends on", - "display_name": "HDFS Service", - "name": "hdfs_service", - "value": null - }, - { - "desc": "The health test thresholds of the overall Agent health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Agents falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Agents falls below the critical threshold.", - "display_name": "Healthy Agent Monitoring Thresholds", - "name": "flume_agents_healthy_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"95.0\"}" - }, - { - "desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.", - "display_name": "Enable Log Event Capture", - "name": "catch_events", - "value": "true" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hbase-gateway.json b/sahara/plugins/cdh/v5_4_0/resources/hbase-gateway.json deleted file mode 100644 index 5dbc173e..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hbase-gateway.json +++ /dev/null @@ -1,56 +0,0 @@ -[ - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Gateway Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for hbase-site.xml.", - "display_name": "HBase Client Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", - "name": "hbase_client_config_safety_valve", - "value": null - }, - { - "desc": "The directory where the client configs will be deployed", - "display_name": "Deploy Directory", - "name": "client_config_root_dir", - "value": "/etc/hbase" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the client configuration for hbase-env.sh", - "display_name": "HBase Client Environment Advanced Configuration Snippet for hbase-env.sh (Safety Valve)", - "name": "hbase_client_env_safety_valve", - "value": null - }, - { - "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", - "display_name": "Alternatives Priority", - "name": "client_config_priority", - "value": "90" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "These are Java command line arguments. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Client Java Configuration Options", - "name": "hbase_client_java_opts", - "value": "-XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:+CMSIncrementalMode -Djava.net.preferIPv4Stack=true" - }, - { - "desc": "Maximum size in bytes for the Java process heap memory. Passed to Java -Xmx.", - "display_name": "Client Java Heap Size in Bytes", - "name": "hbase_client_java_heapsize", - "value": "268435456" - }, - { - "desc": "The minimum log level for Gateway logs", - "display_name": "Gateway Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hbase-hbaserestserver.json b/sahara/plugins/cdh/v5_4_0/resources/hbase-hbaserestserver.json deleted file mode 100644 index 487e376f..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hbase-hbaserestserver.json +++ /dev/null @@ -1,278 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for HBase REST Server", - "name": "hbase_restserver_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Maximum size of the HBase REST Server thread pool. The server can process this number of concurrent requests. Setting this too high can lead to out of memory errors.", - "display_name": "HBase REST Server Maximum Threads", - "name": "hbase_restserver_threads_max", - "value": "100" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "hbaserestserver_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "HBase REST Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "HBASERESTSERVER_role_env_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", - "display_name": "HBase REST Server Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", - "name": "hbase_restserver_config_safety_valve", - "value": null - }, - { - "desc": "Directory where HBase REST Server will place its log files.", - "display_name": "HBase REST Server Log Directory", - "name": "hbase_restserver_log_dir", - "value": "/var/log/hbase" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for HBase REST Server logs. Typically used by log4j or logback.", - "display_name": "HBase REST Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of HBase REST Server in Bytes", - "name": "hbase_restserver_java_heapsize", - "value": "1073741824" - }, - { - "desc": "Enables the health test that the HBase REST Server's process state is consistent with the role configuration", - "display_name": "HBase REST Server Process Health Test", - "name": "hbaserestserver_scm_health_enabled", - "value": "true" - }, - { - "desc": "The port that HBase REST Server binds to.", - "display_name": "HBase REST Server Port", - "name": "hbase_restserver_port", - "value": "20550" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "If true, HBase REST Server Web UI will bind to a wildcard address (0.0.0.0). Otherwise it will bind to a host name. Only available in CDH 4.3 and later.", - "display_name": "HBase REST Server Web UI Bind to Wildcard Address", - "name": "hbase_restserver_info_bind_to_wildcard", - "value": "true" - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "HBase REST Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "When false, all HTTP methods are permitted (GET/PUT/POST/DELETE). When true, only GET is permitted.", - "display_name": "Enable HBase REST Server Read Only Mode", - "name": "hbase_restserver_readonly", - "value": "false" - }, - { - "desc": "The port that HBase REST Server Web UI binds to.", - "display_name": "HBase REST Server Web UI Port", - "name": "hbase_restserver_info_port", - "value": "8085" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When computing the overall HBase REST Server health, consider the host's health.", - "display_name": "HBase REST Server Host Health Test", - "name": "hbaserestserver_host_health_enabled", - "value": "true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for HBase REST Server logs. Typically used by log4j or logback.", - "display_name": "HBase REST Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The host name or IP address of the DNS name server which an HBase REST Server should use to determine the host name used for communication and display purposes.", - "display_name": "HBase REST Server DNS Name Server", - "name": "hbase_restserver_dns_nameserver", - "value": null - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "HBase REST Server will bind to this address.", - "display_name": "HBase REST Server Host Address", - "name": "hbase_restserver_host", - "value": "0.0.0.0" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The name of the DNS network interface from which an HBase REST Server should report its IP address.", - "display_name": "HBase REST Server DNS Network Interface", - "name": "hbase_restserver_dns_interface", - "value": null - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Minimum size of the HBase REST Server thread pool. The server will maintain at least this number of threads in the pool at all times. The thread pool can grow up to the maximum size set by hbase.rest.threads.max.", - "display_name": "HBase REST Server Minimum Threads", - "name": "hbase_restserver_threads_min", - "value": "2" - }, - { - "desc": "The minimum log level for HBase REST Server logs", - "display_name": "HBase REST Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hbase-hbasethriftserver.json b/sahara/plugins/cdh/v5_4_0/resources/hbase-hbasethriftserver.json deleted file mode 100644 index 588ea74e..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hbase-hbasethriftserver.json +++ /dev/null @@ -1,284 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "The \"core size\" of the thread pool. New threads are created on every connection until this many threads are created.", - "display_name": "HBase Thrift Server Min Worker Threads", - "name": "hbase_thriftserver_min_worker_threads", - "value": "200" - }, - { - "desc": "The port that HBase Thrift Server binds to.", - "display_name": "HBase Thrift Server Port", - "name": "hbase_thriftserver_port", - "value": "9090" - }, - { - "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", - "display_name": "HBase Thrift Server Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", - "name": "hbase_thriftserver_config_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for HBase Thrift Server logs. Typically used by log4j or logback.", - "display_name": "HBase Thrift Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of HBase Thrift Server in Bytes", - "name": "hbase_thriftserver_java_heapsize", - "value": "1073741824" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "If true, HBase Thrift Server Web UI will bind to a wildcard address (0.0.0.0). Otherwise it will bind to a host name. Only available in CDH 4.3 and later.", - "display_name": "HBase Thrift Server Web UI Bind to Wildcard Address", - "name": "hbase_thriftserver_info_bind_to_wildcard", - "value": "true" - }, - { - "desc": "Address to bind the HBase Thrift Server to. When using the THsHaServer or the TNonblockingServer, always binds to 0.0.0.0 irrespective of this configuration value.", - "display_name": "HBase Thrift Server Bind Address", - "name": "hbase_thriftserver_bindaddress", - "value": "0.0.0.0" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "Type of HBase Thrift Server.", - "display_name": "HBase Thrift Server Type", - "name": "hbase_thriftserver_type", - "value": "threadpool" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "HBase Thrift Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "HBASETHRIFTSERVER_role_env_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "HBase Thrift Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "When computing the overall HBase Thrift Server health, consider the host's health.", - "display_name": "HBase Thrift Server Host Health Test", - "name": "hbasethriftserver_host_health_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Enables the health test that the HBase Thrift Server's process state is consistent with the role configuration", - "display_name": "HBase Thrift Server Process Health Test", - "name": "hbasethriftserver_scm_health_enabled", - "value": "true" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for HBase Thrift Server", - "name": "hbase_thriftserver_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "Use framed transport. When using the THsHaServer or TNonblockingServer, framed transport is always used irrespective of this configuration value.", - "display_name": "Enable HBase Thrift Server Framed Transport", - "name": "hbase_thriftserver_framed", - "value": "false" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for HBase Thrift Server logs. Typically used by log4j or logback.", - "display_name": "HBase Thrift Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "The port that HBase Thrift Server Web UI binds to.", - "display_name": "HBase Thrift Server Web UI Port", - "name": "hbase_thriftserver_info_port", - "value": "9095" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The host name or IP address of the DNS name server which an HBase Thrift Server should use to determine the host name used for communication and display purposes.", - "display_name": "HBase Thrift Server DNS Name Server", - "name": "hbase_thriftserver_dns_nameserver", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The name of the DNS network interface from which an HBase Thrift Server should report its IP address.", - "display_name": "HBase Thrift Server DNS Network Interface", - "name": "hbase_thriftserver_dns_interface", - "value": null - }, - { - "desc": "Use the TCompactProtocol instead of the default TBinaryProtocol. TCompactProtocol is a binary protocol that is more compact than the default and typically more efficient.", - "display_name": "Enable HBase Thrift Server Compact Protocol", - "name": "hbase_thriftserver_compact", - "value": "false" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Directory where HBase Thrift Server will place its log files.", - "display_name": "HBase Thrift Server Log Directory", - "name": "hbase_thriftserver_log_dir", - "value": "/var/log/hbase" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "hbasethriftserver_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The minimum log level for HBase Thrift Server logs", - "display_name": "HBase Thrift Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hbase-master.json b/sahara/plugins/cdh/v5_4_0/resources/hbase-master.json deleted file mode 100644 index 24212217..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hbase-master.json +++ /dev/null @@ -1,350 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"IPC Server handler.*ClosedChannelException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"IPC Server Responder, call.*output error\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Daughter regiondir does not exist: .*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"File.*might still be open.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"File.*might still be open.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Moving table .+ state to enabled but was already enabled\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Received OPENED for region.*but region was in the state.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", - "display_name": "Health Check Startup Tolerance", - "name": "master_startup_tolerance", - "value": "5" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Number of pooled threads to handle region closing in the master.", - "display_name": "Region Closing Threads", - "name": "hbase_master_executor_closeregion_threads", - "value": "5" - }, - { - "desc": "When computing the overall Master health, consider the host's health.", - "display_name": "Master Host Health Test", - "name": "master_host_health_enabled", - "value": "true" - }, - { - "desc": "Number of pooled threads to handle the recovery of the region servers in the master.", - "display_name": "RegionServer Recovery Threads", - "name": "hbase_master_executor_serverops_threads", - "value": "5" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Directory where Master will place its log files.", - "display_name": "Master Log Directory", - "name": "hbase_master_log_dir", - "value": "/var/log/hbase" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "master_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Time period in seconds to reset long-running metrics (e.g. compactions). This is an HBase specific configuration.", - "display_name": "Extended Period", - "name": "hbase_metrics_extended_period", - "value": "3600" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Master Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "master_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The port that the HBase Master binds to.", - "display_name": "HBase Master Port", - "name": "hbase_master_port", - "value": "60000" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "master_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "List of org.apache.hadoop.hbase.coprocessor.MasterObserver coprocessors that are loaded by default on the active HMaster process. For any implemented coprocessor methods, the listed classes will be called in order. After implementing your own MasterObserver, just put it in HBase's classpath and add the fully qualified class name here.", - "display_name": "HBase Coprocessor Master Classes", - "name": "hbase_coprocessor_master_classes", - "value": "" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of HBase Master in Bytes", - "name": "hbase_master_java_heapsize", - "value": "1073741824" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "master_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "When true, HBase Master will bind to 0.0.0.0. Only available with CDH 4.3 and later.", - "display_name": "HBase Master Bind to Wildcard Address", - "name": "hbase_master_bind_to_wildcard_address", - "value": "true" - }, - { - "desc": "Enables the health test that a client can connect to the HBase Master", - "display_name": "HBase Master Canary Health Test", - "name": "master_canary_health_enabled", - "value": "true" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The port for the HBase Master web UI. Set to -1 to disable the HBase Master web UI.", - "display_name": "HBase Master Web UI Port", - "name": "hbase_master_info_port", - "value": "60010" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "master_gc_duration_window", - "value": "5" - }, - { - "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", - "display_name": "Master Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", - "name": "hbase_master_config_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for HBase Master", - "name": "hbase_master_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Master logs. Typically used by log4j or logback.", - "display_name": "Master Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "Maximum time an HLog remains in the .oldlogdir directory until an HBase Master thread deletes it.", - "display_name": "Maximum Time to Keep HLogs", - "name": "hbase_master_logcleaner_ttl", - "value": "60000" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Master Environment Advanced Configuration Snippet (Safety Valve)", - "name": "MASTER_role_env_safety_valve", - "value": null - }, - { - "desc": "Number of RPC Server instances spun up on HBase Master.", - "display_name": "HBase Master Handler Count", - "name": "hbase_master_handler_count", - "value": "25" - }, - { - "desc": "The address for the HBase Master web UI", - "display_name": "HBase Master Web UI Address", - "name": "hbase_master_info_bindAddress", - "value": null - }, - { - "desc": "A comma-separated list of LogCleanerDelegate(s) that are used in LogsCleaner. WAL/HLog cleaner(s) are called in order, so put the log cleaner that prunes the most log files in the front. To implement your own LogCleanerDelegate, add it to HBase's classpath and add the fully-qualified class name here. You should always add the above default log cleaners in the list, unless you have a special reason not to.", - "display_name": "HBase Master Log Cleaner Plugins", - "name": "hbase_master_logcleaner_plugins", - "value": null - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "The maximum number of rolled log files to keep for Master logs. Typically used by log4j or logback.", - "display_name": "Master Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The host name or IP address of the DNS name server which an HBase Master should use to determine the host name used for communication and display purposes.", - "display_name": "HBase Master DNS Name Server", - "name": "hbase_master_dns_nameserver", - "value": null - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "Number of pooled threads to handle region opening in the master.", - "display_name": "Region Opening Threads", - "name": "hbase_master_executor_openregion_threads", - "value": "5" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The name of the DNS network interface from which an HBase Master should report its IP address.", - "display_name": "HBase Master DNS Network Interface", - "name": "hbase_master_dns_interface", - "value": null - }, - { - "desc": "The minimum log level for Master logs", - "display_name": "Master Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Enables the health test that the Master's process state is consistent with the role configuration", - "display_name": "Master Process Health Test", - "name": "master_scm_health_enabled", - "value": "true" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hbase-regionserver.json b/sahara/plugins/cdh/v5_4_0/resources/hbase-regionserver.json deleted file mode 100644 index 803a4b8a..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hbase-regionserver.json +++ /dev/null @@ -1,584 +0,0 @@ -[ - { - "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", - "display_name": "RegionServer Advanced Configuration Snippet (Safety Valve) for hbase-site.xml", - "name": "hbase_regionserver_config_safety_valve", - "value": null - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Maximum number of HStoreFiles to compact per minor compaction.", - "display_name": "Maximum Number of HStoreFiles Compaction", - "name": "hbase_hstore_compaction_max", - "value": null - }, - { - "desc": "The address for the HBase RegionServer web UI", - "display_name": "HBase RegionServer Web UI Address", - "name": "hbase_regionserver_info_bindAddress", - "value": null - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "Number of threads to use while loading and unloading regions to or from a RegionServer. Can be used to increase the speed of decommissioning or rolling restart operations.", - "display_name": "Region Mover Threads", - "name": "hbase_regionserver_regionmover_thread_count", - "value": "1" - }, - { - "desc": "The health test thresholds of the average size of the HBase RegionServer flush queue over a recent period. See HBase RegionServer Flush Queue Monitoring Period.", - "display_name": "HBase RegionServer Flush Queue Monitoring Thresholds", - "name": "regionserver_flush_queue_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10.0\"}" - }, - { - "desc": "The amount of time to wait for the HBase Region Server to fully start up and connect to the HBase Master before enforcing the connectivity check.", - "display_name": "HBase Region Server Connectivity Tolerance at Startup", - "name": "regionserver_connectivity_tolerance", - "value": "180" - }, - { - "desc": "The hashing algorithm for use in HashFunction. Two values are supported: 'murmur' (for MurmurHash) and 'jenkins' (for JenkinsHash).", - "display_name": "HBase Hash Type", - "name": "hbase_hash_type", - "value": "murmur" - }, - { - "desc": "The port for the HBase RegionServer web UI. Set to -1 to disable RegionServer web UI.", - "display_name": "HBase RegionServer Web UI port", - "name": "hbase_regionserver_info_port", - "value": "60030" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The period over which to compute the moving average of the compaction queue size.", - "display_name": "HBase RegionServer Compaction Queue Monitoring Period", - "name": "regionserver_compaction_queue_window", - "value": "5" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for HBase RegionServer", - "name": "hbase_regionserver_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "RegionServer Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds of the latency that the RegionServer sees for HDFS read operations", - "display_name": "HBase RegionServer HDFS Read Latency Thresholds", - "name": "regionserver_read_latency_thresholds", - "value": "{\"critical\":\"100.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds of the size used by the HStoreFile index. Specified as a percentage of the total heap size.", - "display_name": "Percentage of Heap Used by HStoreFile Index", - "name": "regionserver_store_file_idx_size_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10.0\"}" - }, - { - "desc": "Maximum number of Write-Ahead Log (WAL) files. This value multiplied by HDFS Block Size (dfs.blocksize) is the size of the WAL that will need to be replayed when a server crashes. This value is inversely proportional to the frequency of flushes to disk.", - "display_name": "Maximum number of Write-Ahead Log (WAL) files", - "name": "hbase_regionserver_maxlogs", - "value": "32" - }, - { - "desc": "List of coprocessors that are loaded by default on all tables. For any override coprocessor method, these classes will be called in order. After implementing your own coprocessor, just put it in HBase's classpath and add the fully qualified class name here. A coprocessor can also be loaded on demand by setting HTableDescriptor.", - "display_name": "HBase Coprocessor Region Classes", - "name": "hbase_coprocessor_region_classes", - "value": "" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "regionserver_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "Blocks writes if the size of the memstore increases to the value of 'hbase.hregion.block.memstore' multiplied by the value of 'hbase.hregion.flush.size' bytes. This setting is useful for preventing runaway memstore during spikes in update traffic. Without an upper-bound, memstore fills such that when it flushes, the resultant process of flushing files take a long time to compact or split, or worse, an \"out of memory\" error occurs.", - "display_name": "HBase Memstore Block Multiplier", - "name": "hbase_hregion_memstore_block_multiplier", - "value": "2" - }, - { - "desc": "Sync the HLog to HDFS after this interval, in milliseconds, if it has not accumulated the number of HLog Entries specified to trigger a sync.", - "display_name": "Synch Interval of HLog Entries", - "name": "hbase_regionserver_optionallogflushinterval", - "value": "1000" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \"IPC Server handler.*took.*appending an edit to hlog.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"ABORTING region server serverName.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"DFSOutputStream ResponseProcessor exception.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Error Recovery for block blk.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Failed init\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Problem renewing lease for DFSClient.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"remote error telling master we are up\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\", \"content\": \"Session.*for server.*closing socket connection and attempting reconnect.\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Starting .*compaction on region (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_REGION_STARTED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^completed compaction on region (.+) after (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_REGION_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Starting compaction on (.+) in region (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_COLUMN_FAMILY_STARTED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"COLUMN_FAMILY\", \"group1\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^completed compaction: regionName=(.+), storeName=(.+), fileCount=(.+), fileSize=(.+), priority=(.+), time=(.+); duration=(.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_COLUMN_FAMILY_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"COLUMN_FAMILY\", \"group2\": \"FILE_COUNT\", \"group3\": \"FILE_SIZE\", \"group4\": \"PRIORITY\", \"group5\": \"REQUEST_TIME_NANOS\", \"group6\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Completed compaction: Request = regionName=(.+), storeName=(.+), fileCount=(.+), fileSize=(.+), priority=(.+), time=(.+); duration=(.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_COLUMN_FAMILY_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"COLUMN_FAMILY\", \"group2\": \"FILE_COUNT\", \"group3\": \"FILE_SIZE\", \"group4\": \"PRIORITY\", \"group5\": \"REQUEST_TIME_NANOS\", \"group6\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^aborted compaction: regionName=(.+), storeName=(.+), fileCount=(.+), fileSize=(.+), priority=(.+), time=(.+); duration=(.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_COMPACTION_COLUMN_FAMILY_ABORTED\", \"attribute:SEVERITY\": \"IMPORTANT\", \"group0\": \"REGION\", \"group1\": \"COLUMN_FAMILY\", \"group2\": \"FILE_COUNT\", \"group3\": \"FILE_SIZE\", \"group4\": \"PRIORITY\", \"group5\": \"REQUEST_TIME_NANOS\", \"group6\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Finished memstore flush of .+ for region (.+) in (.+), sequenceid=(.+), compaction requested=(.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_FLUSH_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"DURATION\", \"group2\": \"SEQUENCE_ID\", \"group3\": \"COMPACTION_REQUESTED\" },\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Flush of region (.+) due to global heap pressure$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_FLUSH_DUE_TO_HEAP_PRESSURE\", \"attribute:SEVERITY\": \"IMPORTANT\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"WARN\", \"content\": \"^Region (.+) has too many store files; delaying flush up to 90000ms$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_FLUSH_DELAYED_TOO_MANY_STORE_FILES\", \"attribute:SEVERITY\": \"CRITICAL\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Starting split of region (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_SPLIT_STARTED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Running rollback/cleanup of failed split of (.+);.+$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_SPLIT_ABORTED\", \"attribute:SEVERITY\": \"IMPORTANT\", \"group0\": \"REGION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Region split, hbase:meta updated, and report to master. Parent=(.+), new regions: (.+,.*,.+), (.+,.*,.+). Split took (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_SPLIT_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"DAUGHTER_REGIONS\", \"group2\": \"DAUGHTER_REGIONS\", \"group3\": \"DURATION\"},\n {\"alert\": false, \"rate\": -1, \"threshold\":\"INFO\", \"content\": \"^Region split, META updated, and report to master. Parent=(.+), new regions: (.+,.*,.+), (.+,.*,.+). Split took (.+)$\", \"attribute:CATEGORY\": \"HBASE\", \"attribute:EVENTCODE\": \"EV_HBASE_SPLIT_COMPLETED\", \"attribute:SEVERITY\": \"INFORMATIONAL\", \"group0\": \"REGION\", \"group1\": \"DAUGHTER_REGIONS\", \"group2\": \"DAUGHTER_REGIONS\", \"group3\": \"DURATION\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" - }, - { - "desc": "The host name or IP address of the DNS name server which a RegionServer should use to determine the host name used by the HBase Master for communication and display purposes.", - "display_name": "RegionServer DNS Nameserver", - "name": "hbase_regionserver_dns_nameserver", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The health test thresholds of the total size of RegionServer's memstores. Specified as a percentage of the configured upper limit. See Maximum Size of All Memstores in RegionServer.", - "display_name": "HBase RegionServer Memstore Size Thresholds", - "name": "regionserver_memstore_size_thresholds", - "value": "{\"critical\":\"100.0\",\"warning\":\"95.0\"}" - }, - { - "desc": "Percentage of maximum heap (-Xmx setting) to allocate to block cache used by HFile/StoreFile. To disable, set this value to 0 .", - "display_name": "HFile Block Cache Size", - "name": "hfile_block_cache_size", - "value": "0.4" - }, - { - "desc": "The maximum size, in megabytes, per log file for RegionServer logs. Typically used by log4j or logback.", - "display_name": "RegionServer Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "If the memstores in a region are this size or larger when closing, run a pre-flush process to clear out memstores before putting up the region closed flag and taking the region offline. On close, a flush process is run under the close flag up to empty memory. During this time, the region is offline and no writes are taken. If the memstore content is large, the flush process could take a long time to complete. The pre-flush process cleans out the bulk of the memstore before putting up the close flag and taking the region offline, so that the flush process that runs under the close flag has little to do.", - "display_name": "HBase Memstore Pre-close Flush Size", - "name": "hbase_hregion_preclose_flush_size", - "value": "5242880" - }, - { - "desc": "If this number of HStoreFiles in any one HStore is exceeded, then a compaction is run to rewrite all HStoreFiles files as one HStoreFile. (One HStoreFile is written per flush of memstore.) You can delay compaction by specifying a larger number, but the compaction will take longer when it does run. During a compaction, updates cannot be flushed to disk. Long compactions require memory sufficient to carry the logging of all updates across the duration of the compaction. If too large, clients timeout during compaction.", - "display_name": "HStore Compaction Threshold", - "name": "hbase_hstore_compactionThreshold", - "value": "3" - }, - { - "desc": "The period over which to compute the moving average of the HDFS sync latency of the HBase RegionServer.", - "display_name": "HBase RegionServer HDFS Sync Latency Monitoring Period", - "name": "regionserver_sync_latency_window", - "value": "5" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The maximum size byte array that should come from the MSLAB, in bytes.", - "display_name": "Maximum Byte Array from MSLAB Allocation Scheme", - "name": "hbase_hregion_memstore_mslab_max_allocation", - "value": "262144" - }, - { - "desc": "The health test thresholds for the latency of HDFS write operations that the RegionServer detects", - "display_name": "HBase RegionServer HDFS Sync Latency Thresholds", - "name": "regionserver_sync_latency_thresholds", - "value": "{\"critical\":\"5000.0\",\"warning\":\"500.0\"}" - }, - { - "desc": "Maximum size of all memstores in a RegionServer before new updates are blocked and flushes are forced.", - "display_name": "Maximum Size of All Memstores in RegionServer", - "name": "hbase_regionserver_global_memstore_upperLimit", - "value": "0.4" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Directory where RegionServer will place its log files.", - "display_name": "RegionServer Log Directory", - "name": "hbase_regionserver_log_dir", - "value": "/var/log/hbase" - }, - { - "desc": "The period of time that an HRegion will block updates after reaching the HStoreFile limit that is specified by 'hbase.hstore.blockingStoreFiles'. After this time has elapsed, the HRegion will stop blocking updates even if a compaction has not been completed.", - "display_name": "HStore Blocking Wait Time", - "name": "hbase_hstore_blockingWaitTime", - "value": "90000" - }, - { - "desc": "Timeout for Canary to perform its checks.", - "display_name": "Canary Timeout", - "name": "hbase_regionserver_canary_timeout", - "value": "15" - }, - { - "desc": "Interval, in milliseconds, between messages from the RegionServer to the HBase Master. Use a high value such as 3000 for clusters that have more than 10 hosts.", - "display_name": "HBase RegionServer Message Interval", - "name": "hbase_regionserver_msginterval", - "value": "3000" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The lease period, in milliseconds, for the HBase RegionServer. Clients must report in within this period or else they are considered dead.", - "display_name": "HBase RegionServer Lease Period", - "name": "hbase_regionserver_lease_period", - "value": "60000" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "The minimum log level for RegionServer logs", - "display_name": "RegionServer Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "regionserver_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "The name of the DNS Network Interface from which a RegionServer should report its IP address.", - "display_name": "RegionServer DNS Network Interface", - "name": "hbase_regionserver_dns_interface", - "value": null - }, - { - "desc": "The number of reservation blocks that are used to prevent unstable RegionServers caused by an OOME.", - "display_name": "RegionServer Reservation Blocks", - "name": "hbase_regionserver_nbreservationblocks", - "value": "4" - }, - { - "desc": "The implementation that should be used by the RegionServer for the write-ahead-log.", - "display_name": "WAL Provider", - "name": "hbase_regionserver_wal_provider", - "value": "multiwal" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "regionserver_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The size of the chunks allocated by MSLAB, in bytes.", - "display_name": "Chunk Size Allocated by MSLAB Allocation Scheme", - "name": "hbase_hregion_memstore_mslab_chunksize", - "value": "2097152" - }, - { - "desc": "The HLog file reader implementation.", - "display_name": "HLog Reader Implementation", - "name": "hbase_regionserver_hlog_reader_impl", - "value": null - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The period over which to compute the moving average of the HDFS read latency of the HBase RegionServer.", - "display_name": "HBase RegionServer HDFS Read Latency Monitoring Period", - "name": "regionserver_read_latency_window", - "value": "5" - }, - { - "desc": "The directory for HBase secure bulk Load.", - "display_name": "HBase Secure Bulk Load Directory", - "name": "hbase_bulkload_staging_dir", - "value": "/tmp/hbase-staging" - }, - { - "desc": "An interface that is assignable to HRegionInterface. Used in HBase Client for opening a proxy to a remote HBase RegionServer.", - "display_name": "HBase RegionServer Interface Class", - "name": "hbase_regionserver_class", - "value": null - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "When true, HBase RegionServer will bind to 0.0.0.0. Only available in CDH 4.3 and later.", - "display_name": "HBase RegionServer Bind to Wildcard Address", - "name": "hbase_regionserver_bind_to_wildcard_address", - "value": "true" - }, - { - "desc": "The time, in milliseconds, between 'major' compactions of all HStoreFiles in a region. To disable automated major compactions, set this value to 0.", - "display_name": "HBase HRegion Major Compaction", - "name": "hbase_hregion_majorcompaction", - "value": "604800000" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Enable MemStore-Local Allocation Buffer (MSLAB) Allocation Scheme. Note: This feature is experimental in CDH3.", - "display_name": "Enable MSLAB Allocation Scheme", - "name": "hbase_hregion_memstore_mslab_enabled", - "value": "true" - }, - { - "desc": "Enables the health test that the RegionServer is connected to the Master", - "display_name": "HBase RegionServer to Master Connectivity Test", - "name": "regionserver_master_connectivity_enabled", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": "32768" - }, - { - "desc": "Limit for the number of regions after which no more region splitting should take place. This is not a hard limit for the number of regions but acts as a guideline for the RegionServer to stop splitting after a certain limit.", - "display_name": "HBase Region Split Limit", - "name": "hbase_regionserver_regionSplitLimit", - "value": "2147483647" - }, - { - "desc": "When using MultiWAL as the WALProvider, sets how many write-ahead-logs each region server should run. Will result in this number of HDFS pipelines. Writes for a given Region only go to a single pipeline, spreading total RegionServer load.", - "display_name": "Per-RegionServer Number of WAL Pipelines", - "name": "hbase_regionserver_wal_pipelines", - "value": "1" - }, - { - "desc": "Enables the health test that the RegionServer's process state is consistent with the role configuration", - "display_name": "RegionServer Process Health Test", - "name": "regionserver_scm_health_enabled", - "value": "true" - }, - { - "desc": "Memstore will be flushed to disk if size of the memstore exceeds this value in number of bytes. This value is checked by a thread that runs the frequency specified by hbase.server.thread.wakefrequency.", - "display_name": "HBase Memstore Flush Size", - "name": "hbase_hregion_memstore_flush_size", - "value": "134217728" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "When memstores are being forced to flush to make room in memory, keep flushing until this amount is reached. If this amount is equal to 'hbase.regionserver.global.memstore.upperLimit', then minimum possible flushing will occur when updates are blocked due to memstore limiting.", - "display_name": "Low Watermark for Memstore Flush", - "name": "hbase_regionserver_global_memstore_lowerLimit", - "value": "0.38" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "regionserver_gc_duration_window", - "value": "5" - }, - { - "desc": "Time period in seconds to reset long-running metrics (e.g. compactions). This is an HBase specific configuration.", - "display_name": "Extended Period", - "name": "hbase_metrics_extended_period", - "value": "3600" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "RegionServer Environment Advanced Configuration Snippet (Safety Valve)", - "name": "REGIONSERVER_role_env_safety_valve", - "value": null - }, - { - "desc": "Comma-separated list of codecs that the RegionServer requires to start. Use this setting to make sure that all RegionServers joining a cluster are installed with a particular set of codecs.", - "display_name": "RegionServer Codecs", - "name": "hbase_regionserver_codecs", - "value": "" - }, - { - "desc": "Number of handlers for processing priority requests in a RegionServer.", - "display_name": "HBase RegionServer Meta-Handler Count", - "name": "hbase_regionserver_metahandler_count", - "value": "10" - }, - { - "desc": "The maximum number of rolled log files to keep for RegionServer logs. Typically used by log4j or logback.", - "display_name": "RegionServer Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Period, in milliseconds, at which to roll the commit log.", - "display_name": "HBase RegionServer Log Roll Period", - "name": "hbase_regionserver_logroll_period", - "value": "3600000" - }, - { - "desc": "The period over which to compute the moving average of the flush queue size.", - "display_name": "HBase RegionServer Flush Queue Monitoring Period", - "name": "regionserver_flush_queue_window", - "value": "5" - }, - { - "desc": "Maximum HStoreFile size. If any one of a column families' HStoreFiles has grown to exceed this value, the hosting HRegion is split in two.", - "display_name": "HBase Maximum File Size", - "name": "hbase_hregion_max_filesize", - "value": "10737418240" - }, - { - "desc": "The health test thresholds of the weighted average size of the HBase RegionServer compaction queue over a recent period. See HBase RegionServer Compaction Queue Monitoring Period.", - "display_name": "HBase RegionServer Compaction Queue Monitoring Thresholds", - "name": "regionserver_compaction_queue_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10.0\"}" - }, - { - "desc": "If there are more than this number of HStoreFiles in any one HStore, then updates are blocked for this HRegion until a compaction is completed, or until the value specified for 'hbase.hstore.blockingWaitTime' has been exceeded.", - "display_name": "HStore Blocking Store Files", - "name": "hbase_hstore_blockingStoreFiles", - "value": "10" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "regionserver_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "The HLog file writer implementation.", - "display_name": "HLog Writer Implementation", - "name": "hbase_regionserver_hlog_writer_impl", - "value": null - }, - { - "desc": "Duration between consecutive checks done by the Canary.", - "display_name": "Canary Interval", - "name": "hbase_regionserver_canary_interval", - "value": "6" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The port that an HBase RegionServer binds to.", - "display_name": "HBase RegionServer Port", - "name": "hbase_regionserver_port", - "value": "60020" - }, - { - "desc": "Number of RPC Server instances spun up on RegionServers.", - "display_name": "HBase RegionServer Handler Count", - "name": "hbase_regionserver_handler_count", - "value": "30" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of HBase RegionServer in Bytes", - "name": "hbase_regionserver_java_heapsize", - "value": "4294967296" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "When computing the overall RegionServer health, consider the host's health.", - "display_name": "RegionServer Host Health Test", - "name": "regionserver_host_health_enabled", - "value": "true" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hbase-service.json b/sahara/plugins/cdh/v5_4_0/resources/hbase-service.json deleted file mode 100644 index 177ec1f4..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hbase-service.json +++ /dev/null @@ -1,524 +0,0 @@ -[ - { - "desc": "The user that this service's processes should run as.", - "display_name": "System User", - "name": "process_username", - "value": "hbase" - }, - { - "desc": "Event filters are defined in a JSON object like the following:
\n\n\n{\n \"defaultAction\" : (\"accept\", \"discard\"),\n \"rules\" : [\n {\n \"action\" : (\"accept\", \"discard\"),\n \"fields\" : [\n {\n \"name\" : \"fieldName\",\n \"match\" : \"regex\"\n }\n ]\n }\n ]\n}\n\n\n\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n
\n\n\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n
\n\n\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n
\n\n\nThe following is the list of fields that can be filtered for HBase events:\n
\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "AWS secret access key required to access S3 to store remote snapshots.", - "display_name": "AWS S3 Secret Access Key for Remote Snapshots", - "name": "hbase_snapshot_s3_secret_access_key", - "value": null - }, - { - "desc": "Number of rows to fetch when calling next on a scanner if it is not served from memory. Higher caching values enable faster scanners but require more memory and some calls of next may take longer when the cache is empty.", - "display_name": "HBase Client Scanner Caching", - "name": "hbase_client_scanner_caching", - "value": "100" - }, - { - "desc": "Tables to exclude in the HBase Region Health Canary which will scan a row from every region.", - "display_name": "HBase Region Health Canary Exclude Tables", - "name": "hbase_region_health_canary_exclude_tables", - "value": "" - }, - { - "desc": "The maximum amount of time the HBase master waits for a snapshot to complete.", - "display_name": "HBase Master Snapshot Timeout", - "name": "hbase_snapshot_master_timeout_millis", - "value": "60000" - }, - { - "desc": "AWS S3 path where remote snapshots should be stored.", - "display_name": "AWS S3 Path for Remote Snapshots", - "name": "hbase_snapshot_s3_path", - "value": null - }, - { - "desc": "Start a process to periodically check that RegionServer is alive when RegionServer is started. Note: This canary is different from the Cloudera Service Monitoring canary and is provided by the HBase service itself.", - "display_name": "Enable HBase Canary", - "name": "hbase_regionserver_enable_canary", - "value": "false" - }, - { - "desc": "Enable HDFS short circuit read. This allows a client co-located with the DataNode to read HDFS file blocks directly. This gives a performance boost to distributed clients that are aware of locality.", - "display_name": "Enable HDFS Short Circuit Read", - "name": "dfs_client_read_shortcircuit", - "value": "true" - }, - { - "desc": "Path to ZooKeeper Node holding root region location. This is written by the HBase Master and read by clients and RegionServers. If a relative path is given, the parent folder will be ${zookeeper.znode.parent}. By default, the root location is stored at /hbase/root-region-server.", - "display_name": "ZooKeeper Znode Rootserver", - "name": "zookeeper_znode_rootserver", - "value": "root-region-server" - }, - { - "desc": "When computing the overall HBase cluster health, consider the health of the backup HBase Masters.", - "display_name": "Backup Masters Health Test", - "name": "hbase_backup_masters_health_enabled", - "value": "true" - }, - { - "desc": "Password that protects the private key contained in the server keystore used for encrypted web UIs.", - "display_name": "SSL Server Keystore Key Password", - "name": "ssl_server_keystore_keypassword", - "value": null - }, - { - "desc": "An alert is published if the HBase region health canary runs slowly.", - "display_name": "HBase Region Health Canary Slow Run Alert Enabled", - "name": "hbase_region_health_canary_slow_run_alert_enabled", - "value": "true" - }, - { - "desc": "Name of the scheduler pool to use for MR jobs created during export/import of remote snapshots in AWS S3.", - "display_name": "Scheduler Pool for Remote Snapshots in AWS S3", - "name": "hbase_snapshot_s3_scheduler_pool", - "value": null - }, - { - "desc": "An alert is published if the HBase region health canary detects at least this percentage of total regions are unhealthy. This threshold is used if the explicit count is not set via the hbase_canary_alert_unhealthy_region_count_threshold config.", - "display_name": "HBase Canary Unhealthy Region Percentage Alert Threshold", - "name": "hbase_canary_alert_unhealthy_region_percent_threshold", - "value": "0.1" - }, - { - "desc": "Duration to wait before starting up a 'hedged' read.", - "display_name": "HDFS Hedged Read Delay Threshold", - "name": "hbase_server_dfs_client_hedged_read_threshold_millis", - "value": "500" - }, - { - "desc": "Enable SSL encryption for HBase web UIs.", - "display_name": "Web UI SSL Encryption Enabled", - "name": "hbase_hadoop_ssl_enabled", - "value": "false" - }, - { - "desc": "List of users or groups, who are allowed full privileges, regardless of stored ACLs, across the cluster. Only used when HBase security is enabled.", - "display_name": "HBase Superusers", - "name": "hbase_superuser", - "value": "" - }, - { - "desc": "Use this to allow proxy users on thrift gateway, which is mainly needed for \"doAs\" functionality.", - "display_name": "Enable HBase Thrift Proxy Users", - "name": "hbase_thriftserver_support_proxyuser", - "value": "false" - }, - { - "desc": "Maximum number of hlog entries to replicate in one go. If this is large, and a consumer takes a while to process the events, the HBase RPC call will time out.", - "display_name": "Replication Batch Size", - "name": "hbase_replication_source_nb_capacity", - "value": "1000" - }, - { - "desc": "ZooKeeper session timeout in milliseconds. HBase passes this to the ZooKeeper quorum as the suggested maximum time for a session. See http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions The client sends a requested timeout, the server responds with the timeout that it can give the client.", - "display_name": "ZooKeeper Session Timeout", - "name": "zookeeper_session_timeout", - "value": "60000" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the HBase user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that does not correspond to a host name, such as '_no_host'. Note: This property is used only if HBase REST/Thrift Server Authentication is enabled.", - "display_name": "HBase Proxy User Hosts", - "name": "hbase_proxy_user_hosts_list", - "value": "*" - }, - { - "desc": "Comma-delimited list of groups that you want to allow the HBase user to impersonate. The default '*' allows all groups. To disable entirely, use a string that does not correspond to a group name, such as '_no_group_'. Note: This property is used only if HBase REST/Thrift Server Authentication is enabled.", - "display_name": "HBase Proxy User Groups", - "name": "hbase_proxy_user_groups_list", - "value": "*" - }, - { - "desc": "\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n
\n\n\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n
\n\nEvent trackers are defined in a JSON object like the following:
\n\n\n{\n \"timeToLive\" : [integer],\n \"fields\" : [\n {\n \"type\" : [string],\n \"name\" : [string]\n }\n ]\n}\n\n\n\nWhere:\n
\n\n\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n
\n\n\nThe following is the list of fields that can used to compare HBase events:\n
\nThis file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "Balancer Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "balancer_config_safety_valve", - "value": null - }, - { - "desc": "The policy that should be used to rebalance HDFS storage. The default DataNode policy balances the storage at the DataNode level. This is similar to the balancing policy from prior releases. The BlockPool policy balances the storage at the block pool level as well as at the DataNode level. The BlockPool policy is relevant only to a Federated HDFS service.", - "display_name": "Rebalancing Policy", - "name": "rebalancing_policy", - "value": "DataNode" - }, - { - "desc": "The percentage deviation from average utilization, after which a node will be rebalanced. (for example, '10.0' for 10%)", - "display_name": "Rebalancing Threshold", - "name": "rebalancer_threshold", - "value": "10.0" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Balancer in Bytes", - "name": "balancer_java_heapsize", - "value": "1073741824" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-datanode.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-datanode.json deleted file mode 100644 index 7a8d8bc9..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hdfs-datanode.json +++ /dev/null @@ -1,440 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 5, \"content\":\"Datanode registration failed\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Got a command from standby NN - ignoring command:.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "Specifies the maximum number of threads to use for transferring data in and out of the DataNode.", - "display_name": "Maximum Number of Transfer Threads", - "name": "dfs_datanode_max_xcievers", - "value": "4096" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Port for DataNode's XCeiver Protocol. Combined with the DataNode's hostname to build its address.", - "display_name": "DataNode Transceiver Port", - "name": "dfs_datanode_port", - "value": "50010" - }, - { - "desc": "Comma-separated list of DataNode plug-ins to be activated. If one plug-in cannot be loaded, all the plug-ins are ignored.", - "display_name": "DateNode Plugins", - "name": "dfs_datanode_plugins_list", - "value": "" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "In some workloads, the data read from HDFS is known to be significantly large enough that it is unlikely to be useful to cache it in the operating system buffer cache. In this case, the DataNode may be configured to automatically purge all data from the buffer cache after it is delivered to the client. This may improve performance for some workloads by freeing buffer cache spare usage for more cacheable data. This behavior will always be disabled for workloads that read only short sections of a block (e.g HBase random-IO workloads). This property is supported in CDH3u3 or later deployments.", - "display_name": "Enable purging cache after reads", - "name": "dfs_datanode_drop_cache_behind_reads", - "value": "false" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Comma-delimited list of directories on the local file system where the DataNode stores HDFS block data. Typical values are /data/N/dfs/dn for N = 1, 2, 3... These directories should be mounted using the noatime option and the disks should be configured using JBOD. RAID is not recommended. Warning: be very careful when modifying this property. Removing or changing entries can result in data loss. If you want to hot swap drives in CDH 5.4+, override the value of this property for the specific DataNode role instance whose drive is to be hot-swapped; do not modify the property value in the role group. See Configuring Hot Swap for DataNodes for more important information and caveats.", - "display_name": "DataNode Data Directory", - "name": "dfs_data_dir_list", - "value": null - }, - { - "desc": "The health test thresholds for the weighted average extra time the pause monitor spent paused. Specified as a percentage of elapsed wall clock time.", - "display_name": "Pause Duration Thresholds", - "name": "datanode_pause_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "The number of volumes that are allowed to fail before a DataNode stops offering service. By default, any volume failure will cause a DataNode to shutdown.", - "display_name": "DataNode Failed Volumes Tolerated", - "name": "dfs_datanode_failed_volumes_tolerated", - "value": "0" - }, - { - "desc": "In some workloads, the data written to HDFS is known to be significantly large enough that it is unlikely to be useful to cache it in the operating system buffer cache. In this case, the DataNode may be configured to automatically purge all data from the buffer cache after it is written to disk. This may improve performance for some workloads by freeing buffer cache spare usage for more cacheable data. This property is supported in CDH3u3 or later deployments.", - "display_name": "Enable purging cache after writes", - "name": "dfs_datanode_drop_cache_behind_writes", - "value": "false" - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "If enabled, the DataNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind DataNode to Wildcard Address", - "name": "dfs_datanode_bind_wildcard", - "value": "false" - }, - { - "desc": "The number of server threads for the DataNode.", - "display_name": "Handler Count", - "name": "dfs_datanode_handler_count", - "value": "3" - }, - { - "desc": "When computing the overall DataNode health, consider the host's health.", - "display_name": "DataNode Host Health Test", - "name": "datanode_host_health_enabled", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "The maximum number of rolled log files to keep for DataNode logs. Typically used by log4j or logback.", - "display_name": "DataNode Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "DataNode Environment Advanced Configuration Snippet (Safety Valve)", - "name": "DATANODE_role_env_safety_valve", - "value": null - }, - { - "desc": "Timeout in seconds for the Hue Thrift server running on each DataNode", - "display_name": "Hue Thrift Server Timeout", - "name": "dfs_thrift_timeout", - "value": "60" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Whether DataNodes should use DataNode hostnames when connecting to DataNodes for data transfer. This property is supported in CDH3u4 or later deployments.", - "display_name": "Use DataNode Hostname", - "name": "dfs_datanode_use_datanode_hostname", - "value": "false" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for DataNode", - "name": "datanode_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "datanode_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "datanode_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Enables the health test that the DataNode's process state is consistent with the role configuration", - "display_name": "DataNode Process Health Test", - "name": "datanode_scm_health_enabled", - "value": "true" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "DataNode Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "DataNode Policy for picking which volume should get a new block. The Available Space policy is only available starting with CDH 4.3.", - "display_name": "DataNode Volume Choosing Policy", - "name": "dfs_datanode_volume_choosing_policy", - "value": "org.apache.hadoop.hdfs.server.datanode.fsdataset.RoundRobinVolumeChoosingPolicy" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Port for the DataNode HTTP web UI. Combined with the DataNode's hostname to build its HTTP address.", - "display_name": "DataNode HTTP Web UI Port", - "name": "dfs_datanode_http_port", - "value": "50075" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "Directory where DataNode will place its log files.", - "display_name": "DataNode Log Directory", - "name": "datanode_log_dir", - "value": "/var/log/hadoop-hdfs" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "DataNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "datanode_config_safety_valve", - "value": null - }, - { - "desc": "While reading block files, the DataNode can use the posix_fadvise system call to explicitly page data into the operating system buffer cache ahead of the current reader's position. This can improve performance especially when disks are highly contended. This configuration specifies the number of bytes ahead of the current read position which the DataNode will attempt to read ahead. A value of 0 disables this feature. This property is supported in CDH3u3 or later deployments.", - "display_name": "Number of read ahead bytes", - "name": "dfs_datanode_readahead_bytes", - "value": "4194304" - }, - { - "desc": "The maximum size, in megabytes, per log file for DataNode logs. Typically used by log4j or logback.", - "display_name": "DataNode Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The maximum amount of memory a DataNode may use to cache data blocks in memory. Setting it to zero will disable caching.", - "display_name": "Maximum Memory Used for Caching", - "name": "dfs_datanode_max_locked_memory", - "value": "4294967296" - }, - { - "desc": "Only used when the DataNode Volume Choosing Policy is set to Available Space. Controls how much DataNode volumes are allowed to differ in terms of bytes of free disk space before they are considered imbalanced. If the free space of all the volumes are within this range of each other, the volumes will be considered balanced and block assignments will be done on a pure round robin basis.", - "display_name": "Available Space Policy Balanced Threshold", - "name": "dfs_datanode_available_space_balanced_threshold", - "value": "10737418240" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds of free space in a DataNode. Specified as a percentage of the capacity on the DataNode.", - "display_name": "DataNode Free Space Monitoring Thresholds", - "name": "datanode_free_space_thresholds", - "value": "{\"critical\":\"10.0\",\"warning\":\"20.0\"}" - }, - { - "desc": "Port for the various DataNode Protocols. Combined with the DataNode's hostname to build its IPC port address.", - "display_name": "DataNode Protocol Port", - "name": "dfs_datanode_ipc_port", - "value": "50020" - }, - { - "desc": "The period to review when computing the moving average of extra time the pause monitor spent paused.", - "display_name": "Pause Duration Monitoring Period", - "name": "datanode_pause_duration_window", - "value": "5" - }, - { - "desc": "Minimum number of running threads for the Hue Thrift server running on each DataNode", - "display_name": "Hue Thrift Server Min Threadcount", - "name": "dfs_thrift_threads_min", - "value": "10" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "If this configuration is enabled, the DataNode will instruct the operating system to enqueue all written data to the disk immediately after it is written. This differs from the usual OS policy which may wait for up to 30 seconds before triggering writeback. This may improve performance for some workloads by smoothing the IO profile for data written to disk. This property is supported in CDH3u3 or later deployments.", - "display_name": "Enable immediate enqueuing of data to disk after writes", - "name": "dfs_datanode_sync_behind_writes", - "value": "false" - }, - { - "desc": "The health test thresholds of the number of blocks on a DataNode", - "display_name": "DataNode Block Count Thresholds", - "name": "datanode_block_count_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"500000.0\"}" - }, - { - "desc": "Permissions for the directories on the local file system where the DataNode stores its blocks. The permissions must be octal. 755 and 700 are typical values.", - "display_name": "DataNode Data Directory Permissions", - "name": "dfs_datanode_data_dir_perm", - "value": "700" - }, - { - "desc": "Reserved space in bytes per volume for non Distributed File System (DFS) use.", - "display_name": "Reserved Space for Non DFS Use", - "name": "dfs_datanode_du_reserved", - "value": "10737418240" - }, - { - "desc": "The amount of time to wait for the DataNode to fully start up and connect to the NameNode before enforcing the connectivity check.", - "display_name": "DataNode Connectivity Tolerance at Startup", - "name": "datanode_connectivity_tolerance", - "value": "180" - }, - { - "desc": "The base port where the secure DataNode web UI listens. Combined with the DataNode's hostname to build its secure web UI address.", - "display_name": "Secure DataNode Web UI Port (SSL)", - "name": "dfs_datanode_https_port", - "value": "50475" - }, - { - "desc": "Maximum amount of bandwidth that each DataNode can use for balancing. Specified in bytes per second.", - "display_name": "DataNode Balancing Bandwidth", - "name": "dfs_balance_bandwidthPerSec", - "value": "10485760" - }, - { - "desc": "Maximum number of running threads for the Hue Thrift server running on each DataNode", - "display_name": "Hue Thrift Server Max Threadcount", - "name": "dfs_thrift_threads_max", - "value": "20" - }, - { - "desc": "The health test thresholds of transceivers usage in a DataNode. Specified as a percentage of the total configured number of transceivers.", - "display_name": "DataNode Transceivers Usage Thresholds", - "name": "datanode_transceivers_usage_thresholds", - "value": "{\"critical\":\"95.0\",\"warning\":\"75.0\"}" - }, - { - "desc": "Only used when the DataNode Volume Choosing Policy is set to Available Space. Controls what percentage of new block allocations will be sent to volumes with more available disk space than others. This setting should be in the range 0.0 - 1.0, though in practice 0.5 - 1.0, since there should be no reason to prefer that volumes with less available disk space receive more block allocations.", - "display_name": "Available Space Policy Balanced Preference", - "name": "dfs_datanode_available_space_balanced_preference", - "value": "0.75" - }, - { - "desc": "The health test thresholds of failed volumes in a DataNode.", - "display_name": "DataNode Volume Failures Thresholds", - "name": "datanode_volume_failures_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "The minimum log level for DataNode logs", - "display_name": "DataNode Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "datanode_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of DataNode in Bytes", - "name": "datanode_java_heapsize", - "value": "1073741824" - }, - { - "desc": "Enables the health test that verifies the DataNode is connected to the NameNode", - "display_name": "DataNode Connectivity Health Test", - "name": "datanode_connectivity_health_enabled", - "value": "true" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-failovercontroller.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-failovercontroller.json deleted file mode 100644 index 42671459..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hdfs-failovercontroller.json +++ /dev/null @@ -1,230 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Directory where Failover Controller will place its log files.", - "display_name": "Failover Controller Log Directory", - "name": "failover_controller_log_dir", - "value": "/var/log/hadoop-hdfs" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" - }, - { - "desc": "The RPC timeout for the HA health monitor.", - "display_name": "HA Health Monitor RPC Timeout", - "name": "ha_health_monitor_rpc_timeout_ms", - "value": "45000" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "Failover Controller Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "fc_config_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for Failover Controller logs. Typically used by log4j or logback.", - "display_name": "Failover Controller Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Failover Controller in Bytes", - "name": "failover_controller_java_heapsize", - "value": "268435456" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Failover Controller Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Failover Controller Environment Advanced Configuration Snippet (Safety Valve)", - "name": "FAILOVERCONTROLLER_role_env_safety_valve", - "value": null - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When computing the overall Failover Controller health, consider the host's health.", - "display_name": "Failover Controller Host Health Test", - "name": "failovercontroller_host_health_enabled", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Failover Controller logs. Typically used by log4j or logback.", - "display_name": "Failover Controller Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "failovercontroller_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Enables the health test that the Failover Controller's process state is consistent with the role configuration", - "display_name": "Failover Controller Process Health Test", - "name": "failovercontroller_scm_health_enabled", - "value": "true" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Failover Controller", - "name": "failover_controller_java_opts", - "value": "" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The minimum log level for Failover Controller logs", - "display_name": "Failover Controller Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-gateway.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-gateway.json deleted file mode 100644 index 367af20f..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hdfs-gateway.json +++ /dev/null @@ -1,68 +0,0 @@ -[ - { - "desc": "For advanced use only, a string to be inserted into the client configuration for hdfs-site.xml.", - "display_name": "HDFS Client Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "hdfs_client_config_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Gateway Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The directory where the client configs will be deployed", - "display_name": "Deploy Directory", - "name": "client_config_root_dir", - "value": "/etc/hadoop" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the client configuration for hadoop-env.sh", - "display_name": "HDFS Client Environment Advanced Configuration Snippet for hadoop-env.sh (Safety Valve)", - "name": "hdfs_client_env_safety_valve", - "value": null - }, - { - "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", - "display_name": "Alternatives Priority", - "name": "client_config_priority", - "value": "90" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Maximum size in bytes for the Java process heap memory. Passed to Java -Xmx.", - "display_name": "Client Java Heap Size in Bytes", - "name": "hdfs_client_java_heapsize", - "value": "268435456" - }, - { - "desc": "Enable HDFS short circuit read. This allows a client co-located with the DataNode to read HDFS file blocks directly. This gives a performance boost to distributed clients that are aware of locality.", - "display_name": "Enable HDFS Short Circuit Read", - "name": "dfs_client_read_shortcircuit", - "value": "false" - }, - { - "desc": "These are Java command line arguments. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Client Java Configuration Options", - "name": "hbase_client_java_opts", - "value": "-Djava.net.preferIPv4Stack=true" - }, - { - "desc": "Move deleted files to the trash so that they can be recovered if necessary. This client side configuration takes effect only if the HDFS service-wide trash is disabled (NameNode Filesystem Trash Interval set to 0) and is ignored otherwise. The trash is not automatically emptied when enabled with this configuration.", - "display_name": "Use Trash", - "name": "dfs_client_use_trash", - "value": "false" - }, - { - "desc": "The minimum log level for Gateway logs", - "display_name": "Gateway Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-httpfs.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-httpfs.json deleted file mode 100644 index 6a523844..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hdfs-httpfs.json +++ /dev/null @@ -1,290 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Kerberos principal used by the HttpFS roles.", - "display_name": "Role-Specific Kerberos Principal", - "name": "kerberos_role_princ_name", - "value": "httpfs" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "httpfs_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The user that the HttpFS server process should run as.", - "display_name": "System User", - "name": "httpfs_process_username", - "value": "httpfs" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The group that the HttpFS server process should run as.", - "display_name": "System Group", - "name": "httpfs_process_groupname", - "value": "httpfs" - }, - { - "desc": "When computing the overall HttpFS health, consider the host's health.", - "display_name": "HttpFS Host Health Test", - "name": "httpfs_host_health_enabled", - "value": "true" - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for HttpFS logs. Typically used by log4j or logback.", - "display_name": "HttpFS Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Enables the health test that the HttpFS's process state is consistent with the role configuration", - "display_name": "HttpFS Process Health Test", - "name": "httpfs_scm_health_enabled", - "value": "true" - }, - { - "desc": "Password of the keystore used by HttpFS role for SSL.", - "display_name": "HttpFS Keystore Password", - "name": "httpfs_https_keystore_password", - "value": null - }, - { - "desc": "The password for the HttpFS TLS/SSL Certificate Trust Store File. Note that this password is not required to access the trust store: this field can be left blank. This password provides optional integrity checking of the file. The contents of trust stores are certificates, and certificates are public information.", - "display_name": "HttpFS TLS/SSL Certificate Trust Store Password", - "name": "httpfs_https_truststore_password", - "value": null - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "Directory where HttpFS will place its log files.", - "display_name": "HttpFS Log Directory", - "name": "httpfs_log_dir", - "value": "/var/log/hadoop-httpfs" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "HttpFS Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "For advanced use only, a string to be inserted into httpfs-site.xml for this role only.", - "display_name": "HttpFS Advanced Configuration Snippet (Safety Valve) for httpfs-site.xml", - "name": "httpfs_config_safety_valve", - "value": null - }, - { - "desc": "The maximum size, in megabytes, per log file for HttpFS logs. Typically used by log4j or logback.", - "display_name": "HttpFS Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for HttpFS", - "name": "httpfs_java_opts", - "value": "" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "HttpFS Environment Advanced Configuration Snippet (Safety Valve)", - "name": "HTTPFS_role_env_safety_valve", - "value": null - }, - { - "desc": "The secret to use for signing client authentication tokens.", - "display_name": "Signature Secret", - "name": "hdfs_httpfs_signature_secret", - "value": "hadoop httpfs secret" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Location of the keystore file used by HttpFS role for SSL.", - "display_name": "HttpFS Keystore File", - "name": "httpfs_https_keystore_file", - "value": "/var/run/hadoop-httpfs/.keystore" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of HttpFS in Bytes", - "name": "httpfs_java_heapsize", - "value": "268435456" - }, - { - "desc": "The port for the administration interface.", - "display_name": "Administration Port", - "name": "hdfs_httpfs_admin_port", - "value": "14001" - }, - { - "desc": "The location on disk of the trust store, in .jks format, used to confirm the authenticity of TLS/SSL servers that HttpFS might connect to. This is used when HttpFS is the client in a TLS/SSL connection. This trust store must contain the certificate(s) used to sign the service(s) being connected to. If this parameter is not provided, the default list of well-known certificate authorities is used instead.", - "display_name": "HttpFS TLS/SSL Certificate Trust Store File", - "name": "httpfs_https_truststore_file", - "value": null - }, - { - "desc": "Address of the load balancer used for HttpFS roles. Should be specified in host:port format. Note: Changing this property will regenerate Kerberos keytabs for all HttpFS roles.", - "display_name": "HttpFS Load Balancer", - "name": "httpfs_load_balancer", - "value": null - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "The port where the REST interface to HDFS is available. The REST interface is served over HTTPS if SSL is enabled for HttpFS, or over HTTP otherwise.", - "display_name": "REST Port", - "name": "hdfs_httpfs_http_port", - "value": "14000" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The minimum log level for HttpFS logs", - "display_name": "HttpFS Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Use SSL for HttpFS.", - "display_name": "Use SSL", - "name": "httpfs_use_ssl", - "value": "false" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-journalnode.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-journalnode.json deleted file mode 100644 index 5782c8d6..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hdfs-journalnode.json +++ /dev/null @@ -1,308 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of JournalNode in Bytes", - "name": "journalNode_java_heapsize", - "value": "268435456" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "journalnode_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "journalnode_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "Enables the health test that the JournalNode's process state is consistent with the role configuration", - "display_name": "JournalNode Process Health Test", - "name": "journalnode_scm_health_enabled", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Directory on the local file system where NameNode edits are written.", - "display_name": "JournalNode Edits Directory", - "name": "dfs_journalnode_edits_dir", - "value": null - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Directory on the local file system where NameNode edits are written..", - "display_name": "JournalNode Edits Directory Free Space Monitoring Absolute Thresholds", - "name": "journalnode_edits_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Enables the health check that verifies the active NameNode's sync status to the JournalNode", - "display_name": "Active NameNode Sync Status Health Check", - "name": "journalnode_sync_status_enabled", - "value": "true" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "When computing the overall JournalNode health, consider the host's health.", - "display_name": "JournalNode Host Health Test", - "name": "journalnode_host_health_enabled", - "value": "true" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The amount of time at JournalNode startup allowed for the active NameNode to get in sync with the JournalNode.", - "display_name": "Active NameNode Sync Status Startup Tolerance", - "name": "journalnode_sync_status_startup_tolerance", - "value": "180" - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The base port where the secure JournalNode web UI listens. Combined with the JournalNode's hostname to build its secure web UI address.", - "display_name": "Secure JournalNode Web UI Port (SSL)", - "name": "dfs_journalnode_https_port", - "value": "8481" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "journalnode_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "JournalNode Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The health test thresholds for JournalNode fsync latency.", - "display_name": "JournalNode Fsync Latency Thresholds", - "name": "journalnode_fsync_latency_thresholds", - "value": "{\"critical\":\"3000.0\",\"warning\":\"1000.0\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "JournalNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "jn_config_safety_valve", - "value": null - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "journalnode_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "Port for the JournalNode's RPC. Combined with the JournalNode's hostname to build its RPC address.", - "display_name": "JournalNode RPC Port", - "name": "dfs_journalnode_rpc_port", - "value": "8485" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for JournalNode logs. Typically used by log4j or logback.", - "display_name": "JournalNode Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "If enabled, the JournalNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind JournalNode to Wildcard Address", - "name": "journalnode_bind_wildcard", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "journalnode_gc_duration_window", - "value": "5" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Directory on the local file system where NameNode edits are written.. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Directory on the local file system where NameNode edits are written. Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "JournalNode Edits Directory Free Space Monitoring Percentage Thresholds", - "name": "journalnode_edits_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Directory where JournalNode will place its log files.", - "display_name": "JournalNode Log Directory", - "name": "journalnode_log_dir", - "value": "/var/log/hadoop-hdfs" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "JournalNode Environment Advanced Configuration Snippet (Safety Valve)", - "name": "JOURNALNODE_role_env_safety_valve", - "value": null - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for JournalNode", - "name": "journalNode_java_opts", - "value": "" - }, - { - "desc": "The maximum number of rolled log files to keep for JournalNode logs. Typically used by log4j or logback.", - "display_name": "JournalNode Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Port for the JournalNode HTTP web UI. Combined with the JournalNode hostname to build its HTTP address.", - "display_name": "JournalNode HTTP Port", - "name": "dfs_journalnode_http_port", - "value": "8480" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "The minimum log level for JournalNode logs", - "display_name": "JournalNode Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-namenode.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-namenode.json deleted file mode 100644 index 6d02ba2b..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hdfs-namenode.json +++ /dev/null @@ -1,548 +0,0 @@ -[ - { - "desc": "Name of the journal located on each JournalNode filesystem.", - "display_name": "Quorum-based Storage Journal name", - "name": "dfs_namenode_quorum_journal_name", - "value": null - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Indicate whether or not to avoid reading from stale DataNodes for which heartbeat messages have not been received by the NameNode for more than Stale DataNode Time Interval. Stale DataNodes are moved to the end of the node list returned for reading. See dfs.namenode.avoid.write.stale.datanode for a similar setting for writes.", - "display_name": "Avoid Reading Stale DataNode", - "name": "dfs_namenode_avoid_read_stale_datanode", - "value": "false" - }, - { - "desc": "The base port where the DFS NameNode web UI listens. If the port number is 0, then the server starts on a free port. Combined with the NameNode's hostname to build its HTTP address.", - "display_name": "NameNode Web UI Port", - "name": "dfs_http_port", - "value": "50070" - }, - { - "desc": "Enables the health test that the NameNode is not in safemode", - "display_name": "NameNode Safemode Health Test", - "name": "namenode_safe_mode_enabled", - "value": "true" - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "namenode_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "The health check thresholds of the NameNode's RPC latency.", - "display_name": "NameNode RPC Latency Thresholds", - "name": "namenode_rpc_latency_thresholds", - "value": "{\"critical\":\"5000.0\",\"warning\":\"1000.0\"}" - }, - { - "desc": "The period to review when computing the moving average of extra time the pause monitor spent paused.", - "display_name": "Pause Duration Monitoring Period", - "name": "namenode_pause_duration_window", - "value": "5" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The maximum number of outgoing replication threads a node can have at one time. This limit is waived for the highest priority replications. Configure dfs.namenode.replication.max-streams-hard-limit to set the absolute limit, including the highest-priority replications.", - "display_name": "Maximum Number of Replication Threads on a DataNode", - "name": "dfs_namenode_replication_max_streams", - "value": "20" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "NameNode Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "When the ratio of number stale DataNodes to total DataNodes marked is greater than this ratio, permit writing to stale nodes to prevent causing hotspots.", - "display_name": "Write Stale DataNode Ratio", - "name": "dfs_namenode_write_stale_datanode_ratio", - "value": "0.5" - }, - { - "desc": "The health test thresholds for the weighted average extra time the pause monitor spent paused. Specified as a percentage of elapsed wall clock time.", - "display_name": "Pause Duration Thresholds", - "name": "namenode_pause_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Enables the health test that the NameNode's process state is consistent with the role configuration", - "display_name": "NameNode Process Health Test", - "name": "namenode_scm_health_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "Enables the health test of the rolling metadata upgrade status of the NameNode. This covers rolling metadata upgrades. Nonrolling metadata upgrades are covered in a separate health test.", - "display_name": "HDFS Rolling Metadata Upgrade Status Health Test", - "name": "namenode_rolling_upgrade_status_enabled", - "value": "true" - }, - { - "desc": "Mountpoints that are mapped to this NameNode's Nameservice.", - "display_name": "Mountpoints", - "name": "nameservice_mountpoints", - "value": "/" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "namenode_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"},\n {\"alert\": false, \"rate\": 1, \"threshold\":\"INFO\", \"content\":\"Triggering checkpoint.*\"}\n ]\n}\n" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Namenode in Bytes", - "name": "namenode_java_heapsize", - "value": "1073741824" - }, - { - "desc": "The maximum size, in megabytes, per log file for NameNode logs. Typically used by log4j or logback.", - "display_name": "NameNode Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "Determines where on the local file system the NameNode should store the name table (fsimage). For redundancy, enter a comma-delimited list of directories to replicate the name table in all of the directories. Typical values are /data/N/dfs/nn where N=1..3.", - "display_name": "NameNode Data Directories", - "name": "dfs_name_dir_list", - "value": null - }, - { - "desc": "Default time interval for marking a DataNode as \"stale\". If the NameNode has not received heartbeat messages from a DataNode for more than this time interval, the DataNode is marked and treated as \"stale\" by default.", - "display_name": "Stale DataNode Time Interval", - "name": "dfs_namenode_stale_datanode_interval", - "value": "30000" - }, - { - "desc": "The port where the NameNode runs the HDFS protocol. Combined with the NameNode's hostname to build its address.", - "display_name": "NameNode Port", - "name": "namenode_port", - "value": "8020" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The period to review when computing the moving average of the NameNode's RPC latency.", - "display_name": "NameNode RPC Latency Monitoring Window", - "name": "namenode_rpc_latency_window", - "value": "5" - }, - { - "desc": "Optional port for the service-rpc address which can be used by HDFS daemons instead of sharing the RPC address used by the clients.", - "display_name": "NameNode Service RPC Port", - "name": "dfs_namenode_servicerpc_address", - "value": null - }, - { - "desc": "The health check thresholds for the number of out-of-sync JournalNodes for this NameNode.", - "display_name": "NameNode Out-Of-Sync JournalNodes Thresholds", - "name": "namenode_out_of_sync_journal_nodes_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Determines where on the local file system the NameNode should store the name table (fsimage). For redundancy, enter a comma-delimited list of directories to replicate the name table in all of the directories. Typical values are /data/N/dfs/nn where N=1..3..", - "display_name": "NameNode Data Directories Free Space Monitoring Absolute Thresholds", - "name": "namenode_data_directories_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The absolute maximum number of outgoing replication threads a given node can have at one time. The regular limit (dfs.namenode.replication.max-streams) is waived for highest-priority block replications. Highest replication priority is for blocks that are at a very high risk of loss if the disk or server on which they remain fails. These are usually blocks with only one copy, or blocks with zero live copies but a copy in a node being decommissioned. dfs.namenode.replication.max-streams-hard-limit provides a limit on the total number of outgoing replication threads, including threads of all priorities.", - "display_name": "Hard Limit on the Number of Replication Threads on a Datanode", - "name": "dfs_namenode_replication_max_streams_hard_limit", - "value": "40" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "NameNode Environment Advanced Configuration Snippet (Safety Valve)", - "name": "NAMENODE_role_env_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Directory where NameNode will place its log files.", - "display_name": "NameNode Log Directory", - "name": "namenode_log_dir", - "value": "/var/log/hadoop-hdfs" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Enables the health test of the metadata upgrade status of the NameNode. This covers nonrolling metadata upgrades. Rolling metadata upgrades are covered in a separate health test.", - "display_name": "HDFS Metadata Upgrade Status Health Test", - "name": "namenode_upgrade_status_enabled", - "value": "true" - }, - { - "desc": "Nameservice of this NameNode. The Nameservice represents the interface to this NameNode and its High Availability partner. The Nameservice also represents the namespace associated with a federated NameNode.", - "display_name": "NameNode Nameservice", - "name": "dfs_federation_namenode_nameservice", - "value": null - }, - { - "desc": "The minimum log level for NameNode logs", - "display_name": "NameNode Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Specifies the percentage of blocks that should satisfy the minimal replication requirement defined by dfs.replication.min. Enter a value less than or equal to 0 to wait for any particular percentage of blocks before exiting safemode. Values greater than 1 will make safemode permanent.", - "display_name": "Safemode Threshold Percentage", - "name": "dfs_safemode_threshold_pct", - "value": "0.999" - }, - { - "desc": "Directories on the local file system to store the NameNode edits. If not set, the edits are stored in the NameNode's Data Directories. The value of this configuration is automatically generated to be the Quorum-based Storage URI if there are JournalNodes and this NameNode is not Highly Available.", - "display_name": "NameNode Edits Directories", - "name": "dfs_namenode_edits_dir", - "value": null - }, - { - "desc": "If set to false and if one of the replicas of the NameNode storage fails, such as temporarily failure of NFS, this directory is not used until the NameNode restarts. If enabled, failed storage is re-checked on every checkpoint and, if it becomes valid, the NameNode will try to restore the edits and fsimage.", - "display_name": "Restore NameNode Directories at Checkpoint Time", - "name": "dfs_name_dir_restore", - "value": "false" - }, - { - "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", - "display_name": "Health Check Startup Tolerance", - "name": "namenode_startup_tolerance", - "value": "5" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for NameNode", - "name": "namenode_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "Enable Automatic Failover to maintain High Availability. Requires a ZooKeeper service and a High Availability NameNode partner.", - "display_name": "Enable Automatic Failover", - "name": "autofailover_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Determines where on the local file system the NameNode should store the name table (fsimage). For redundancy, enter a comma-delimited list of directories to replicate the name table in all of the directories. Typical values are /data/N/dfs/nn where N=1..3.. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Determines where on the local file system the NameNode should store the name table (fsimage). For redundancy, enter a comma-delimited list of directories to replicate the name table in all of the directories. Typical values are /data/N/dfs/nn where N=1..3. Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "NameNode Data Directories Free Space Monitoring Percentage Thresholds", - "name": "namenode_data_directories_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into dfs_hosts_exclude.txt for this role only.", - "display_name": "NameNode Advanced Configuration Snippet (Safety Valve) for dfs_hosts_exclude.txt", - "name": "namenode_hosts_exclude_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "This determines the percentage amount of block invalidations (deletes) to do over a single DataNode heartbeat deletion command. The final deletion count is determined by applying this percentage to the number of live nodes in the system. The resultant number is the number of blocks from the deletion list chosen for proper invalidation over a single heartbeat of a single DataNode.", - "display_name": "Invalidate Work Percentage Per Iteration", - "name": "dfs_namenode_invalidate_work_pct_per_iteration", - "value": "0.32" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "namenode_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The time between two periodic file system checkpoints.", - "display_name": "Filesystem Checkpoint Period", - "name": "fs_checkpoint_period", - "value": "3600" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "When computing the overall NameNode health, consider the host's health.", - "display_name": "NameNode Host Health Test", - "name": "namenode_host_health_enabled", - "value": "true" - }, - { - "desc": "The access time for HDFS file is precise upto this value. Setting the value of 0 disables access times for HDFS. When using the NFS Gateway role, make sure this property is enabled.", - "display_name": "Access Time Precision", - "name": "dfs_access_time_precision", - "value": "3600000" - }, - { - "desc": "Minimum number of running threads for the Hue Thrift server running on the NameNode", - "display_name": "Hue Thrift Server Min Threadcount", - "name": "dfs_thrift_threads_min", - "value": "10" - }, - { - "desc": "The number of transactions after which the NameNode or SecondaryNameNode will create a checkpoint of the namespace, regardless of whether the checkpoint period has expired.", - "display_name": "Filesystem Checkpoint Transaction Threshold", - "name": "fs_checkpoint_txns", - "value": "1000000" - }, - { - "desc": "Maximum number of running threads for the Hue Thrift server running on the NameNode", - "display_name": "Hue Thrift Server Max Threadcount", - "name": "dfs_thrift_threads_max", - "value": "20" - }, - { - "desc": "Full path to a custom topology script on the host file system. The topology script is used to determine the rack location of nodes. If left blank, a topology script will be provided that uses your hosts' rack information, visible in the \"Hosts\" page.", - "display_name": "Topology Script File Name", - "name": "topology_script_file_name", - "value": null - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "Indicate whether or not to avoid writing to stale DataNodes for which heartbeat messages have not been received by the NameNode for more than Stale DataNode Time Interval. Writes avoid using stale DataNodes unless more than a configured ratio (dfs.namenode.write.stale.datanode.ratio) of DataNodes are marked as stale. See dfs.namenode.avoid.read.stale.datanode for a similar setting for reads.", - "display_name": "Avoid Writing Stale DataNode", - "name": "dfs_namenode_avoid_write_stale_datanode", - "value": "false" - }, - { - "desc": "The health test thresholds of failed status directories in a NameNode.", - "display_name": "NameNode Directory Failures Thresholds", - "name": "namenode_directory_failures_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Number of minutes between trash checkpoints. Also controls the number of minutes after which a trash checkpoint directory is deleted. To disable the trash feature, enter 0.", - "display_name": "Filesystem Trash Interval", - "name": "fs_trash_interval", - "value": "1440" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "NameNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "namenode_config_safety_valve", - "value": null - }, - { - "desc": "This determines the total amount of block transfers to begin in parallel at a DataNode for replication, when such a command list is being sent over a DataNode heartbeat by the NameNode. The actual number is obtained by multiplying this value by the total number of live nodes in the cluster. The result number is the number of blocks to transfer immediately, per DataNode heartbeat.", - "display_name": "Replication Work Multiplier Per Iteration", - "name": "dfs_namenode_replication_work_multiplier_per_iteration", - "value": "10" - }, - { - "desc": "The maximum number of rolled log files to keep for NameNode logs. Typically used by log4j or logback.", - "display_name": "NameNode Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "Determines extension of safemode in milliseconds after the threshold level is reached.", - "display_name": "Safemode Extension", - "name": "dfs_safemode_extension", - "value": "30000" - }, - { - "desc": "Timeout in seconds for the Hue Thrift server running on the NameNode", - "display_name": "Hue Thrift Server Timeout", - "name": "dfs_thrift_timeout", - "value": "60" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The number of server threads for the NameNode used for service calls. Only used when NameNode Service RPC Port is configured.", - "display_name": "NameNode Service Handler Count", - "name": "dfs_namenode_service_handler_count", - "value": "30" - }, - { - "desc": "The health test thresholds of the number of transactions since the last HDFS namespace checkpoint. Specified as a percentage of the configured checkpointing transaction limit.", - "display_name": "Filesystem Checkpoint Transactions Monitoring Thresholds", - "name": "namenode_checkpoint_transactions_thresholds", - "value": "{\"critical\":\"400.0\",\"warning\":\"200.0\"}" - }, - { - "desc": "The health test thresholds of the age of the HDFS namespace checkpoint. Specified as a percentage of the configured checkpoint interval.", - "display_name": "Filesystem Checkpoint Age Monitoring Thresholds", - "name": "namenode_checkpoint_age_thresholds", - "value": "{\"critical\":\"400.0\",\"warning\":\"200.0\"}" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "The base port where the secure NameNode web UI listens.", - "display_name": "Secure NameNode Web UI Port (SSL)", - "name": "dfs_https_port", - "value": "50470" - }, - { - "desc": "For advanced use only, a string to be inserted into dfs_hosts_allow.txt for this role only.", - "display_name": "NameNode Advanced Configuration Snippet (Safety Valve) for dfs_hosts_allow.txt", - "name": "namenode_hosts_allow_safety_valve", - "value": null - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "If enabled, the NameNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind NameNode to Wildcard Address", - "name": "namenode_bind_wildcard", - "value": "false" - }, - { - "desc": "Comma-separated list of NameNode plug-ins to be activated. If one plug-in cannot be loaded, all the plug-ins are ignored.", - "display_name": "NameNode Plugins", - "name": "dfs_namenode_plugins_list", - "value": "" - }, - { - "desc": "Directory on a shared storage device, such as a Quorum-based Storage URI or a local directory that is an NFS mount from a NAS, to store the NameNode edits. The value of this configuration is automatically generated to be the Quourm Journal URI if there are JournalNodes and this NameNode is Highly Available.", - "display_name": "Shared Edits Directory", - "name": "dfs_namenode_shared_edits_dir", - "value": null - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The number of server threads for the NameNode.", - "display_name": "NameNode Handler Count", - "name": "dfs_namenode_handler_count", - "value": "30" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "Specifies the number of DataNodes that must be live before the name node exits safemode. Enter a value less than or equal to 0 to take the number of live DataNodes into account when deciding whether to remain in safemode during startup. Values greater than the number of DataNodes in the cluster will make safemode permanent.", - "display_name": "Safemode Minimum DataNodes", - "name": "dfs_safemode_min_datanodes", - "value": "0" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-nfsgateway.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-nfsgateway.json deleted file mode 100644 index f92bb37e..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hdfs-nfsgateway.json +++ /dev/null @@ -1,254 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "nfsgateway_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "NFS clients often reorder writes. As a result, sequential writes can arrive at the NFS Gateway in random order. This directory is used to temporarily save out-of-order writes before writing to HDFS. For each file, the out-of-order writes are dumped after they are accumulated to exceed certain threshold (e.g., 1MB) in memory. Please make sure this directory has enough space. For example, if the application uploads 10 files with each having 100MB, it is recommended that this directory have roughly 1GB of space in case write reorder happens (in the worst case) to every file.", - "display_name": "Temporary Dump Directory", - "name": "dfs_nfs3_dump_dir", - "value": "/tmp/.hdfs-nfs" - }, - { - "desc": "Enables the health test that the NFS Gateway's process state is consistent with the role configuration", - "display_name": "NFS Gateway Process Health Test", - "name": "nfsgateway_scm_health_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of NFS Gateway in Bytes", - "name": "nfsgateway_java_heapsize", - "value": "268435456" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "NFS Gateway Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "nfsgateway_config_safety_valve", - "value": null - }, - { - "desc": "The port number of the system portmap or rpcbind service. This configuration is used by Cloudera Manager to verify if the system portmap or rpcbind service is running before starting NFS Gateway role. Cloudera Manager does not manage the system portmap or rpcbind service.", - "display_name": "Portmap (or Rpcbind) Port", - "name": "nfs3_portmap_port", - "value": "111" - }, - { - "desc": "The NFS Gateway server port.", - "display_name": "NFS Gateway Server Port", - "name": "nfs3_server_port", - "value": "2049" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for NFS Gateway", - "name": "nfsgateway_java_opts", - "value": "" - }, - { - "desc": "The maximum number of rolled log files to keep for NFS Gateway logs. Typically used by log4j or logback.", - "display_name": "NFS Gateway Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "When computing the overall NFS Gateway health, consider the host's health.", - "display_name": "NFS Gateway Host Health Test", - "name": "nfsgateway_host_health_enabled", - "value": "true" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "By default, NFS Gateway exported directories can be mounted by any client. For better access control, update this property with a list of host names and access privileges separated by whitespace characters. Host name format can be a single host, a Java regular expression, or an IPv4 address. The access privilege uses rw to specify readwrite and ro to specify readonly access. If the access privilege is not provided, the default is read-only. Examples of host name format and access privilege: \"192.168.0.0/22 rw\", \"host.*.example.com\", \"host1.test.org ro\".", - "display_name": "Allowed Hosts and Privileges", - "name": "dfs_nfs_exports_allowed_hosts", - "value": "* rw" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The port number of the mount daemon implemented inside the NFS Gateway server role.", - "display_name": "NFS Gateway MountD Port", - "name": "nfs3_mountd_port", - "value": "4242" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "NFS Gateway Environment Advanced Configuration Snippet (Safety Valve)", - "name": "NFSGATEWAY_role_env_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "NFS Gateway Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The maximum size, in megabytes, per log file for NFS Gateway logs. Typically used by log4j or logback.", - "display_name": "NFS Gateway Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The minimum log level for NFS Gateway logs", - "display_name": "NFS Gateway Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Directory where NFS Gateway will place its log files.", - "display_name": "NFS Gateway Log Directory", - "name": "nfsgateway_log_dir", - "value": "/var/log/hadoop-hdfs" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-secondarynamenode.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-secondarynamenode.json deleted file mode 100644 index f4ecd63b..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hdfs-secondarynamenode.json +++ /dev/null @@ -1,308 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.IOException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.net.SocketClosedException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.io.EOFException\"},\n {\"alert\": false, \"rate\": 0, \"exceptiontype\": \"java.nio.channels.CancelledKeyException\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Unknown job [^ ]+ being deleted.*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"Error executing shell command .+ No such process.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\".*attempt to override final parameter.+\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\":\"[^ ]+ is a deprecated filesystem name. Use.*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"},\n {\"alert\": false, \"rate\": 1, \"threshold\":\"INFO\", \"content\":\"Triggering checkpoint.*\"}\n ]\n}\n" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Secondary NameNode", - "name": "secondarynamenode_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Determines where on the local file system the DFS SecondaryNameNode should store the temporary images to merge. For redundancy, enter a comma-delimited list of directories to replicate the image in all of the directories. Typical values are /data/N/dfs/snn for N = 1, 2, 3....", - "display_name": "HDFS Checkpoint Directories Free Space Monitoring Absolute Thresholds", - "name": "secondarynamenode_checkpoint_directories_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Secondary namenode in Bytes", - "name": "secondary_namenode_java_heapsize", - "value": "1073741824" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The number of transactions after which the NameNode or SecondaryNameNode will create a checkpoint of the namespace, regardless of whether the checkpoint period has expired.", - "display_name": "Filesystem Checkpoint Transaction Threshold", - "name": "fs_checkpoint_txns", - "value": "1000000" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "secondarynamenode_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "SecondaryNameNode Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "secondarynamenode_config_safety_valve", - "value": null - }, - { - "desc": "Enables the health test that the SecondaryNameNode's process state is consistent with the role configuration", - "display_name": "SecondaryNameNode Process Health Test", - "name": "secondarynamenode_scm_health_enabled", - "value": "true" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "secondarynamenode_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "SecondaryNameNode Environment Advanced Configuration Snippet (Safety Valve)", - "name": "SECONDARYNAMENODE_role_env_safety_valve", - "value": null - }, - { - "desc": "If enabled, the SecondaryNameNode binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind SecondaryNameNode to Wildcard Address", - "name": "secondary_namenode_bind_wildcard", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "SecondaryNameNode Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The time between two periodic file system checkpoints.", - "display_name": "Filesystem Checkpoint Period", - "name": "fs_checkpoint_period", - "value": "3600" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "secondarynamenode_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Determines where on the local file system the DFS SecondaryNameNode should store the temporary images to merge. For redundancy, enter a comma-delimited list of directories to replicate the image in all of the directories. Typical values are /data/N/dfs/snn for N = 1, 2, 3.... Specified as a percentage of the capacity on that filesystem. This setting is not used if a Determines where on the local file system the DFS SecondaryNameNode should store the temporary images to merge. For redundancy, enter a comma-delimited list of directories to replicate the image in all of the directories. Typical values are /data/N/dfs/snn for N = 1, 2, 3... Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "HDFS Checkpoint Directories Free Space Monitoring Percentage Thresholds", - "name": "secondarynamenode_checkpoint_directories_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "When computing the overall SecondaryNameNode health, consider the host's health.", - "display_name": "SecondaryNameNode Host Health Test", - "name": "secondarynamenode_host_health_enabled", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for SecondaryNameNode logs. Typically used by log4j or logback.", - "display_name": "SecondaryNameNode Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "secondarynamenode_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "secondarynamenode_gc_duration_window", - "value": "5" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Determines where on the local file system the DFS SecondaryNameNode should store the temporary images to merge. For redundancy, enter a comma-delimited list of directories to replicate the image in all of the directories. Typical values are /data/N/dfs/snn for N = 1, 2, 3...", - "display_name": "HDFS Checkpoint Directories", - "name": "fs_checkpoint_dir_list", - "value": null - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "The maximum number of rolled log files to keep for SecondaryNameNode logs. Typically used by log4j or logback.", - "display_name": "SecondaryNameNode Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "Nameservice of this SecondaryNameNode", - "display_name": "SecondaryNameNode Nameservice", - "name": "dfs_secondarynamenode_nameservice", - "value": null - }, - { - "desc": "The minimum log level for SecondaryNameNode logs", - "display_name": "SecondaryNameNode Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "The base port where the secure SecondaryNameNode web UI listens.", - "display_name": "Secure SecondaryNameNode Web UI Port (SSL)", - "name": "dfs_secondary_https_port", - "value": "50495" - }, - { - "desc": "Directory where SecondaryNameNode will place its log files.", - "display_name": "SecondaryNameNode Log Directory", - "name": "secondarynamenode_log_dir", - "value": "/var/log/hadoop-hdfs" - }, - { - "desc": "The SecondaryNameNode HTTP port. If the port is 0, then the server starts on a free port. Combined with the SecondaryNameNode's hostname to build its HTTP address.", - "display_name": "SecondaryNameNode Web UI Port", - "name": "dfs_secondary_http_port", - "value": "50090" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hdfs-service.json b/sahara/plugins/cdh/v5_4_0/resources/hdfs-service.json deleted file mode 100644 index 6d34b4ca..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hdfs-service.json +++ /dev/null @@ -1,746 +0,0 @@ -[ - { - "desc": "Timeout in milliseconds for the parallel RPCs made in DistributedFileSystem#getFileBlockStorageLocations(). This value is only emitted for Impala.", - "display_name": "HDFS File Block Storage Location Timeout", - "name": "dfs_client_file_block_storage_locations_timeout", - "value": "10000" - }, - { - "desc": "The domain to use for the HTTP cookie that stores the authentication token. In order for authentiation to work correctly across all Hadoop nodes' web-consoles the domain must be correctly set. Important: when using IP addresses, browsers ignore cookies with domain settings. For this setting to work properly all nodes in the cluster must be configured to generate URLs with hostname.domain names on it.", - "display_name": "Hadoop HTTP Authentication Cookie Domain", - "name": "hadoop_http_auth_cookie_domain", - "value": "" - }, - { - "desc": "The user that this service's processes should run as (except the HttpFS server, which has its own user)", - "display_name": "System User", - "name": "process_username", - "value": "hdfs" - }, - { - "desc": "Event filters are defined in a JSON object like the following:
\n\n\n{\n \"defaultAction\" : (\"accept\", \"discard\"),\n \"rules\" : [\n {\n \"action\" : (\"accept\", \"discard\"),\n \"fields\" : [\n {\n \"name\" : \"fieldName\",\n \"match\" : \"regex\"\n }\n ]\n }\n ]\n}\n\n\n\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n
\n\n\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n
\n\n\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n
\n\n\nThe following is the list of fields that can be filtered for HDFS events:\n
\n\nNote: Do not edit this property in the classic layout. Switch to the new layout to be able to use preconfigured redaction rules and test your rules inline.
Use this property to define a list of rules to be followed for redacting sensitive information from log files and query strings. Click + to add a new redaction rule. You can choose between one of the preconfigured rules or add a custom rule. When specifying a custom rule, the Search field should contain a regular expression that will be matched against the data. If a match is found, it will be replaced by the contents of the Replace field.
Trigger is an optional field. It can be used to specify a simple string which will be searched in the data. If the string is found, only then will the redactor attempt to find a match for the Search regex. If no Trigger is specified, redaction will always be carried out by matching the Search regular expression. The Trigger field exists solely for performance reasons: simple string matching is faster than regular expression matching.
Test your rules by entering sample text into the Test Redaction Rules text box and click Test Redaction. If no rules match, the text you entered will be returned unchanged.
", - "display_name": "Log and Query Redaction Policy", - "name": "redaction_policy", - "value": null - }, - { - "desc": "Comma-delimited list of groups to allow the HttpFS user to impersonate. The default '*' allows all groups. To disable entirely, use a string that does not correspond to a group name, such as '_no_group_'.", - "display_name": "HttpFS Proxy User Groups", - "name": "httpfs_proxy_user_groups_list", - "value": "*" - }, - { - "desc": "Allows the flume user to impersonate any members of a comma-delimited list of groups. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", - "display_name": "Flume Proxy User Groups", - "name": "flume_proxy_user_groups_list", - "value": "*" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the mapred user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", - "display_name": "Mapred Proxy User Hosts", - "name": "mapred_proxy_user_hosts_list", - "value": "*" - }, - { - "desc": "For advanced use only, a list of configuration properties that will be used by the Service Monitor instead of the current client configuration for the service.", - "display_name": "Service Monitor Client Config Overrides", - "name": "smon_client_config_overrides", - "value": "The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "Enable SSL encryption for HDFS, MapReduce, and YARN web UIs, as well as encrypted shuffle for MapReduce and Yarn.", - "display_name": "Hadoop SSL Enabled", - "name": "hdfs_hadoop_ssl_enabled", - "value": "false" - }, - { - "desc": "List of Kerberos realms that Hadoop services should trust. If empty, defaults to the default_realm property configured in the krb5.conf file. After changing this value and restarting the service, all services depending on this service must also be restarted. Adds mapping rules for each domain to the hadoop.security.auth_to_local property in core-site.xml.", - "display_name": "Trusted Kerberos Realms", - "name": "trusted_realms", - "value": "" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "Enables the health check that verifies that the failover controllers associated with this service are healthy and running.", - "display_name": "Failover Controllers Healthy", - "name": "failover_controllers_healthy_enabled", - "value": "true" - }, - { - "desc": "The attribute of the group object that identifies the users that are members of the group. The default will usually be appropriate for any LDAP installation.", - "display_name": "Hadoop User Group Mapping LDAP Group Membership Attribute", - "name": "hadoop_group_mapping_ldap_member_attr", - "value": "member" - }, - { - "desc": "Comma separated list of users allowed to do short circuit read. A short circuit read allows a client co-located with the data to read HDFS file blocks directly from HDFS. If empty, will default to the DataNode process' user.", - "display_name": "DataNode Local Path Access Users", - "name": "dfs_block_local_path_access_user", - "value": null - }, - { - "desc": "The timeout, in milliseconds, to use with the Cloudera Manager agent-based fencer.", - "display_name": "Timeout for Cloudera Manager Fencing Strategy", - "name": "dfs_ha_fencing_cloudera_manager_timeout_millis", - "value": "10000" - }, - { - "desc": "Enable collection of audit events from the service's roles.", - "display_name": "Enable Audit Collection", - "name": "navigator_audit_enabled", - "value": "true" - }, - { - "desc": "Maximum bandwidth used for image transfer in bytes per second. This can help keep normal NameNode operations responsive during checkpointing. A default value of 0 indicates that throttling is disabled.", - "display_name": "FsImage Transfer Bandwidth", - "name": "dfs_image_transfer_bandwidthPerSec", - "value": "0" - }, - { - "desc": "The user the management services impersonates when connecting to HDFS. If no value is specified, the HDFS superuser is used.", - "display_name": "HDFS User to Impersonate", - "name": "hdfs_user_to_impersonate", - "value": null - }, - { - "desc": "The short name of the Hue Kerberos principal. Normally, you do not need to specify this configuration. Cloudera Manager auto-configures this property so that Hue and Cloudera Manamgent Service work properly.", - "display_name": "Hue's Kerberos Principal Short Name", - "name": "hue_kerberos_principal_shortname", - "value": null - }, - { - "desc": "File path to the SSL keystore containing the SSL certificate required by the LDAP server.", - "display_name": "Hadoop User Group Mapping LDAP SSL Keystore", - "name": "hadoop_group_mapping_ldap_keystore", - "value": "" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the HDFS user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", - "display_name": "HDFS Proxy User Hosts", - "name": "hdfs_proxy_user_hosts_list", - "value": "*" - }, - { - "desc": "The minimal block replication.", - "display_name": "Minimal Block Replication", - "name": "dfs_replication_min", - "value": "1" - }, - { - "desc": "The maximal block replication.", - "display_name": "Maximal Block Replication", - "name": "dfs_replication_max", - "value": "512" - }, - { - "desc": "The service monitor will use these permissions to create the directory and files to test if the hdfs service is healthy. Permissions are specified using the 10-character unix-symbolic format e.g. '-rwxr-xr-x'.", - "display_name": "HDFS Health Canary Directory Permissions", - "name": "firehose_hdfs_canary_directory_permissions", - "value": "-rwxrwxrwx" - }, - { - "desc": "Enable authorization", - "display_name": "Hadoop Secure Authorization", - "name": "hadoop_security_authorization", - "value": "false" - }, - { - "desc": "The Key Management Server used by HDFS. This must be set to use encryption for data at rest.", - "display_name": "KMS Service", - "name": "kms_service", - "value": null - }, - { - "desc": "The attribute of the group object that identifies the group name. The default will usually be appropriate for all LDAP systems.", - "display_name": "Hadoop User Group Mapping LDAP Group Name Attribute", - "name": "hadoop_group_mapping_ldap_group_name_attr", - "value": "cn" - }, - { - "desc": "Enables DataNode support for the experimental DistributedFileSystem.getFileVBlockStorageLocations API. Applicable to CDH 4.1 and onwards.", - "display_name": "Enable HDFS Block Metadata API", - "name": "dfs_datanode_hdfs_blocks_metadata_enabled", - "value": "true" - }, - { - "desc": "The tolerance window that will be used in HDFS service tests that depend on detection of the active NameNode.", - "display_name": "Active NameNode Detection Window", - "name": "hdfs_active_namenode_detecton_window", - "value": "3" - }, - { - "desc": "Default block replication. The number of replications to make when the file is created. The default value is used if a replication number is not specified.", - "display_name": "Replication Factor", - "name": "dfs_replication", - "value": "3" - }, - { - "desc": "Comma-delimited list of groups that you want to allow the HTTP user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'. This is used by WebHCat.", - "display_name": "HTTP Proxy User Groups", - "name": "HTTP_proxy_user_groups_list", - "value": "*" - }, - { - "desc": "The name of the system group shared by all the core Hadoop services.", - "display_name": "Shared Hadoop Group Name", - "name": "hdfs_hadoop_group_name", - "value": "hadoop" - }, - { - "desc": "For paths where authorization is enforced by Sentry Synchronization, file permissions will use this parameter as the group. This group should normally include the hive and impala users.", - "display_name": "Sentry Authorization Provider Group", - "name": "sentry_authorization_provider_hdfs_group", - "value": "hive" - }, - { - "desc": "ACLs (Access Control Lists) enhance the existing HDFS permission model to support controlling file access for arbitrary combinations of users and groups instead of a single owner, single group, and all other users. When ACLs are disabled, the NameNode rejects all attempts to set an ACL.", - "display_name": "Enable Access Control Lists", - "name": "dfs_namenode_acls_enabled", - "value": "false" - }, - { - "desc": "An additional filter to use when searching for LDAP users. The default will usually be appropriate for Active Directory installations. If connecting to a generic LDAP server, ''sAMAccountName'' will likely be replaced with ''uid''. {0} is a special string used to denote where the username fits into the filter.", - "display_name": "Hadoop User Group Mapping LDAP User Search Filter", - "name": "hadoop_group_mapping_ldap_user_filter", - "value": "(&(objectClass=user)(sAMAccountName={0}))" - }, - { - "desc": "List of fencing methods to use for service fencing. shell(./cloudera_manager_agent_fencer.py) is a fencing mechanism designed to use the Cloudera Manager agent. The sshfence method uses SSH. If using custom fencers (that may communicate with shared store, power units, or network switches), use the shell to invoke them.", - "display_name": "HDFS High Availability Fencing Methods", - "name": "dfs_ha_fencing_methods", - "value": "shell(./cloudera_manager_agent_fencer.py)" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the environment of HDFS replication jobs.", - "display_name": "HDFS Replication Advanced Configuration Snippet (Safety Valve)", - "name": "hdfs_replication_env_safety_valve", - "value": null - }, - { - "desc": "The default block size in bytes for new HDFS files. Note that this value is also used as the HBase Region Server HLog block size.", - "display_name": "HDFS Block Size", - "name": "dfs_block_size", - "value": "134217728" - }, - { - "desc": "Path on the DataNode's local file system to a UNIX domain socket used for communication between the DataNode and local HDFS clients. This socket is used for Short Circuit Reads. Only the HDFS System User and \"root\" should have write access to the parent directory and all of its ancestors. This property is supported in CDH 4.2 or later deployments.", - "display_name": "UNIX Domain Socket path", - "name": "dfs_domain_socket_path", - "value": "/var/run/hdfs-sockets/dn" - }, - { - "desc": "Algorithm to encrypt data transfer between DataNodes and clients, and among DataNodes. 3des is more cryptographically secure, but rc4 is substantially faster.", - "display_name": "Data Transfer Encryption Algorithm", - "name": "dfs_encrypt_data_transfer_algorithm", - "value": "rc4" - }, - { - "desc": "The health check thresholds of the number of under-replicated blocks. Specified as a percentage of the total number of blocks.", - "display_name": "Under-replicated Block Monitoring Thresholds", - "name": "hdfs_under_replicated_blocks_thresholds", - "value": "{\"critical\":\"40.0\",\"warning\":\"10.0\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml. Applies to configurations of all roles in this service except client configuration.", - "display_name": "HDFS Service Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml", - "name": "hdfs_service_config_safety_valve", - "value": null - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the HTTP user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'. This is used by WebHCat.", - "display_name": "HTTP Proxy User Hosts", - "name": "HTTP_proxy_user_hosts_list", - "value": "*" - }, - { - "desc": "\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n
\n\n\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n
\n\nEvent trackers are defined in a JSON object like the following:
\n\n\n{\n \"timeToLive\" : [integer],\n \"fields\" : [\n {\n \"type\" : [string],\n \"name\" : [string]\n }\n ]\n}\n\n\n\nWhere:\n
\n\n\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n
\n\n\nThe following is the list of fields that can be used to compare HDFS events:\n
\n\nThe configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "Minimum number of worker threads in the Hive Metastore Server's thread pool", - "display_name": "Min Hive Metastore Server Threads", - "name": "hive_metastore_min_threads", - "value": "200" - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "When computing the overall Hive Metastore Server health, consider the host's health.", - "display_name": "Hive Metastore Server Host Health Test", - "name": "hivemetastore_host_health_enabled", - "value": "true" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Hive Metastore Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Hive Metastore Server in Bytes", - "name": "hive_metastore_java_heapsize", - "value": "1073741824" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "hivemetastore_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Port on which Hive Metastore Server will listen for connections.", - "display_name": "Hive Metastore Server Port", - "name": "hive_metastore_port", - "value": "9083" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", - "display_name": "Hive Metastore Server Advanced Configuration Snippet (Safety Valve) for hive-site.xml", - "name": "hive_metastore_config_safety_valve", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Hive Metastore Server logs. Typically used by log4j or logback.", - "display_name": "Hive Metastore Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Enables the health test that the Hive Metastore Server's process state is consistent with the role configuration", - "display_name": "Hive Metastore Server Process Health Test", - "name": "hivemetastore_scm_health_enabled", - "value": "true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "Directory where Hive Metastore Server will place its log files.", - "display_name": "Hive Metastore Server Log Directory", - "name": "hive_log_dir", - "value": "/var/log/hive" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The delegation token store implementation class. Use DBTokenStore for Highly Available Metastore Configuration.", - "display_name": "Hive Metastore Delegation Token Store", - "name": "hive_metastore_delegation_token_store", - "value": "org.apache.hadoop.hive.thrift.MemoryTokenStore" - }, - { - "desc": "The minimum log level for Hive Metastore Server logs", - "display_name": "Hive Metastore Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Hive Metastore Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "hive_metastore_env_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hive-hiveserver2.json b/sahara/plugins/cdh/v5_4_0/resources/hive-hiveserver2.json deleted file mode 100644 index 2c8634c3..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hive-hiveserver2.json +++ /dev/null @@ -1,500 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The check interval for session/operation timeout, in milliseconds, which can be disabled by setting to zero or a negative value. ", - "display_name": "Session Check Interval", - "name": "hiveserver2_session_check_interval", - "value": "0" - }, - { - "desc": "Number of cores per Spark executor.", - "display_name": "Spark Executor Cores", - "name": "hiveserver2_spark_executor_cores", - "value": "1" - }, - { - "desc": "Ratio between 0.0 and 1.0 of entries in the vectorized group-by aggregation hash that is flushed when the memory threshold is exceeded.", - "display_name": "Vectorized GroupBy Flush Ratio", - "name": "hiveserver2_vectorized_groupby_flush_ratio", - "value": "0.1" - }, - { - "desc": "Operation will be closed when not accessed for this duration of time, in milliseconds; disable by setting to zero. For a positive value, checked for operations in terminal state only (FINISHED, CANCELED, CLOSED, ERROR). For a negative value, checked for all of the operations regardless of state.", - "display_name": "Idle Operation Timeout", - "name": "hiveserver2_idle_operation_timeout", - "value": "0" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Address of the load balancer used for HiveServer2 roles, specified in host:port format. If port is not specified, the port used by HiveServer2 is used. Note: Changing this property regenerates Kerberos keytabs for all HiveServer2 roles.", - "display_name": "HiveServer2 Load Balancer", - "name": "hiverserver2_load_balancer", - "value": null - }, - { - "desc": "Initial number of executors used by the application at any given time. This is required if the dynamic executor allocation feature is enabled.", - "display_name": "Initial Number of Executors", - "name": "hiveserver2_spark_dynamic_allocation_initial_executors", - "value": "1" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Directory in HDFS where Hive writes intermediate data between MapReduce jobs. If not specified, Hive uses a default location..", - "display_name": "Hive HDFS Scratch Directory Free Space Monitoring Absolute Thresholds", - "name": "hiveserver2_exec_scratch_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "HiveServer2 will impersonate the beeline client user when talking to other services such as Mapreduce and Hdfs.", - "display_name": "HiveServer2 Enable Impersonation", - "name": "hiveserver2_enable_impersonation", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Enable optimization that checks if a query can be answered using statistics. If so, answers the query using only statistics stored in metastore.", - "display_name": "Enable Stats Optimization", - "name": "hiveserver2_compute_query_using_stats", - "value": "true" - }, - { - "desc": "Lower bound on the number of executors used by the application at any given time. This is used by dynamic executor allocation", - "display_name": "Lower Bound on Number of Executors", - "name": "hiveserver2_spark_dynamic_allocation_min_executors", - "value": "1" - }, - { - "desc": "When enabled, Spark will add and remove executors dynamically to Hive jobs. This is done based on the workload.", - "display_name": "Enable Dynamic Executor Allocation", - "name": "hiveserver2_spark_dynamic_allocation_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "Port on which HiveServer2 will listen for connections.", - "display_name": "HiveServer2 Port", - "name": "hs2_thrift_address_port", - "value": "10000" - }, - { - "desc": "Enable optimization that converts common join into MapJoin based on input file size.", - "display_name": "Enable MapJoin Optimization", - "name": "hiveserver2_enable_mapjoin", - "value": "true" - }, - { - "desc": "The desired file size after merging. This should be larger than hive.merge.smallfiles.avgsize.", - "display_name": "Desired File Size After Merging", - "name": "hiveserver2_merge_size_per_task", - "value": "268435456" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Merge small files at the end of a Spark job. When enabled, a map-only job is created to merge the files in the destination table/partitions.", - "display_name": "Enable Merging Small Files - Spark Job", - "name": "hiveserver2_merge_sparkfiles", - "value": "true" - }, - { - "desc": "When the average output file size of a job is less than the value of this property, Hive will start an additional map-only job to merge the output files into bigger files. This is only done for map-only jobs if hive.merge.mapfiles is true, for map-reduce jobs if hive.merge.mapredfiles is true, and for Spark jobs if hive.merge.sparkfiles is true.", - "display_name": "Small File Average Size Merge Threshold", - "name": "hiveserver2_merge_smallfiles_avgsize", - "value": "16777216" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Local Directory where Hive stores jars and data when performing a MapJoin optimization. If not specified, Hive uses a default location..", - "display_name": "Hive Local Scratch Directory Free Space Monitoring Absolute Thresholds", - "name": "hiveserver2_exec_local_scratch_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", - "display_name": "HiveServer2 Advanced Configuration Snippet (Safety Valve) for hive-site.xml", - "name": "hive_hs2_config_safety_valve", - "value": null - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Local directory where Hive stores jars downloaded for remote file systems (HDFS). If not specified, Hive uses a default location.", - "display_name": "Hive Downloaded Resources Directory", - "name": "hiveserver2_downloaded_resources_dir", - "value": null - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "The max percentage of heap to be used for hash in ReduceSink operator for Top-K selection. A 0 means the optimization is disabled. Values accepted are between 0 and 1.", - "display_name": "Maximum ReduceSink Top-K Memory Usage", - "name": "hiveserver2_limit_pushdown_memory_usage", - "value": "0.1" - }, - { - "desc": "Whether to try sorted merge bucket (SMB) join.", - "display_name": "Hive Optimize Sorted Merge Bucket Join", - "name": "hiveserver2_optimize_bucketmapjoin_sortedmerge", - "value": "false" - }, - { - "desc": "When computing the overall HiveServer2 health, consider the host's health.", - "display_name": "HiveServer2 Host Health Test", - "name": "hiveserver2_host_health_enabled", - "value": "true" - }, - { - "desc": "Maximum size of each Spark executor's Java heap memory when Hive is running on Spark.", - "display_name": "Spark Executor Maximum Java Heap Size", - "name": "hiveserver2_spark_executor_memory", - "value": "268435456" - }, - { - "desc": "Maximum size of each Spark driver's Java heap memory when Hive is running on Spark.", - "display_name": "Spark Driver Maximum Java Heap Size", - "name": "hiveserver2_spark_driver_memory", - "value": "268435456" - }, - { - "desc": "If Hive auto convert join is on, and the sum of the size for n-1 of the tables/partitions for a n-way join is smaller than the specified size, the join is directly converted to a MapJoin (there is no conditional task).", - "display_name": "Hive Auto Convert Join Noconditional Size", - "name": "hiveserver2_auto_convert_join_noconditionaltask_size", - "value": "20971520" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Local Directory where Hive stores jars and data when performing a MapJoin optimization. If not specified, Hive uses a default location.. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Local Directory where Hive stores jars and data when performing a MapJoin optimization. If not specified, Hive uses a default location. Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Hive Local Scratch Directory Free Space Monitoring Percentage Thresholds", - "name": "hiveserver2_exec_local_scratch_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Directory in HDFS where Hive writes intermediate data between MapReduce jobs. If not specified, Hive uses a default location.. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Directory in HDFS where Hive writes intermediate data between MapReduce jobs. If not specified, Hive uses a default location. Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Hive HDFS Scratch Directory Free Space Monitoring Percentage Thresholds", - "name": "hiveserver2_exec_scratch_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "HiveServer2 Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "HiveServer2 Environment Advanced Configuration Snippet (Safety Valve)", - "name": "hive_hs2_env_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Directory in HDFS where Hive writes intermediate data between MapReduce jobs. If not specified, Hive uses a default location.", - "display_name": "Hive HDFS Scratch Directory", - "name": "hiveserver2_exec_scratchdir", - "value": null - }, - { - "desc": "Some select queries can be converted to a single FETCH task instead of MapReduce task, minimizing latency. A value of none disables all conversion, minimal converts simple queries such as SELECT * and filter on partition columns, and more will convert SELECT queries including FILTERS.", - "display_name": "Fetch Task Query Conversion", - "name": "hiveserver2_fetch_task_conversion", - "value": "minimal" - }, - { - "desc": "Remove extra map-reduce jobs if the data is already clustered by the same key, eliminating the need to repartition the dataset again.", - "display_name": "Enable ReduceDeDuplication Optimization", - "name": "hiveserver2_optimize_reducededuplication", - "value": "true" - }, - { - "desc": "In vectorized group-by, the number of row entries added to the hash table before re-checking average variable size for memory usage estimation.", - "display_name": "Vectorized GroupBy Check Interval", - "name": "hiveserver2_vectorized_groupby_checkinterval", - "value": "4096" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "Session will be closed when not accessed for this duration of time, in milliseconds; disable by setting to zero or a negative value.", - "display_name": "Idle Session Timeout", - "name": "hiveserver2_idle_session_timeout", - "value": "0" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for HiveServer2 logs. Typically used by log4j or logback.", - "display_name": "HiveServer2 Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "Whether to vectorize the reduce side of query execution.", - "display_name": "Enable Reduce-Side Vectorization", - "name": "hiveserver2_vectorized_reduce_enabled", - "value": "false" - }, - { - "desc": "Merge small files at the end of a map-reduce job. When enabled, a map-only job is created to merge the files in the destination table/partitions.", - "display_name": "Enable Merging Small Files - Map-Reduce Job", - "name": "hiveserver2_merge_mapredfiles", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Minimum number of worker threads in HiveServer2's thread pool", - "display_name": "Min HiveServer2 Threads", - "name": "hiveserver2_min_threads", - "value": "5" - }, - { - "desc": "Enabled the Calcite based Cost Based Optimizer for HiveServer2.", - "display_name": "Enable Cost Based Optimizer for Hive", - "name": "hiveserver2_enable_cbo", - "value": "false" - }, - { - "desc": "Portion of total memory used in map-side partial aggregation. When exceeded, the partially aggregated results will be flushed from the map task to the reducers.", - "display_name": "Ratio of Memory Usage for Map-Side Aggregation", - "name": "hiveserver2_map_aggr_hash_memory_ratio", - "value": "0.5" - }, - { - "desc": "Enable map-side partial aggregation, which cause the mapper to generate fewer rows. This reduces the data to be sorted and distributed to reducers.", - "display_name": "Enable Map-Side Aggregation", - "name": "hiveserver2_map_aggr", - "value": "true" - }, - { - "desc": "Number of Spark executors assigned to each application. This should not be set when Dynamic Executor Allocation is enabled.", - "display_name": "Spark Executors Per Application", - "name": "hiveserver2_spark_executor_instances", - "value": null - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for HiveServer2", - "name": "hiveserver2_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "The minimum log level for HiveServer2 logs", - "display_name": "HiveServer2 Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Local Directory where Hive stores jars and data when performing a MapJoin optimization. If not specified, Hive uses a default location.", - "display_name": "Hive Local Scratch Directory", - "name": "hiveserver2_exec_local_scratchdir", - "value": null - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "When dynamic partition is enabled, reducers keep only one record writer at all times, which lowers the memory pressure on reducers.", - "display_name": "Enable Sorted Dynamic Partition Optimizer", - "name": "hiveserver2_optimize_sort_dynamic_partition", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for HiveServer2 logs. Typically used by log4j or logback.", - "display_name": "HiveServer2 Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "Enables the health test that the HiveServer2's process state is consistent with the role configuration", - "display_name": "HiveServer2 Process Health Test", - "name": "hiveserver2_scm_health_enabled", - "value": "true" - }, - { - "desc": "Merge small files at the end of a map-only job. When enabled, a map-only job is created to merge the files in the destination table/partitions.", - "display_name": "Enable Merging Small Files - Map-Only Job", - "name": "hiveserver2_merge_mapfiles", - "value": "true" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of HiveServer2 in Bytes", - "name": "hiveserver2_java_heapsize", - "value": "268435456" - }, - { - "desc": "Above this size, queries will not be converted to fetch tasks.", - "display_name": "Fetch Task Query Conversion Threshold", - "name": "hiveserver2_fetch_task_conversion_threshold", - "value": "268435456" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "Directory where HiveServer2 will place its log files.", - "display_name": "HiveServer2 Log Directory", - "name": "hive_log_dir", - "value": "/var/log/hive" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "This is the amount of extra off-heap memory that can be requested from YARN, per driver. This, together with spark.driver.memory, is the total memory that YARN can use to create JVM for a driver process.", - "display_name": "Spark Driver Memory Overhead", - "name": "hiveserver2_spark_yarn_driver_memory_overhead", - "value": "26" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Maximum number of worker threads in HiveServer2's thread pool", - "display_name": "Max HiveServer2 Threads", - "name": "hiveserver2_max_threads", - "value": "100" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "hiveserver2_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "When the number of ReduceSink operators after merging is less than this number, the ReduceDeDuplication optimization will be disabled.", - "display_name": "Mininum Reducers for ReduceDeDuplication Optimization", - "name": "hiveserver2_optimize_reducededuplication_min_reducer", - "value": "4" - }, - { - "desc": "This is the amount of extra off-heap memory that can be requested from YARN, per executor process. This, together with spark.executor.memory, is the total memory that YARN can use to create JVM for an executor process.", - "display_name": "Spark Executor Memory Overhead", - "name": "hiveserver2_spark_yarn_executor_memory_overhead", - "value": "26" - }, - { - "desc": "Enable optimization that vectorizes query execution by streamlining operations by processing a block of 1024 rows at a time.", - "display_name": "Enable Vectorization Optimization", - "name": "hiveserver2_vectorized_enabled", - "value": "true" - }, - { - "desc": "The number of rows with the same key value to be cached in memory per SMB-joined table.", - "display_name": "Hive SMB Join Cache Rows", - "name": "hiveserver2_smbjoin_cache_rows", - "value": "10000" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hive-metastore-sentry-safety.xml b/sahara/plugins/cdh/v5_4_0/resources/hive-metastore-sentry-safety.xml deleted file mode 100644 index 8395f76b..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hive-metastore-sentry-safety.xml +++ /dev/null @@ -1,15 +0,0 @@ -\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n
\n\n\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n
\n\nEvent trackers are defined in a JSON object like the following:
\n\n\n{\n \"timeToLive\" : [integer],\n \"fields\" : [\n {\n \"type\" : [string],\n \"name\" : [string]\n }\n ]\n}\n\n\n\nWhere:\n
\n\n\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n
\n\n\nThe following is the list of fields that can be used to compare Hive events:\n
\n\nEvent filters are defined in a JSON object like the following:
\n\n\n{\n \"defaultAction\" : (\"accept\", \"discard\"),\n \"rules\" : [\n {\n \"action\" : (\"accept\", \"discard\"),\n \"fields\" : [\n {\n \"name\" : \"fieldName\",\n \"match\" : \"regex\"\n }\n ]\n }\n ]\n}\n\n\n\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n
\n\n\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n
\n\n\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n
\n\n\nThe following is the list of fields that can be filtered for Hive events:\n
\n\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "Instead of talking to Hive Metastore Server for Metastore information, Hive clients will talk directly to the Metastore database.", - "display_name": "Bypass Hive Metastore Server", - "name": "hive_bypass_metastore_server", - "value": "false" - }, - { - "desc": "Smaller than this size, Hive uses a single-threaded copy; larger than this size, Hive uses DistCp.", - "display_name": "Hive Copy Large File Size", - "name": "hive_exec_copyfile_maxsize", - "value": "33554432" - }, - { - "desc": "User for Hive Metastore database", - "display_name": "Hive Metastore Database User", - "name": "hive_metastore_database_user", - "value": "hive" - }, - { - "desc": "Perform DataNucleus validation of metadata during startup. Note: when enabled, Hive will log DataNucleus warnings even though Hive will function normally.", - "display_name": "Hive Metastore Database DataNucleus Metadata Validation", - "name": "hive_metastore_database_datanucleus_metadata_validation", - "value": "false" - }, - { - "desc": "Cloudera does not support Hive on Spark in CDH 5.4. Enable Hive to use Spark for execution even though it is not supported. For evaluation purposes only.", - "display_name": "Enable Hive on Spark (Unsupported)", - "name": "enable_hive_on_spark", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for navigator.client.properties.", - "display_name": "Hive Client Advanced Configuration Snippet (Safety Valve) for navigator.client.properties", - "name": "navigator_client_config_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds of the overall WebHCat Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" WebHCat Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" WebHCat Servers falls below the critical threshold.", - "display_name": "Healthy WebHCat Server Monitoring Thresholds", - "name": "hive_webhcats_healthy_thresholds", - "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into sentry-site.xml. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Hive Service Advanced Configuration Snippet (Safety Valve) for sentry-site.xml", - "name": "hive_server2_sentry_safety_valve", - "value": null - }, - { - "desc": "Size per reducer. If the input size is 10GiB and this is set to 1GiB, Hive will use 10 reducers.", - "display_name": "Hive Bytes Per Reducer", - "name": "hive_bytes_per_reducer", - "value": "67108864" - }, - { - "desc": "Enable collection of audit events from the service's roles.", - "display_name": "Enable Audit Collection", - "name": "navigator_audit_enabled", - "value": "true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Allows URIs when defining privileges in per-database policy files. Warning: Typically, this configuration should be disabled. Enabling it would allow database policy file owner (which is generally not Hive admin user) to grant load privileges to any directory with read access to Hive admin user, including databases controlled by other database policy files.", - "display_name": "Allow URIs in Database Policy File", - "name": "sentry_allow_uri_db_policyfile", - "value": "false" - }, - { - "desc": "In unsecure mode, setting this property to true will cause the Metastore Server to execute DFS operations using the client's reported user and group permissions. Cloudera Manager will set this for all clients and servers.", - "display_name": "Set User and Group Information", - "name": "hive_set_ugi", - "value": "true" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "When set, this parameter is used to convert the username into the LDAP Distinguished Name (DN), so that the resulting DN looks like uid=username,X. For example, if this parameter is set to \"ou=People,dc=cloudera,dc=com\", and the username passed in is \"mike\", the resulting authentication passed to the LDAP server will look like \"uid=mike,ou=People,dc=cloudera,dc=com\". This parameter is generally most useful when authenticating against an OpenLDAP server. This parameter is mutually exclusive with LDAP Domain.", - "display_name": "LDAP BaseDN", - "name": "hiveserver2_ldap_basedn", - "value": null - }, - { - "desc": "MapReduce jobs are run against this service.", - "display_name": "MapReduce Service", - "name": "mapreduce_yarn_service", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hive-site.xml b/sahara/plugins/cdh/v5_4_0/resources/hive-site.xml deleted file mode 100644 index 7e70626f..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hive-site.xml +++ /dev/null @@ -1,61 +0,0 @@ - - - -The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of WebHCat Server in Bytes", - "name": "hive_webhcat_java_heapsize", - "value": "268435456" - }, - { - "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", - "display_name": "WebHCat Server Advanced Configuration Snippet (Safety Valve) for hive-site.xml", - "name": "hive_webhcat_hive_config_safety_valve", - "value": null - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "webhcat_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "WebHCat Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "WebHCat Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "hive_webhcat_env_safety_valve", - "value": null - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for WebHCat Server", - "name": "hive_webhcat_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for WebHCat Server logs. Typically used by log4j or logback.", - "display_name": "WebHCat Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "Enables the health test that the WebHCat Server's process state is consistent with the role configuration", - "display_name": "WebHCat Server Process Health Test", - "name": "webhcat_scm_health_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into webhcat-site.xml for this role only.", - "display_name": "WebHCat Server Advanced Configuration Snippet (Safety Valve) for webhcat-site.xml", - "name": "hive_webhcat_config_safety_valve", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "When computing the overall WebHCat Server health, consider the host's health.", - "display_name": "WebHCat Server Host Health Test", - "name": "webhcat_host_health_enabled", - "value": "true" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The minimum log level for WebHCat Server logs", - "display_name": "WebHCat Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Directory where WebHCat Server will place its log files.", - "display_name": "WebHCat Server Log Directory", - "name": "hcatalog_log_dir", - "value": "/var/log/hcatalog" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hue-hue_server.json b/sahara/plugins/cdh/v5_4_0/resources/hue-hue_server.json deleted file mode 100644 index aa1b426e..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hue-hue_server.json +++ /dev/null @@ -1,188 +0,0 @@ -[ - { - "desc": "For advanced use only, a string to be inserted into hue_safety_valve_server.ini for this role only.", - "display_name": "Hue Server Advanced Configuration Snippet (Safety Valve) for hue_safety_valve_server.ini", - "name": "hue_server_hue_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Enable HTTPS for the Hue web server.", - "display_name": "Enable HTTPS", - "name": "ssl_enable", - "value": "false" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "hue_server_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Directory where Hue Server will place its log files.", - "display_name": "Hue Server Log Directory", - "name": "hue_server_log_dir", - "value": "/var/log/hue" - }, - { - "desc": "Timeout in seconds for Thrift calls to HiveServer2 and Impala.", - "display_name": "HiveServer2 and Impala Thrift Connection Timeout", - "name": "hs2_conn_timeout", - "value": "120" - }, - { - "desc": "An optional, custom one-line HTML code to display as a banner on top of all Hue Server web pages. Useful in displaying cluster identity of the Hue Server.", - "display_name": "Top Banner Custom HTML", - "name": "banner_html", - "value": null - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Location on HDFS where the jobsub examples and templates are stored.", - "display_name": "Jobsub Examples and Templates Directory", - "name": "hue_server_remote_data_dir", - "value": "/user/hue/jobsub" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "When computing the overall Hue Server health, consider the host's health.", - "display_name": "Hue Server Host Health Test", - "name": "hue_server_host_health_enabled", - "value": "true" - }, - { - "desc": "Port to use to connect to the Hue server.", - "display_name": "Hue HTTP Port", - "name": "hue_http_port", - "value": "8888" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Hue Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "HUE_SERVER_role_env_safety_valve", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Path to the SSL private key on the host running the Hue web server. This file must be readable by the Hue system user. Hue only supports a key without a passphrase.", - "display_name": "Local Path to SSL Private Key", - "name": "ssl_private_key", - "value": null - }, - { - "desc": "Path to the SSL certificate on the host running the Hue web server. This file must be readable by the Hue system user.", - "display_name": "Local Path to SSL Certificate", - "name": "ssl_certificate", - "value": null - }, - { - "desc": "If enabled, the Hue server binds to the wildcard address (\"0.0.0.0\") for its ports.", - "display_name": "Bind Hue Server to Wildcard Address", - "name": "hue_server_bind_wildcard", - "value": "false" - }, - { - "desc": "Enables the health test that the Hue Server's process state is consistent with the role configuration", - "display_name": "Hue Server Process Health Test", - "name": "hue_server_scm_health_enabled", - "value": "true" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Random string used for secure hashing in the session store.", - "display_name": "Secret Key", - "name": "secret_key", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hue-kt_renewer.json b/sahara/plugins/cdh/v5_4_0/resources/hue-kt_renewer.json deleted file mode 100644 index 7fbee5cd..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hue-kt_renewer.json +++ /dev/null @@ -1,134 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Interval in seconds with which Hue's Kerberos ticket will get renewed.", - "display_name": "Hue Keytab Renewal Interval", - "name": "keytab_reinit_frequency", - "value": "3600" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Enables the health test that the Kerberos Ticket Renewer's process state is consistent with the role configuration", - "display_name": "Kerberos Ticket Renewer Process Health Test", - "name": "kt_renewer_scm_health_enabled", - "value": "true" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Kerberos Ticket Renewer Environment Advanced Configuration Snippet (Safety Valve)", - "name": "KT_RENEWER_role_env_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "When computing the overall Kerberos Ticket Renewer health, consider the host's health.", - "display_name": "Kerberos Ticket Renewer Host Health Test", - "name": "kt_renewer_host_health_enabled", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "kt_renewer_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Directory where Kerberos Ticket Renewer will place its log files.", - "display_name": "Kerberos Ticket Renewer Log Directory", - "name": "kt_renewer_log_dir", - "value": "/var/log/hue" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/hue-service.json b/sahara/plugins/cdh/v5_4_0/resources/hue-service.json deleted file mode 100644 index 7c2a7b8e..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/hue-service.json +++ /dev/null @@ -1,350 +0,0 @@ -[ - { - "desc": "Create users in Hue when they try to login with their LDAP credentials. For use when using LdapBackend for Hue authentication.", - "display_name": "Create LDAP users on login", - "name": "create_users_on_login", - "value": "true" - }, - { - "desc": "Number of threads used by the Hue web server.", - "display_name": "Hue Web Server Threads", - "name": "cherrypy_server_threads", - "value": "10" - }, - { - "desc": "Name of the HBase service that this Hue service instance depends on", - "display_name": "HBase Service", - "name": "hbase_service", - "value": null - }, - { - "desc": "The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "Name of the Oozie service that this Hue service instance depends on", - "display_name": "Oozie Service", - "name": "oozie_service", - "value": null - }, - { - "desc": "The distinguished name to use as a search base for finding users and groups. This should be similar to 'dc=hadoop,dc=mycompany,dc=com'.", - "display_name": "LDAP Search Base", - "name": "base_dn", - "value": null - }, - { - "desc": "Comma-separated list of regular expressions, which match 'host:port' of requested proxy target.", - "display_name": "Whitelist", - "name": "whitelist", - "value": "(localhost|127\\.0\\.0\\.1):(50030|50070|50060|50075)" - }, - { - "desc": "Name of the Sentry service that this Hue service instance depends on", - "display_name": "Sentry Service", - "name": "sentry_service", - "value": null - }, - { - "desc": "Thrift server to use for HBase app.", - "display_name": "HBase Thrift Server", - "name": "hue_hbase_thrift", - "value": null - }, - { - "desc": "Base filter for searching for groups. For Active Directory, this is typically '(objectClass=group)'.", - "display_name": "LDAP Group Filter", - "name": "group_filter", - "value": null - }, - { - "desc": "Time zone name.", - "display_name": "Time Zone", - "name": "time_zone", - "value": "America/Los_Angeles" - }, - { - "desc": "The health test thresholds of the overall Kerberos Ticket Renewer health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Kerberos Ticket Renewers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Kerberos Ticket Renewers falls below the critical threshold.", - "display_name": "Healthy Kerberos Ticket Renewer Monitoring Thresholds", - "name": "hue_kt_renewers_healthy_thresholds", - "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" - }, - { - "desc": "The user that this service's processes should run as.", - "display_name": "System User", - "name": "process_username", - "value": "hue" - }, - { - "desc": "The health test thresholds of the overall Hue Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Hue Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Hue Servers falls below the critical threshold.", - "display_name": "Healthy Hue Server Monitoring Thresholds", - "name": "hue_hue_servers_healthy_thresholds", - "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The group name attribute in the LDAP schema. For Active Directory, this is typically 'cn'.", - "display_name": "LDAP Group Name Attribute", - "name": "group_name_attr", - "value": null - }, - { - "desc": "HDFS directory used for storing temporary files.", - "display_name": "HDFS Temporary Directory", - "name": "hdfs_tmp_dir", - "value": "/tmp" - }, - { - "desc": "Name of the Solr service that this Hue service instance depends on", - "display_name": "Solr Service", - "name": "solr_service", - "value": null - }, - { - "desc": "If the database is SQLite3, this is the filename of the database to use, and the directory of this file must be writable by the 'hue' user.", - "display_name": "Hue Database Directory", - "name": "database_dir", - "value": "/var/lib/hue/desktop.db" - }, - { - "desc": "The name of a default group that users will be added to at creation time.", - "display_name": "Default User Group", - "name": "default_user_group", - "value": null - }, - { - "desc": "When you enable anonymous usage data collection Hue tracks anonymised pages and application versions in order to gather information about each application's usage levels. The data collected does not include any hostnames or IDs. Data collection option is available on CDH 4.4 and later deployments.", - "display_name": "Enable Usage Data Collection", - "name": "usage_data_collection_enable", - "value": "true" - }, - { - "desc": "Name of the ZooKeeper service that this Hue service instance depends on", - "display_name": "ZooKeeper Service", - "name": "zookeeper_service", - "value": null - }, - { - "desc": "Distinguished name of the user to bind as. Not required if the LDAP server supports anonymous searches. For Active Directory, this would be similar to 'hadoop-admin@mycompany.com'.", - "display_name": "LDAP Bind User", - "name": "bind_dn", - "value": null - }, - { - "desc": "The URL of the LDAP Server; similar to 'ldap://auth.mycompany.com' or 'ldaps://auth.mycompany.com'.", - "display_name": "LDAP URL", - "name": "ldap_url", - "value": null - }, - { - "desc": "Whether to use StartTLS (as opposed to ldaps) to communicate securely with the LDAP server. This is only effective when the LDAP certificate is specified.", - "display_name": "Use StartTLS", - "name": "use_start_tls", - "value": "true" - }, - { - "desc": "The password of the bind user.", - "display_name": "LDAP Bind Password", - "name": "bind_password", - "value": null - }, - { - "desc": "Only applies to Active Directory. The Active Directory Domain will be similar to 'MYCOMPANY.COM'.", - "display_name": "NT Domain", - "name": "nt_domain", - "value": null - }, - { - "desc": "Kerberos principal used by all roles of this service.", - "display_name": "Kerberos Principal", - "name": "kerberos_princ_name", - "value": "hue" - }, - { - "desc": "Class that defines extra accessor methods for user objects.", - "display_name": "User Augmentor", - "name": "user_augmentor", - "value": "desktop.auth.backend.DefaultUserAugmentor" - }, - { - "desc": "File where the database gets dumped to or loaded from.", - "display_name": "Database Dump File", - "name": "database_dump_file", - "value": "/tmp/hue_database_dump.json" - }, - { - "desc": "LDAP Username Pattern for use with non-Active Directory LDAP implementations. Must contain the special '<username>' string for replacement during authentication.", - "display_name": "LDAP Username Pattern", - "name": "ldap_username_pattern", - "value": null - }, - { - "desc": "The group that this service's processes should run as.", - "display_name": "System Group", - "name": "process_groupname", - "value": "hue" - }, - { - "desc": "In debug mode, Django displays a detailed traceback when an exception occurs. Debugging information may contain sensitive data. Django remembers every SQL query it executes in debug mode, which will rapidly consume memory.", - "display_name": "Enable Django Debug Mode", - "name": "django_debug_enable", - "value": "false" - }, - { - "desc": "LDAP certificate for authentication over TLS", - "display_name": "LDAP Certificate", - "name": "ldap_cert", - "value": null - }, - { - "desc": "Type of database used for Hue", - "display_name": "Hue Database Type", - "name": "database_type", - "value": "sqlite3" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "When computing the overall HUE health, consider Beeswax Server's health", - "display_name": "Beeswax Server Role Health Test", - "name": "hue_beeswax_server_health_enabled", - "value": "true" - }, - { - "desc": "HTTPFS role or Namenode (if webhdfs is enabled) that hue can use to communicate with HDFS.", - "display_name": "HDFS Web Interface Role", - "name": "hue_webhdfs", - "value": null - }, - { - "desc": "Mode of authenticating login credentials. Select desktop.auth.backend.LdapBackend to use LDAP to authenticate login credentials. LDAP requires you to also set the LDAP URL, NT Domain, and optionally LDAP certificate if you are using secure LDAP. Select desktop.auth.backend.PamBackend to use PAM to authenticate login credentials.", - "display_name": "Authentication Backend", - "name": "auth_backend", - "value": "desktop.auth.backend.AllowFirstUserDjangoBackend" - }, - { - "desc": "Name of the Sqoop service that this Hue service instance depends on", - "display_name": "Sqoop Service", - "name": "sqoop_service", - "value": null - }, - { - "desc": "The username to use to log into the Hue database. Not necessary for SQLite3.", - "display_name": "Hue Database Username", - "name": "database_user", - "value": "hue" - }, - { - "desc": "Enable debug output in HTTP Internal Server Error (status 500) responses. Debugging information may contain sensitive data. If Enable Django Debug Mode is set, this is automatically enabled.", - "display_name": "Enable Debugging of Internal Server Error Responses", - "name": "http_500_debug_enable", - "value": "false" - }, - { - "desc": "Comma-separated list of regular expressions, which match any prefix of 'host:port/path' of requested proxy target. This does not support matching GET parameters.", - "display_name": "Blacklist", - "name": "blacklist", - "value": "()" - }, - { - "desc": "For advanced use only, a string to be inserted into sentry-site.xml. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Hue Service Advanced Configuration Snippet (Safety Valve) for sentry-site.xml", - "name": "hue_sentry_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Hue Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "hue_service_env_safety_valve", - "value": null - }, - { - "desc": "Name of the Impala service that this Hue service instance depends on", - "display_name": "Impala Service", - "name": "impala_service", - "value": null - }, - { - "desc": "Name of Hue database.", - "display_name": "Hue Database Name", - "name": "database_name", - "value": "hue" - }, - { - "desc": "The attribute of the group object that identifies the members of the group. For Active Directory, this is typically 'member'.", - "display_name": "LDAP Group Membership Attribute", - "name": "group_member_attr", - "value": null - }, - { - "desc": "Name of the Hive service that this Hue service instance depends on", - "display_name": "Hive Service", - "name": "hive_service", - "value": null - }, - { - "desc": "The username attribute in the LDAP schema. For Active Directory, this is typically 'sAMAccountName'.", - "display_name": "LDAP Username Attribute", - "name": "user_name_attr", - "value": null - }, - { - "desc": "Default encoding for site data.", - "display_name": "Default Site Encoding", - "name": "default_site_encoding", - "value": "utf" - }, - { - "desc": "Search Bind Authentication connects to the LDAP server using credentials provided in the 'bind_dn' and 'bind_password' configurations. If these configurations are not set, then an anonymous search is performed.", - "display_name": "Use Search Bind Authentication", - "name": "search_bind_authentication", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into hue_safety_valve.ini. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Hue Service Advanced Configuration Snippet (Safety Valve) for hue_safety_valve.ini", - "name": "hue_service_safety_valve", - "value": null - }, - { - "desc": "Port on host where the Hue database is running. Not necessary for SQLite3.", - "display_name": "Hue Database Port", - "name": "database_port", - "value": "3306" - }, - { - "desc": "Password for Hue database. Not necessary for SQLite3.", - "display_name": "Hue Database Password", - "name": "database_password", - "value": "" - }, - { - "desc": "The base filter for searching for users. For Active Directory, this is typically '(objectClass=user)'.", - "display_name": "LDAP User Filter", - "name": "user_filter", - "value": null - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "Name of host where the Hue database is running. Not necessary for SQLite3.", - "display_name": "Hue Database Hostname", - "name": "database_host", - "value": "localhost" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/impala-catalogserver.json b/sahara/plugins/cdh/v5_4_0/resources/impala-catalogserver.json deleted file mode 100644 index e586377c..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/impala-catalogserver.json +++ /dev/null @@ -1,266 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", - "display_name": "Health Check Startup Tolerance", - "name": "catalogserver_startup_tolerance", - "value": "5" - }, - { - "desc": "Port where Catalog Server debug web server runs.", - "display_name": "Catalog Server HTTP Server Port", - "name": "catalogserver_webserver_port", - "value": "25020" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "catalogserver_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "catalogserver_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Port where Catalog Server is exported.", - "display_name": "Catalog Server Service Port", - "name": "catalog_service_port", - "value": "26000" - }, - { - "desc": "Buffer log messages logged at this level or lower (-1 means don't buffer; 0 means buffer INFO only; 1 means buffer WARNING only, ...)", - "display_name": "Catalog Server Log Buffer Level", - "name": "logbuflevel", - "value": "0" - }, - { - "desc": "If true, loads catalog metadata in the background. If false, metadata is loaded lazily (on access). Only effective in CDH 5 and Impala 1.2.4 and higher.", - "display_name": "Load Catalog in Background", - "name": "load_catalog_in_background", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "Local path to the certificate presented by the Catalogd debug webserver. This file must be in .pem format. If empty, webserver SSL/TLS support is not enabled.", - "display_name": "SSL/TLS Certificate for Catalog Server Webserver", - "name": "webserver_certificate_file", - "value": null - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "catalogserver_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "The amount of time to wait for the Catalog Server to fully start up and connect to the StateStore before enforcing the connectivity check.", - "display_name": "Catalog Server Connectivity Tolerance at Startup", - "name": "catalogserver_connectivity_tolerance", - "value": "180" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Timeout for requests to the Hive Metastore Server from Catalog Server. Consider increasing this if you have tables with a lot of metadata and see timeout errors.", - "display_name": "Catalog Server Hive Metastore Connection Timeout", - "name": "hive_metastore_timeout", - "value": "3600" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Username for Catalog Server web server authentication.", - "display_name": "Catalog Server Web Server Username", - "name": "webserver_htpassword_user", - "value": null - }, - { - "desc": "Password for Catalog Server web server authentication.", - "display_name": "Catalog Server Web Server User Password", - "name": "webserver_htpassword_password", - "value": null - }, - { - "desc": "Enables the health test that the Impala Catalog Server's process state is consistent with the role configuration", - "display_name": "Impala Catalog Server Process Health Test", - "name": "catalogserver_scm_health_enabled", - "value": "true" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be added (verbatim) to Catalog Server command line flags. Key names should begin with a hyphen(-). For example: -log_filename=foo.log", - "display_name": "Catalog Server Command Line Argument Advanced Configuration Snippet (Safety Valve)", - "name": "catalogd_cmd_args_safety_valve", - "value": null - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "Catalog Server HDFS Advanced Configuration Snippet (Safety Valve)", - "name": "catalogd_hdfs_site_conf_safety_valve", - "value": null - }, - { - "desc": "Directory where Catalog Server will place its log files.", - "display_name": "Catalog Server Log Directory", - "name": "log_dir", - "value": "/var/log/catalogd" - }, - { - "desc": "Enables the health test that verifies the Catalog Server is connected to the StateStore", - "display_name": "Catalog Server Connectivity Health Test", - "name": "catalogserver_connectivity_health_enabled", - "value": "true" - }, - { - "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", - "display_name": "Catalog Server HBase Advanced Configuration Snippet (Safety Valve)", - "name": "catalogd_hbase_conf_safety_valve", - "value": null - }, - { - "desc": "Enable/Disable Catalog Server web server. This web server contains useful information about Catalog Server daemon.", - "display_name": "Enable Catalog Server Web Server", - "name": "catalogd_enable_webserver", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Impala Catalog Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "CATALOGSERVER_role_env_safety_valve", - "value": null - }, - { - "desc": "When computing the overall Impala Catalog Server health, consider the host's health.", - "display_name": "Impala Catalog Server Host Health Test", - "name": "catalogserver_host_health_enabled", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Impala Catalog Server logs. Typically used by log4j or logback.", - "display_name": "Impala Catalog Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", - "display_name": "Catalog Server Hive Advanced Configuration Snippet (Safety Valve)", - "name": "catalogd_hive_conf_safety_valve", - "value": null - }, - { - "desc": "Verbose logging level for the GLog logger. These messages are always logged at 'INFO' log level, so this setting has no effect if Logging Threshold is set to 'WARN' or above.", - "display_name": "Catalog Server Verbose Log Level", - "name": "log_verbose_level", - "value": "1" - }, - { - "desc": "The health test thresholds on the resident size of the process.", - "display_name": "Resident Set Size Thresholds", - "name": "process_resident_set_size_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "The minimum log level for Impala Catalog Server logs", - "display_name": "Impala Catalog Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "The number of log files that are kept for each severity level before all older log files are removed. The number has to be greater than 1 to keep at least the current log file open. If set to 0, all log files are retained and log rotation is effectively disabled.", - "display_name": "Catalog Server Maximum Log Files", - "name": "max_log_files", - "value": "10" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/impala-impalad.json b/sahara/plugins/cdh/v5_4_0/resources/impala-impalad.json deleted file mode 100644 index 6220f199..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/impala-impalad.json +++ /dev/null @@ -1,440 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Directories where Impala Daemon will write data such as spilling information to disk to free up memory. This can potentially be large amounts of data.. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Directories where Impala Daemon will write data such as spilling information to disk to free up memory. This can potentially be large amounts of data. Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Impala Daemon Scratch Directories Free Space Monitoring Percentage Thresholds", - "name": "impalad_scratch_directories_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "User-defined function (UDF) libraries are copied from HDFS into this local directory.", - "display_name": "Local UDF Library Dir", - "name": "local_library_dir", - "value": "/var/lib/impala/udfs" - }, - { - "desc": "Encrypt and verify the integrity of all data spilled to disk as part of a query. This feature is only supported for Impala 2.0 and higher and CDH 5.2 and higher (which includes Impala 2.0).", - "display_name": "Disk Spill Encryption", - "name": "disk_spill_encryption", - "value": "false" - }, - { - "desc": "Maximum number of query results a client may request to be cached on a per-query basis to support restarting fetches. This option guards against unreasonably large result caches requested by clients. Requests exceeding this maximum will be rejected.", - "display_name": "Result Cache Maximum Size", - "name": "impalad_result_cache_max_size", - "value": "100000" - }, - { - "desc": "Number of seconds to wait between attempts during Llama registration.", - "display_name": "Llama Registration Wait Seconds", - "name": "llama_registration_wait_secs", - "value": "3" - }, - { - "desc": "Abort Impala startup if there are improper configs or running on unsupported hardware.", - "display_name": "Abort on Config Error", - "name": "abort_on_config_error", - "value": "true" - }, - { - "desc": "The timeout used by the Cloudera Manager Agent's query monitor when communicating with the Impala Daemon web server, specified in seconds.", - "display_name": "Query Monitoring Timeout", - "name": "executing_queries_timeout_seconds", - "value": "5.0" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The maximum size (in entries) of the Impala daemon lineage log file before a new one is created.", - "display_name": "Impala Daemon Maximum Lineage Log File Size", - "name": "max_lineage_log_file_size", - "value": "5000" - }, - { - "desc": "Directories where Impala Daemon will write data such as spilling information to disk to free up memory. This can potentially be large amounts of data.", - "display_name": "Impala Daemon Scratch Directories", - "name": "scratch_dirs", - "value": null - }, - { - "desc": "Buffer log messages logged at this level or lower (-1 means don't buffer; 0 means buffer INFO only; 1 means buffer WARNING only, ...)", - "display_name": "Impala Daemon Log Buffer Level", - "name": "logbuflevel", - "value": "0" - }, - { - "desc": "Enables lineage generation by Impala daemons. The lineage log file is placed in the directory specified by the 'Impala Daemon Lineage Log Directory' parameter.", - "display_name": "Enable Impala Lineage Generation", - "name": "enable_lineage_log", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be added (verbatim) to Impala Daemon command-line flags. Key names should begin with a hyphen(-). For example: -log_filename=foo.log", - "display_name": "Impala Daemon Command Line Argument Advanced Configuration Snippet (Safety Valve)", - "name": "impalad_cmd_args_safety_valve", - "value": null - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "Enable or disable the Impala Daemon web server. This web server contains useful information about Impala Daemon.", - "display_name": "Enable Impala Daemon Web Server", - "name": "impalad_enable_webserver", - "value": "true" - }, - { - "desc": "The location on disk of the certificate, in .pem format, used to confirm the authenticity of the LDAP server certificate. This is the Certificate Authority (CA) certificate, and it was used to sign the LDAP server certificate. If not set, Impala by default trusts all certificates supplied by the LDAP server, which means that an attacker could potentially intercept otherwise encrypted usernames and passwords.", - "display_name": "LDAP Server CA Certificate", - "name": "impalad_ldap_ca_certificate", - "value": null - }, - { - "desc": "Port on which HiveServer2 client requests are served by Impala Daemons.", - "display_name": "Impala Daemon HiveServer2 Port", - "name": "hs2_port", - "value": "21050" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Username for Impala Daemon webserver authentication.", - "display_name": "Impala Daemon Web Server Username", - "name": "webserver_htpassword_user", - "value": null - }, - { - "desc": "Enables audit event generation by Impala daemons. The audit log file will be placed in the directory specified by 'Impala Daemon Audit Log Directory' parameter.", - "display_name": "Enable Impala Audit Event Generation", - "name": "enable_audit_event_log", - "value": "false" - }, - { - "desc": "Maximum number of seconds that Impala attempts to register or re-register with Llama. If registration is unsuccessful, Impala cancels the action with an error, which could result in an impalad startup failure or a cancelled query. A setting of -1 seconds means try indefinitely.", - "display_name": "Llama Registration Timeout Seconds", - "name": "llama_registration_timeout_secs", - "value": "30" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"any\"}" - }, - { - "desc": "When computing the overall Impala Daemon health, consider the host's health.", - "display_name": "Impala Daemon Host Health Test", - "name": "impalad_host_health_enabled", - "value": "true" - }, - { - "desc": "The directory in which Impala daemon audit event log files are written. If \"Impala Audit Event Generation\" property is enabled, Impala will generate its audit logs in this directory.", - "display_name": "Impala Daemon Audit Log Directory", - "name": "audit_event_log_dir", - "value": "/var/log/impalad/audit" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "An XML string to use verbatim for the contents of fair-scheduler.xml for Impala Daemons. This configuration only has effect on Impala versions 1.3 or greater.", - "display_name": "Impala Daemon Fair Scheduler Advanced Configuration", - "name": "impalad_fair_scheduler_safety_valve", - "value": null - }, - { - "desc": "A list of key-value pairs of additional query options to pass to the Impala Daemon command line, separated by ','.", - "display_name": "Impala Daemon Query Options Advanced Configuration Snippet (Safety Valve)", - "name": "default_query_options", - "value": "" - }, - { - "desc": "Port on which Beeswax client requests are served by Impala Daemons.", - "display_name": "Impala Daemon Beeswax Port", - "name": "beeswax_port", - "value": "21000" - }, - { - "desc": "Port where StateStoreSubscriberService is running.", - "display_name": "StateStoreSubscriber Service Port", - "name": "state_store_subscriber_port", - "value": "23000" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Impala Daemon Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "Enables the health check that determines if the Impala daemon is ready to process queries.", - "display_name": "Impala Daemon Ready Status Health Check", - "name": "impalad_ready_status_check_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Port on which ImpalaBackendService is exported.", - "display_name": "Impala Daemon Backend Port", - "name": "be_port", - "value": "22000" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "impalad_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "An XML snippet to append to llama-site.xml for Impala Daemons. This configuration only has effect on Impala versions 1.3 or greater.", - "display_name": "Impala Daemon Llama Site Advanced Configuration", - "name": "impala_llama_site_conf_safety_valve", - "value": null - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "Address of the load balancer used for Impala daemons. Should be specified in host:port format. If this is specified and Kerberos is enabled, Cloudera Manager adds a principal for 'impala/<load_balancer_host>@<realm>' to the keytab for all Impala daemons.", - "display_name": "Impala Daemons Load Balancer", - "name": "impalad_load_balancer", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Maximum number of times a request to reserve, expand, or release resources is attempted until the request is cancelled.", - "display_name": "Llama Maximum Request Attempts", - "name": "llama_max_request_attempts", - "value": "5" - }, - { - "desc": "For advanced use only, a string to be inserted into hbase-site.xml for this role only.", - "display_name": "Impala Daemon HBase Advanced Configuration Snippet (Safety Valve)", - "name": "impalad_hbase_conf_safety_valve", - "value": null - }, - { - "desc": "The maximum size, in megabytes, per log file for Impala Daemon logs. Typically used by log4j or logback.", - "display_name": "Impala Daemon Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Impala Daemon Environment Advanced Configuration Snippet (Safety Valve)", - "name": "IMPALAD_role_env_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "impalad_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "The polling period of the Impala query monitor in the Cloudera Manager Agent, specified in seconds. If set to zero, query monitoring is disabled.", - "display_name": "Query Monitoring Period", - "name": "query_monitoring_period_seconds", - "value": "1.0" - }, - { - "desc": "For advanced use only, a string to be inserted into hive-site.xml for this role only.", - "display_name": "Impala Daemon Hive Advanced Configuration Snippet (Safety Valve)", - "name": "impala_hive_conf_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Directory where Impala Daemon will place its log files.", - "display_name": "Impala Daemon Log Directory", - "name": "log_dir", - "value": "/var/log/impalad" - }, - { - "desc": "The amount of time at Impala Daemon startup allowed for the Impala Daemon to start accepting new queries for processing.", - "display_name": "Impala Daemon Ready Status Startup Tolerance", - "name": "impalad_ready_status_check_startup_tolerance", - "value": "180" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Enables the health test that verifies the Impala Daemon is connected to the StateStore.", - "display_name": "Impala Daemon Connectivity Health Test", - "name": "impalad_connectivity_health_enabled", - "value": "true" - }, - { - "desc": "Port where Impala debug web server runs.", - "display_name": "Impala Daemon HTTP Server Port", - "name": "impalad_webserver_port", - "value": "25000" - }, - { - "desc": "Verbose logging level for the GLog logger. These messages are always logged at 'INFO' log level, so this setting has no effect if Logging Threshold is set to 'WARN' or above. ", - "display_name": "Impala Daemon Verbose Log Level", - "name": "log_verbose_level", - "value": "1" - }, - { - "desc": "The health test thresholds on the resident size of the process.", - "display_name": "Resident Set Size Thresholds", - "name": "process_resident_set_size_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Enables the health test that the Impala Daemon's process state is consistent with the role configuration", - "display_name": "Impala Daemon Process Health Test", - "name": "impalad_scm_health_enabled", - "value": "true" - }, - { - "desc": "The maximum size (in queries) of the Impala Daemon audit event log file before a new one is created.", - "display_name": "Impala Daemon Maximum Audit Log File Size", - "name": "max_audit_event_log_file_size", - "value": "5000" - }, - { - "desc": "Timeout for requests to the Hive Metastore Server from Impala. Consider increasing this if you have tables with a lot of metadata and see timeout errors.", - "display_name": "Impala Daemon Hive Metastore Connection Timeout", - "name": "hive_metastore_timeout", - "value": "3600" - }, - { - "desc": "Memory limit in bytes for Impala Daemon, enforced by the daemon itself. If reached, queries running on the Impala Daemon may be killed. Leave it blank to let Impala pick its own limit. Use a value of -1 B to specify no limit.", - "display_name": "Impala Daemon Memory Limit", - "name": "impalad_memory_limit", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The amount of time to wait for the Impala Daemon to fully start up and connect to the StateStore before enforcing the connectivity check.", - "display_name": "Impala Daemon Connectivity Tolerance at Startup", - "name": "impalad_connectivity_tolerance", - "value": "180" - }, - { - "desc": "Port where Llama notification callback should be started", - "display_name": "Llama Callback Port", - "name": "llama_callback_port", - "value": "28000" - }, - { - "desc": "Local path to the certificate presented by the Impala daemon debug webserver. This file must be in .pem format. If empty, webserver SSL/TLS support is not enabled.", - "display_name": "SSL/TLS Certificate for Impala Daemon Webserver", - "name": "webserver_certificate_file", - "value": null - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "impalad_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's Directories where Impala Daemon will write data such as spilling information to disk to free up memory. This can potentially be large amounts of data..", - "display_name": "Impala Daemon Scratch Directories Free Space Monitoring Absolute Thresholds", - "name": "impalad_scratch_directories_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into hdfs-site.xml for this role only.", - "display_name": "Impala Daemon HDFS Advanced Configuration Snippet (Safety Valve)", - "name": "impala_hdfs_site_conf_safety_valve", - "value": null - }, - { - "desc": "The directory in which Impala daemon lineage log files are written. If \"Impala Lineage Generation\" property is enabled, Impala generates its lineage logs in this directory.", - "display_name": "Impala Daemon Lineage Log Directory", - "name": "lineage_event_log_dir", - "value": "/var/log/impalad/lineage" - }, - { - "desc": "Password for Impala Daemon webserver authentication.", - "display_name": "Impala Daemon Web Server User Password", - "name": "webserver_htpassword_password", - "value": null - }, - { - "desc": "The minimum log level for Impala Daemon logs", - "display_name": "Impala Daemon Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "The number of log files that are kept for each severity level before all older log files are removed. The number has to be greater than 1 to keep at least the current log file open. If set to 0, all log files are retained and log rotation is effectively disabled.", - "display_name": "Impala Maximum Log Files", - "name": "max_log_files", - "value": "10" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/impala-llama.json b/sahara/plugins/cdh/v5_4_0/resources/impala-llama.json deleted file mode 100644 index 30cf4039..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/impala-llama.json +++ /dev/null @@ -1,368 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Kerberos principal used by the Impala Llama ApplicationMaster roles.", - "display_name": "Role-Specific Kerberos Principal", - "name": "kerberos_role_princ_name", - "value": "llama" - }, - { - "desc": "Whether Llama should cache allocated resources on release.", - "display_name": "Enable Resource Caching", - "name": "llama_am_cache_enabled", - "value": "true" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Directory where Llama will place its log files.", - "display_name": "Llama Log Directory", - "name": "llama_log_dir", - "value": "/var/log/impala-llama" - }, - { - "desc": "Queues Llama ApplicationMaster should connect to at start up.", - "display_name": "Core Queues", - "name": "llama_am_core_queues", - "value": "" - }, - { - "desc": "Maximum amount of time the backed off reservations will be in 'backed off' state. The actual amount time is a random value between the minimum and the maximum.", - "display_name": "Anti-Deadlock Maximum Delay", - "name": "llama_am_gang_anti_deadlock_max_delay_ms", - "value": "30000" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "ACL for Impala ApplicationMaster clients. The ACL is a comma-separated list of user and group names. The user and group list is separated by a blank. For e.g. \"alice,bob users,wheel\". A special value of \"*\" means all users are allowed. These take effect only if security is enabled.", - "display_name": "Client ACLs", - "name": "llama_am_server_thrift_client_acl", - "value": "*" - }, - { - "desc": "For advanced use only, a string to be inserted into llama-site.xml for this role only.", - "display_name": "Impala Llama ApplicationMaster Advanced Configuration Snippet (Safety Valve) for llama-site.xml", - "name": "llama_config_valve", - "value": null - }, - { - "desc": "When computing the overall Impala Llama ApplicationMaster health, consider the host's health.", - "display_name": "Impala Llama ApplicationMaster Host Health Test", - "name": "llama_host_health_enabled", - "value": "true" - }, - { - "desc": "Port on which the Llama ApplicationMaster listens to HTTP requests.", - "display_name": "Llama HTTP Port", - "name": "llama_http_port", - "value": "15001" - }, - { - "desc": "Minimum number of threads used by the Llama ApplicationMaster auxiliary service uses for serving client requests.", - "display_name": "Thrift Server Minimum Threads", - "name": "llama_am_server_thrift_server_min_threads", - "value": "10" - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "Enter an XML string that will be inserted verbatim into the Fair Scheduler allocations file. Overrides the configuration set using the Pools configuration UI. This configuration only has effect on Impala versions 1.3 or greater.", - "display_name": "Fair Scheduler XML Advanced Configuration Snippet (Safety Valve)", - "name": "llama_fair_scheduler_safety_valve", - "value": null - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Client notification retry interval, in milliseconds.", - "display_name": "Client Notification Retry Interval", - "name": "llama_am_server_thrift_client_notifier_retry_interval_ms", - "value": "5000" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The group that the Llama processes should run as.", - "display_name": "Llama System Group", - "name": "process_groupname", - "value": "llama" - }, - { - "desc": "Timeout policy for resources being cached.", - "display_name": "Resource Caching Idle Timeout", - "name": "llama_am_cache_eviction_timeout_policy_idle_timeout_ms", - "value": "30000" - }, - { - "desc": "Port on which the Llama ApplicationMaster listens to administrative requests on its administrative Thrift interface.", - "display_name": "Llama Thrift Admin Port", - "name": "llama_am_server_thrift_admin_address", - "value": "15002" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Impala Llama ApplicationMaster in Bytes", - "name": "llama_java_heapsize", - "value": "268435456" - }, - { - "desc": "Interval of time without any new allocation that will trigger the Impala ApplicationMaster anti-deadlock logic.", - "display_name": "Anti-Deadlock No Allocation Limit Interval", - "name": "llama_am_gang_anti_deadlock_no_allocation_limit_ms", - "value": "30000" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Maximum number of threads used by the Llama ApplicationMaster auxiliary service uses for serving client requests.", - "display_name": "Thrift Server Maximum Threads", - "name": "llama_am_server_thrift_server_max_threads", - "value": "50" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Impala Llama ApplicationMaster Environment Advanced Configuration Snippet (Safety Valve)", - "name": "LLAMA_role_env_safety_valve", - "value": null - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Time in milliseconds after which Llama will discard its AM for a queue that has been empty of reservations. Does not apply to queues specified with the Core Queues property.", - "display_name": "Queue Expiration Age", - "name": "llama_am_queue_expire_ms", - "value": "300000" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Impala Llama ApplicationMaster Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "Minimum amount of time the backed off reservations will be in 'backed off' state. The actual amount time is a random value between the minimum and the maximum.", - "display_name": "Anti-Deadlock Minimum Delay", - "name": "llama_am_gang_anti_deadlock_min_delay_ms", - "value": "10000" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "llama_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "If enabled, the Impala Llama ApplicationMaster binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind Impala Llama ApplicationMaster to Wildcard Address", - "name": "llama_bind_wildcard", - "value": "false" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The maximum size, in megabytes, per log file for Impala Llama ApplicationMaster logs. Typically used by log4j or logback.", - "display_name": "Impala Llama ApplicationMaster Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Llama Server", - "name": "llama_java_opts", - "value": "" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Llama ApplicationMaster heartbeat interval, in milliseconds. On each heartbeat the ApplicationMaster submits new reservations to YARN ResourceManager and gets updates from it.", - "display_name": "AM Heartbeat Interval", - "name": "llama_am_server_thrift_client_notifier_heartbeat_ms", - "value": "5000" - }, - { - "desc": "The user that the Llama process should run as.", - "display_name": "Llama System User", - "name": "process_username", - "value": "llama" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Percentage of resources that will be backed off by the Impala ApplicationMaster anti-deadlock logic. Random reservations will be backed off until the percentage of backed off resources reaches this percentage.", - "display_name": "Anti-Deadlock Backoff Percentage", - "name": "llama_am_gang_anti_deadlock_backoff_percent", - "value": "30" - }, - { - "desc": "Maximum number of retries for a client notification. After the maximum number of client notification retries has been reached without success the client is considered lost and all its reservations are released. A successful client notification resets the retries count.", - "display_name": "Maximum Client Notification Retries", - "name": "llama_am_server_thrift_client_notifier_max_retries", - "value": "5" - }, - { - "desc": "Port on which the Llama ApplicationMaster serves its Thrift interface.", - "display_name": "Llama Thrift Port", - "name": "llama_port", - "value": "15000" - }, - { - "desc": "Whether to break resource requests into smaller requests of standard size before the resource cache. The sizes are taken from Yarn settings Container Memory Increment and Container Virtual CPU Cores Increment.", - "display_name": "Enable Resource Cache Normalization", - "name": "llama_am_resource_normalizing_enabled", - "value": "true" - }, - { - "desc": "The maximum number of rolled log files to keep for Impala Llama ApplicationMaster logs. Typically used by log4j or logback.", - "display_name": "Impala Llama ApplicationMaster Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "Socket time, in milliseconds, used Llama ApplicationMaster auxiliary service for all its server and client Thrift connections.", - "display_name": "Thrift Transport Timeout", - "name": "llama_am_server_thrift_transport_timeout_ms", - "value": "60000" - }, - { - "desc": "Enables the health test that the Impala Llama ApplicationMaster's process state is consistent with the role configuration", - "display_name": "Impala Llama ApplicationMaster Process Health Test", - "name": "llama_scm_health_enabled", - "value": "true" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "ACL for Impala ApplicationMaster admins. The ACL is a comma-separated list of user and group names. The user and group list is separated by a blank. For e.g. \"alice,bob users,wheel\". A special value of \"*\" means all users are allowed. These take effect only if security is enabled.", - "display_name": "Administrative Interface ACLs", - "name": "llama_am_server_thrift_admin_acl", - "value": "*" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "The minimum log level for Impala Llama ApplicationMaster logs", - "display_name": "Impala Llama ApplicationMaster Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/impala-service.json b/sahara/plugins/cdh/v5_4_0/resources/impala-service.json deleted file mode 100644 index 0c160685..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/impala-service.json +++ /dev/null @@ -1,356 +0,0 @@ -[ - { - "desc": "The health test thresholds for the assignment locality health test. Specified as a percentage of total assignments.", - "display_name": "Assignment Locality Ratio Thresholds", - "name": "impala_assignment_locality_thresholds", - "value": "{\"critical\":\"5.0\",\"warning\":\"80.0\"}" - }, - { - "desc": "Name of the HBase service that this Impala service instance depends on", - "display_name": "HBase Service", - "name": "hbase_service", - "value": null - }, - { - "desc": "When set, this parameter is used to convert the username into the LDAP Distinguished Name (DN), so that the resulting DN looks like uid=username,X. For example, if this parameter is set to \"ou=People,dc=cloudera,dc=com\", and the username passed in is \"mike\", the resulting authentication passed to the LDAP server looks like \"uid=mike,ou=People,dc=cloudera,dc=com\". This parameter is frequently useful when authenticating against an OpenLDAP server. This parameter is mutually exclusive with LDAP Domain and LDAP Pattern.", - "display_name": "LDAP BaseDN", - "name": "ldap_baseDN", - "value": null - }, - { - "desc": "The user that this Impala's processes should run as (except Llama, which has its own user).", - "display_name": "Impala System User (except Llama)", - "name": "process_username", - "value": "impala" - }, - { - "desc": "JSON representation of all the configurations that the Fair Scheduler can take on across all schedules. Typically edited using the Pools configuration UI. This configuration only has effect on Impala versions 1.3 or greater.", - "display_name": "Fair Scheduler Allocations", - "name": "impala_scheduled_allocations", - "value": "{\"defaultMinSharePreemptionTimeout\":null,\"defaultQueueSchedulingPolicy\":null,\"fairSharePreemptionTimeout\":null,\"queueMaxAMShareDefault\":null,\"queueMaxAppsDefault\":null,\"queuePlacementRules\":null,\"queues\":[{\"aclAdministerApps\":null,\"aclSubmitApps\":null,\"minSharePreemptionTimeout\":null,\"name\":\"root\",\"queues\":[{\"aclAdministerApps\":null,\"aclSubmitApps\":null,\"minSharePreemptionTimeout\":null,\"name\":\"default\",\"queues\":[],\"schedulablePropertiesList\":[{\"impalaMaxMemory\":null,\"impalaMaxQueuedQueries\":null,\"impalaMaxRunningQueries\":null,\"maxAMShare\":null,\"maxResources\":null,\"maxRunningApps\":null,\"minResources\":null,\"scheduleName\":\"default\",\"weight\":null}],\"schedulingPolicy\":null}],\"schedulablePropertiesList\":[{\"impalaMaxMemory\":null,\"impalaMaxQueuedQueries\":null,\"impalaMaxRunningQueries\":null,\"maxAMShare\":null,\"maxResources\":null,\"maxRunningApps\":null,\"minResources\":null,\"scheduleName\":\"default\",\"weight\":null}],\"schedulingPolicy\":null}],\"userMaxAppsDefault\":null,\"users\":[]}" - }, - { - "desc": "Event filters are defined in a JSON object like the following:
\n\n\n{\n \"defaultAction\" : (\"accept\", \"discard\"),\n \"rules\" : [\n {\n \"action\" : (\"accept\", \"discard\"),\n \"fields\" : [\n {\n \"name\" : \"fieldName\",\n \"match\" : \"regex\"\n }\n ]\n }\n ]\n}\n\n\n\nA filter has a default action and a list of rules, in order of precedence.\nEach rule defines an action, and a list of fields to match against the\naudit event.\n
\n\n\nA rule is \"accepted\" if all the listed field entries match the audit\nevent. At that point, the action declared by the rule is taken.\n
\n\n\nIf no rules match the event, the default action is taken. Actions\ndefault to \"accept\" if not defined in the JSON object.\n
\n\n\nThe following is the list of fields that can be filtered for Impala events:\n
\n\n\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n
\n\n\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n
\n\nEvent trackers are defined in a JSON object like the following:
\n\n\n{\n \"timeToLive\" : [integer],\n \"fields\" : [\n {\n \"type\" : [string],\n \"name\" : [string]\n }\n ]\n}\n\n\n\nWhere:\n
\n\n\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n
\n\n\nThe following is the list of fields that can be used to compare Impala events:\n
\n\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "If true, attempts to establish a TLS (Transport Layer Security) connection with the LDAP server. Only supported in Impala 1.4 or CDH 5.1 or higher. Not required when using an LDAP URI with prefix ldaps://, because that already specifies TLS.", - "display_name": "Enable LDAP TLS", - "name": "enable_ldap_tls", - "value": "false" - }, - { - "desc": "Controls which queries admin users can see in the queries list view", - "display_name": "Admin Users Query List Visibility Settings", - "name": "admin_query_list_settings", - "value": "ALL" - }, - { - "desc": "Encrypt communication between clients (like ODBC, JDBC, and the Impala shell) and the Impala daemon using Transport Layer Security (TLS) (formerly known as Secure Socket Layer (SSL)).", - "display_name": "Enable TLS/SSL for Impala Client Services", - "name": "client_services_ssl_enabled", - "value": "false" - }, - { - "desc": "The minimum number of assignments that must occur during the test time period before the threshold values will be checked. Until this number of assignments have been observed in the test time period the health test will be disabled.", - "display_name": "Assignment Locality Minimum Assignments", - "name": "impala_assignment_locality_minimum", - "value": "10" - }, - { - "desc": "Enable collection of lineage from the service's roles.", - "display_name": "Enable Lineage Collection", - "name": "navigator_lineage_enabled", - "value": "true" - }, - { - "desc": "Name of YARN service to use for resource management integration between Impala and YARN. This service dependency and the existence of a Llama role is required for using said integration.", - "display_name": "YARN Service for Resource Management", - "name": "yarn_service", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for navigator.lineage.client.properties.", - "display_name": "Impala Client Advanced Configuration Snippet (Safety Valve) for navigator.lineage.client.properties", - "name": "navigator_lineage_client_config_safety_valve", - "value": null - }, - { - "desc": "Local path to the X509 certificate that will identify the Impala daemon to clients during SSL/TLS connections. This file must be in PEM format.", - "display_name": "SSL/TLS Certificate for Clients", - "name": "ssl_server_certificate", - "value": null - }, - { - "desc": "Timeout in milliseconds for all HBase RPCs made by Impala. Overrides configuration in HBase service.", - "display_name": "HBase RPC Timeout", - "name": "hbase_rpc_timeout", - "value": "3000" - }, - { - "desc": "Name of the Hive service that this Impala service instance depends on", - "display_name": "Hive Service", - "name": "hive_service", - "value": null - }, - { - "desc": "Enable collection of audit events from the service's roles.", - "display_name": "Enable Audit Collection", - "name": "navigator_audit_enabled", - "value": "true" - }, - { - "desc": "A list specifying the rules to run to determine which Fair Scheduler configuration to use. Typically edited using the Rules configuration UI. This configuration only has effect on Impala versions 1.3 or greater.", - "display_name": "Fair Scheduler Configuration Rules", - "name": "impala_schedule_rules", - "value": "[]" - }, - { - "desc": "When checked, LDAP-based authentication for users is enabled. Usernames and passwords are transmitted in the clear unless encryption is turned on. To encrypt the network traffic from the Impala daemon to the LDAP server, use either an ldaps:// URI or select 'Enable LDAP TLS'. To encrypt network traffic from clients to the Impala daemon, specify 'Enable TLS/SSL for Impala Client Services'.", - "display_name": "Enable LDAP Authentication", - "name": "enable_ldap_auth", - "value": "false" - }, - { - "desc": "When computing the overall IMPALA health, consider Impala Catalog Server's health", - "display_name": "Impala Catalog Server Role Health Test", - "name": "impala_catalogserver_health_enabled", - "value": "true" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "Configures the maximum number of concurrently running queries for admission control when using a single pool. -1 indicates no limit and 0 indicates all incoming requests will be rejected. Ignored when Dynamic Resource Pools for Admission Control is enabled.", - "display_name": "Single Pool Max Running Queries", - "name": "admission_control_single_pool_max_requests", - "value": "200" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/impala-statestore.json b/sahara/plugins/cdh/v5_4_0/resources/impala-statestore.json deleted file mode 100644 index 3f650226..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/impala-statestore.json +++ /dev/null @@ -1,236 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "statestore_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", - "display_name": "Health Check Startup Tolerance", - "name": "statestore_startup_tolerance", - "value": "5" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be added (verbatim) to StateStore command line flags.", - "display_name": "Statestore Command Line Argument Advanced Configuration Snippet (Safety Valve)", - "name": "statestore_cmd_args_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "statestore_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Enables the health test that the Impala StateStore's process state is consistent with the role configuration", - "display_name": "Impala StateStore Process Health Test", - "name": "statestore_scm_health_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Port where StateStoreService is exported.", - "display_name": "StateStore Service Port", - "name": "state_store_port", - "value": "24000" - }, - { - "desc": "When computing the overall Impala StateStore health, consider the host's health.", - "display_name": "Impala StateStore Host Health Test", - "name": "statestore_host_health_enabled", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Username for Statestore webserver authentication.", - "display_name": "Statestore Web Server Username", - "name": "webserver_htpassword_user", - "value": null - }, - { - "desc": "Local path to the certificate presented by the StateStore debug webserver. This file must be in .pem format. If empty, webserver SSL/TLS support is not enabled.", - "display_name": "SSL/TLS Certificate for Statestore Webserver", - "name": "webserver_certificate_file", - "value": null - }, - { - "desc": "Maximum number of tasks allowed to be pending at the thread manager underlying the StateStore Thrift server (0 allows infinitely many pending tasks)", - "display_name": "Maximum StateStore Pending Tasks", - "name": "state_store_pending_task_count_max", - "value": "0" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Port where StateStore debug web server runs.", - "display_name": "StateStore HTTP Server Port", - "name": "statestore_webserver_port", - "value": "25010" - }, - { - "desc": "Directory where StateStore will place its log files.", - "display_name": "StateStore Log Directory", - "name": "log_dir", - "value": "/var/log/statestore" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Impala StateStore logs. Typically used by log4j or logback.", - "display_name": "Impala StateStore Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "Number of worker threads for the thread manager underlying the StateStore Thrift server.", - "display_name": "StateStore Worker Threads", - "name": "state_store_num_server_worker_threads", - "value": "4" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Verbose logging level for the GLog logger. These messages are always logged at 'INFO' log level, so this setting has no effect if Logging Threshold is set to 'WARN' or above. ", - "display_name": "StateStore Verbose Log Level", - "name": "log_verbose_level", - "value": "1" - }, - { - "desc": "The health test thresholds on the resident size of the process.", - "display_name": "Resident Set Size Thresholds", - "name": "process_resident_set_size_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Buffer log messages logged at this level or lower (-1 means don't buffer; 0 means buffer INFO only; 1 means buffer WARNING only, ...)", - "display_name": "StateStore Log Buffer Level", - "name": "logbuflevel", - "value": "0" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Enable/Disable StateStore web server. This web server contains useful information about StateStore daemon.", - "display_name": "Enable StateStore Web Server", - "name": "statestore_enable_webserver", - "value": "true" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Impala StateStore Environment Advanced Configuration Snippet (Safety Valve)", - "name": "STATESTORE_role_env_safety_valve", - "value": null - }, - { - "desc": "Password for Statestore webserver authentication.", - "display_name": "Statestore Web Server User Password", - "name": "webserver_htpassword_password", - "value": null - }, - { - "desc": "The minimum log level for Impala StateStore logs", - "display_name": "Impala StateStore Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "The number of log files that are kept for each severity level before all older log files are removed. The number has to be greater than 1 to keep at least the current log file open. If set to 0, all log files are retained and log rotation is effectively disabled.", - "display_name": "StateStore Maximum Log Files", - "name": "max_log_files", - "value": "10" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "statestore_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/kms-kms.json b/sahara/plugins/cdh/v5_4_0/resources/kms-kms.json deleted file mode 100644 index 76960485..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/kms-kms.json +++ /dev/null @@ -1,320 +0,0 @@ -[ - { - "desc": "Allows the oozie superuser to impersonate any members of a comma-delimited list of groups. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", - "display_name": "Oozie Proxy User Groups", - "name": "hadoop_kms_proxyuser_oozie_groups", - "value": "*" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "kms_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Allows the mapred superuser to impersonate any members of a comma-delimited list of groups. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", - "display_name": "Mapred Proxy User Groups", - "name": "hadoop_kms_proxyuser_mapred_groups", - "value": "*" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the mapred user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", - "display_name": "Mapred Proxy User Hosts", - "name": "hadoop_kms_proxyuser_mapred_hosts", - "value": "*" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the httpfs user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", - "display_name": "HttpFS Proxy User Hosts", - "name": "hadoop_kms_proxyuser_httpfs_hosts", - "value": "*" - }, - { - "desc": "Maximum number of threads used to handle KMS requests.", - "display_name": "KMS Max Threads", - "name": "kms_max_threads", - "value": "250" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the oozie user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", - "display_name": "Oozie Proxy User Hosts", - "name": "hadoop_kms_proxyuser_oozie_hosts", - "value": "*" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the yarn user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", - "display_name": "YARN Proxy User Hosts", - "name": "hadoop_kms_proxyuser_yarn_hosts", - "value": "*" - }, - { - "desc": "Enables the health test that the Key Management Server's process state is consistent with the role configuration", - "display_name": "Key Management Server Process Health Test", - "name": "kms_scm_health_enabled", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The password for the Key Management Server JKS keystore file.", - "display_name": "Key Management Server TLS/SSL Server JKS Keystore File Password", - "name": "ssl_server_keystore_password", - "value": "" - }, - { - "desc": "Allows the hive superuser to impersonate any members of a comma-delimited list of groups. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", - "display_name": "Hive Proxy User Groups", - "name": "hadoop_kms_proxyuser_hive_groups", - "value": "*" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the HTTP user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", - "display_name": "HTTP Proxy User Hosts", - "name": "hadoop_kms_proxyuser_HTTP_hosts", - "value": "*" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the hdfs user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", - "display_name": "HDFS Proxy User Hosts", - "name": "hadoop_kms_proxyuser_hdfs_hosts", - "value": "*" - }, - { - "desc": "Allows the hdfs superuser to impersonate any members of a comma-delimited list of groups. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", - "display_name": "HDFS Proxy User Groups", - "name": "hadoop_kms_proxyuser_hdfs_groups", - "value": "*" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into kms-acls.xml for this role only.", - "display_name": "Key Management Server Advanced Configuration Snippet (Safety Valve) for kms-acls.xml", - "name": "kms-acls.xml_role_safety_valve", - "value": null - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the hue user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", - "display_name": "Hue Proxy User Hosts", - "name": "hadoop_kms_proxyuser_hue_hosts", - "value": "*" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Encrypt communication between clients and Key Management Server using Transport Layer Security (TLS) (formerly known as Secure Socket Layer (SSL)).", - "display_name": "Enable TLS/SSL for Key Management Server", - "name": "ssl_enabled", - "value": "false" - }, - { - "desc": "The log directory for log files of the role Key Management Server.", - "display_name": "Key Management Server Log Directory", - "name": "log_dir", - "value": "/var/log/hadoop-kms" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the hive user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", - "display_name": "Hive Proxy User Hosts", - "name": "hadoop_kms_proxyuser_hive_hosts", - "value": "*" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Key Management Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Key Management Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "KMS_role_env_safety_valve", - "value": null - }, - { - "desc": "Port used by clients to interact with the KMS.", - "display_name": "KMS HTTP Port", - "name": "kms_http_port", - "value": "16000" - }, - { - "desc": "Allows the yarn superuser to impersonate any members of a comma-delimited list of groups. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", - "display_name": "YARN Proxy User Groups", - "name": "hadoop_kms_proxyuser_yarn_groups", - "value": "*" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "Allows the hue superuser to impersonate any members of a comma-delimited list of groups. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", - "display_name": "Hue Proxy User Groups", - "name": "hadoop_kms_proxyuser_hue_groups", - "value": "*" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the flume user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", - "display_name": "Flume Proxy User Hosts", - "name": "hadoop_kms_proxyuser_flume_hosts", - "value": "*" - }, - { - "desc": "The maximum size, in megabytes, per log file for Key Management Server logs. Typically used by log4j or logback.", - "display_name": "Key Management Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The password for the JavaKeyStoreProvider keystore file.", - "display_name": "JavaKeyStoreProvider Password", - "name": "hadoop_security_keystore_javaKeyStoreProvider_password", - "value": null - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When computing the overall Key Management Server health, consider the host's health.", - "display_name": "Key Management Server Host Health Test", - "name": "kms_host_health_enabled", - "value": "true" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "A comma-separated list of users (no spaces) for whom to disallow access to key material. These users can still fetch key metadata and create encrypted encryption keys, but are unable to do any other KMS operations. Typically, HDFS superusers will be specified here.", - "display_name": "KMS Blacklist Users", - "name": "kms_blacklist_users", - "value": "" - }, - { - "desc": "The path to the TLS/SSL keystore file containing the server certificate and private key used for TLS/SSL. Used when Key Management Server is acting as a TLS/SSL server. The keystore must be in JKS format.", - "display_name": "Key Management Server TLS/SSL Server JKS Keystore File Location", - "name": "ssl_server_keystore_location", - "value": "" - }, - { - "desc": "Allows the httpfs superuser to impersonate any members of a comma-delimited list of groups. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", - "display_name": "HttpFS Proxy User Groups", - "name": "hadoop_kms_proxyuser_httpfs_groups", - "value": "*" - }, - { - "desc": "Directory where configuration and binaries are staged before starting KMS. Does not normally need to be modified.", - "display_name": "KMS Staging Directory", - "name": "kms_staging_dir", - "value": "/var/lib/hadoop-kms" - }, - { - "desc": "For advanced use only, a string to be inserted into core-site.xml for this role only.", - "display_name": "Key Management Server Advanced Configuration Snippet (Safety Valve) for core-site.xml", - "name": "core-site.xml_role_safety_valve", - "value": null - }, - { - "desc": "The maximum number of rolled log files to keep for Key Management Server logs. Typically used by log4j or logback.", - "display_name": "Key Management Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "Allows the flume superuser to impersonate any members of a comma-delimited list of groups. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", - "display_name": "Flume Proxy User Groups", - "name": "hadoop_kms_proxyuser_flume_groups", - "value": "*" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Allows the HTTP superuser to impersonate any members of a comma-delimited list of groups. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", - "display_name": "HTTP Proxy User Groups", - "name": "hadoop_kms_proxyuser_HTTP_groups", - "value": "*" - }, - { - "desc": "Directory of the keystore file kms.keystore used by JavaKeyStoreProvider that backs the KMS.", - "display_name": "JavaKeyStoreProvider Directory", - "name": "hadoop_security_key_provider_dir", - "value": "/var/lib/kms" - }, - { - "desc": "For advanced use only, a string to be inserted into kms-site.xml for this role only.", - "display_name": "Key Management Server Advanced Configuration Snippet (Safety Valve) for kms-site.xml", - "name": "kms-site.xml_role_safety_valve", - "value": null - }, - { - "desc": "The minimum log level for Key Management Server logs", - "display_name": "Key Management Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Maximum heap size of the KMS.", - "display_name": "KMS Heap Size", - "name": "kms_heap_size", - "value": "1073741824" - }, - { - "desc": "Port used to access the KMS' embedded Tomcat admin console.", - "display_name": "KMS Admin Port", - "name": "kms_admin_port", - "value": "16001" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/kms-service.json b/sahara/plugins/cdh/v5_4_0/resources/kms-service.json deleted file mode 100644 index 662c1e5b..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/kms-service.json +++ /dev/null @@ -1,50 +0,0 @@ -[ - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The group that this service's processes should run as.", - "display_name": "System Group", - "name": "process_groupname", - "value": "kms" - }, - { - "desc": "Authentication type for the KMS. Can either be \"simple\" or \"kerberos\".", - "display_name": "Authentication Type", - "name": "hadoop_kms_authentication_type", - "value": "simple" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The user that this service's processes should run as.", - "display_name": "System User", - "name": "process_username", - "value": "kms" - }, - { - "desc": "The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Java KeyStore KMS Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "KMS_service_env_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/ks_indexer-hbase_indexer.json b/sahara/plugins/cdh/v5_4_0/resources/ks_indexer-hbase_indexer.json deleted file mode 100644 index 50bca7d8..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/ks_indexer-hbase_indexer.json +++ /dev/null @@ -1,242 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Kerberos principal used by the Lily HBase Indexer roles.", - "display_name": "Role-Specific Kerberos Principal", - "name": "kerberos_role_princ_name", - "value": "hbase" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "HTTP port used by HBase Indexer.", - "display_name": "HBase Indexer HTTP Port", - "name": "hbase_indexer_http_port", - "value": "11060" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "hbase_indexer_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for Lily HBase Indexer logs. Typically used by log4j or logback.", - "display_name": "Lily HBase Indexer Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The group that the HBase Indexer process should run as.", - "display_name": "System Group", - "name": "hbase_indexer_process_groupname", - "value": "hbase" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Lily HBase Indexer", - "name": "hbase_indexer_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "For advanced use only, a string to be inserted into hbase-indexer-site.xml for this role only.", - "display_name": "Lily HBase Indexer Advanced Configuration Snippet (Safety Valve) for hbase-indexer-site.xml", - "name": "hbase_indexer_config_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Lily HBase Indexer Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Lily HBase Indexer Environment Advanced Configuration Snippet (Safety Valve)", - "name": "HBASE_INDEXER_role_env_safety_valve", - "value": null - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Lily HBase Indexer logs. Typically used by log4j or logback.", - "display_name": "Lily HBase Indexer Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Directory where HBase Indexer will place its log files.", - "display_name": "HBase Indexer Log Directory", - "name": "hbase_indexer_log_dir", - "value": "/var/log/hbase-solr" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "When computing the overall Lily HBase Indexer health, consider the host's health.", - "display_name": "Lily HBase Indexer Host Health Test", - "name": "hbase_indexer_host_health_enabled", - "value": "true" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Enables the health test that the Lily HBase Indexer's process state is consistent with the role configuration", - "display_name": "Lily HBase Indexer Process Health Test", - "name": "hbase_indexer_scm_health_enabled", - "value": "true" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The minimum log level for Lily HBase Indexer logs", - "display_name": "Lily HBase Indexer Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "The user that the HBase Indexer process should run as.", - "display_name": "System User", - "name": "hbase_indexer_process_username", - "value": "hbase" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Lily HBase Indexer in Bytes", - "name": "hbase_indexer_java_heapsize", - "value": "1073741824" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/ks_indexer-service.json b/sahara/plugins/cdh/v5_4_0/resources/ks_indexer-service.json deleted file mode 100644 index 4adb518f..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/ks_indexer-service.json +++ /dev/null @@ -1,110 +0,0 @@ -[ - { - "desc": "Name of the Solr service that this Key-Value Store Indexer service instance depends on", - "display_name": "Solr Service", - "name": "solr_service", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Name of the HBase service that this Key-Value Store Indexer service instance depends on", - "display_name": "HBase Service", - "name": "hbase_service", - "value": null - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Key-Value Store Indexer Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "ks_indexer_env_safety_valve", - "value": null - }, - { - "desc": "Text that goes verbatim into custom-mimetypes.xml file used by HBase Indexers.", - "display_name": "Custom Mime-types File", - "name": "custom_mimetypes_file", - "value": "\n\n\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/mapred-site.xml b/sahara/plugins/cdh/v5_4_0/resources/mapred-site.xml deleted file mode 100644 index 5228ed17..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/mapred-site.xml +++ /dev/null @@ -1,157 +0,0 @@ - - - -The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Oozie Server", - "name": "oozie_java_opts", - "value": "" - }, - { - "desc": "The admin port Oozie server runs.", - "display_name": "Oozie Admin Port", - "name": "oozie_admin_port", - "value": "11001" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing the moving average of extra time the pause monitor spent paused.", - "display_name": "Pause Duration Monitoring Period", - "name": "oozie_server_pause_duration_window", - "value": "5" - }, - { - "desc": "Type of the database used by Oozie Server.", - "display_name": "Oozie Server Database Type", - "name": "oozie_database_type", - "value": "derby" - }, - { - "desc": "If true, enables the Oozie Server web console. ExtJS 2.2 zip archive must be extracted to /var/lib/oozie on the same host as the Oozie Server.", - "display_name": "Enable Oozie Server Web Console", - "name": "oozie_web_console", - "value": "false" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "oozie_server_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Username for connecting to the database used by Oozie Server. Does not apply if you are using Derby as the database type.", - "display_name": "Oozie Server Database User", - "name": "oozie_database_user", - "value": "sa" - }, - { - "desc": "The location on disk of the trust store, in .jks format, used to confirm the authenticity of TLS/SSL servers that Oozie might connect to. This is used when Oozie is the client in a TLS/SSL connection. This trust store must contain the certificate(s) used to sign the service(s) being connected to. If this parameter is not provided, the default list of well-known certificate authorities is used instead.", - "display_name": "Oozie TLS/SSL Certificate Trust Store File", - "name": "oozie_https_truststore_file", - "value": null - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "oozie_server_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds of the weighted average size of the Oozie Server callable queue over a recent period. See also Oozie Server Callable Queue Monitoring Period.", - "display_name": "Oozie Server Callable Queue Monitoring Threshold", - "name": "oozie_server_callable_queue_threshold", - "value": "{\"critical\":\"95.0\",\"warning\":\"80.0\"}" - }, - { - "desc": "Directory where the Oozie Server places its data. Only applicable when using Derby as the database type.", - "display_name": "Oozie Server Data Directory", - "name": "oozie_data_dir", - "value": "/var/lib/oozie/data" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The password for the Oozie JKS keystore file.", - "display_name": "Oozie TLS/SSL Server JKS Keystore File Password", - "name": "oozie_https_keystore_password", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Completed workflow jobs older than this value, in days, will be purged by the PurgeService.", - "display_name": "Days to Keep Completed Workflow Jobs", - "name": "purgeservice_older_than", - "value": "30" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "oozie_server_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Oozie Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The path to the TLS/SSL keystore file containing the server certificate and private key used for TLS/SSL. Used when Oozie is acting as a TLS/SSL server. The keystore must be in JKS format.", - "display_name": "Oozie TLS/SSL Server JKS Keystore File Location", - "name": "oozie_https_keystore_file", - "value": "/var/lib/oozie/.keystore" - }, - { - "desc": "The from address to be used for mailing all emails for Oozie email action", - "display_name": "Oozie Email Action From Address", - "name": "oozie_email_from_address", - "value": "oozie@localhost" - }, - { - "desc": "Port of the Oozie Server while using SSL.", - "display_name": "Oozie HTTPS Port", - "name": "oozie_https_port", - "value": "11443" - }, - { - "desc": "Comma-separated list of SchemaService workflow extension schemas for additional action types. From CDH 5.4 and higher, this parameter is used only to specify additional schemas for workflows. All schemas included in that release will be added automatically and do not need to be specified.", - "display_name": "Oozie SchemaService Workflow Extension Schemas", - "name": "oozie_workflow_extension_schemas", - "value": "" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The SMTP server port to use for Oozie email action", - "display_name": "Oozie Email Action SMTP Port", - "name": "oozie_email_smtp_prt", - "value": "25" - }, - { - "desc": "Port of Oozie Server", - "display_name": "Oozie HTTP Port", - "name": "oozie_http_port", - "value": "11000" - }, - { - "desc": "The maximum number of rolled log files to keep for Oozie Server logs. Typically used by log4j or logback.", - "display_name": "Oozie Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "720" - }, - { - "desc": "Workflow Status metrics collection interval.", - "display_name": "Workflow Status Metrics Collection Interval", - "name": "oozie_job_metric_collection_interval", - "value": "1" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Name of the database used by Oozie Server.", - "display_name": "Oozie Server Database Name", - "name": "oozie_database_name", - "value": "oozie" - }, - { - "desc": "Directory where Oozie Server will place its log files.", - "display_name": "Oozie Server Log Directory", - "name": "oozie_log_dir", - "value": "/var/log/oozie" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Oozie Server in Bytes", - "name": "oozie_java_heapsize", - "value": "1073741824" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Whether to use the Codehale-based metrics for instrumentation. Enabling this disables the 'instrumentation' REST endpoint and enables the 'metrics' REST endpoint (<hostname:port>/v2/admin/metrics).", - "display_name": "Enable The Metrics Instrumentation Service", - "name": "oozie_use_metric_instrumentation", - "value": "true" - }, - { - "desc": "If true, enables version check for Oozie Server and installed shared libraries.", - "display_name": "Enable Oozie Server Shared Libraries Version Check", - "name": "oozie_server_shared_lib_version_check_enabled", - "value": "true" - }, - { - "desc": "Comma-separated list of ActionService executor extension classes. Only action types with associated executors can be used in workflows. For CDH 5.4 and higher, this parameter is used only to specify additional classes for workflows. All executor extension classes included in that release will be added automatically and do not need to be specified.", - "display_name": "Oozie ActionService Executor Extension Classes", - "name": "oozie_executor_extension_classes", - "value": "" - }, - { - "desc": "Whether to purge completed workflows and their corresponding coordinator actions of long-running coordinator jobs if the completed workflow jobs are older than the value specified in oozie.service.PurgeService.older.than.", - "display_name": "Enable Purge for Long-Running Coordinator Jobs", - "name": "purgeservice_purge_old_coord_action", - "value": "false" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Completed bundle jobs older than this value, in days, will be purged by the PurgeService.", - "display_name": "Days to Keep Completed Bundle Jobs", - "name": "purgeservice_bundle_older_than", - "value": "7" - }, - { - "desc": "Maximum callable queue size", - "display_name": "Maximum Callable Queue Size", - "name": "oozie_service_callablequeueservice_queue_size", - "value": "10000" - }, - { - "desc": "Completed coordinator jobs older than this value, in days, will be purged by the PurgeService.", - "display_name": "Days to Keep Completed Coordinator Jobs", - "name": "purgeservice_coord_older_than", - "value": "7" - }, - { - "desc": "For advanced use only, a string to be inserted into oozie-site.xml for this role only.", - "display_name": "Oozie Server Advanced Configuration Snippet (Safety Valve) for oozie-site.xml", - "name": "oozie_config_safety_valve", - "value": null - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "The SMTP server host to use for Oozie email action", - "display_name": "Oozie Email Action SMTP Host", - "name": "oozie_email_smtp_host", - "value": "localhost" - }, - { - "desc": "The minimum log level for Oozie Server logs", - "display_name": "Oozie Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Hostname of the database used by Oozie Server. If the port is non-default for your database type, use host:port notation. Does not apply if you are using Derby as the database type.", - "display_name": "Oozie Server Database Host", - "name": "oozie_database_host", - "value": "localhost" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Oozie Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "OOZIE_SERVER_role_env_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/oozie-service.json b/sahara/plugins/cdh/v5_4_0/resources/oozie-service.json deleted file mode 100644 index 10533d9e..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/oozie-service.json +++ /dev/null @@ -1,140 +0,0 @@ -[ - { - "desc": "Namespace used by this Oozie service in ZooKeeper when High Availability is enabled.", - "display_name": "ZooKeeper Namespace", - "name": "oozie_zookeeper_namespace", - "value": "oozie" - }, - { - "desc": "The user that this service's processes should run as.", - "display_name": "System User", - "name": "process_username", - "value": "oozie" - }, - { - "desc": "URL of the JMS Broker used by the Oozie service in JMS integration is enabled.", - "display_name": "JMS Broker", - "name": "oozie_jms_broker", - "value": "tcp://localhost:61616" - }, - { - "desc": "Encrypt communication between clients and Oozie using Transport Layer Security (TLS) (formerly known as Secure Socket Layer (SSL)).", - "display_name": "Enable TLS/SSL for Oozie", - "name": "oozie_use_ssl", - "value": "false" - }, - { - "desc": "List of event listeners used by the Oozie service. Listeners needed for JMS or SLA integration are automatically emitted if they are enabled.", - "display_name": "Oozie Event Listeners", - "name": "oozie_event_listeners", - "value": "" - }, - { - "desc": "Name of the ZooKeeper service that this Oozie service instance depends on", - "display_name": "ZooKeeper Service", - "name": "zookeeper_service", - "value": null - }, - { - "desc": "Whether to configure Oozie properties needed for SLA integration", - "display_name": "Enable SLA Integration", - "name": "oozie_use_sla", - "value": "false" - }, - { - "desc": "Kerberos principal used by all roles of this service.", - "display_name": "Kerberos Principal", - "name": "kerberos_princ_name", - "value": "oozie" - }, - { - "desc": "Root of the directory in HDFS where the Oozie ShareLibs are stored. The libraries are stored in the share/lib subdirectory under the specified root directory.", - "display_name": "ShareLib Root Directory", - "name": "oozie_sharelib_rootdir", - "value": "/user/oozie" - }, - { - "desc": "The group that this service's processes should run as.", - "display_name": "System Group", - "name": "process_groupname", - "value": "oozie" - }, - { - "desc": "Whether to configure Oozie properties needed for JMS integration", - "display_name": "Enable JMS Integration", - "name": "oozie_use_jms", - "value": "false" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Oozie Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "oozie_env_safety_valve", - "value": null - }, - { - "desc": "A list of credential class mappings for CredentialsProvider.", - "display_name": "Oozie Credential Classes", - "name": "oozie_credential_classes", - "value": "hcat=org.apache.oozie.action.hadoop.HCatCredentials,hbase=org.apache.oozie.action.hadoop.HbaseCredentials,hive2=org.apache.oozie.action.hadoop.Hive2Credentials" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Address of the load balancer used if Oozie HA is enabled. Should be specified in host:port format.", - "display_name": "Oozie Load Balancer", - "name": "oozie_load_balancer", - "value": null - }, - { - "desc": "The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "The health test thresholds of the overall Oozie Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Oozie Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Oozie Servers falls below the critical threshold.", - "display_name": "Healthy Oozie Server Monitoring Thresholds", - "name": "oozie_servers_healthy_thresholds", - "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Coordinator Job Lookup trigger command is scheduled at this interval (in seconds).", - "display_name": "Coordinator Job Lookup Interval", - "name": "oozie_service_coord_lookup_interval", - "value": "300" - }, - { - "desc": "Name of the Hive service that this Oozie service instance depends on. This is used to configure Oozie HCat integration.", - "display_name": "Hive Service", - "name": "hive_service", - "value": null - }, - { - "desc": "Service to run MapReduce jobs against", - "display_name": "MapReduce Service", - "name": "mapreduce_yarn_service", - "value": null - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "Use ACLs on Znode while a secure ZooKeeper is used for Oozie High Availability. Note: This config is not emitted if ZooKeeper is not secure.", - "display_name": "Use ACLs on Znode", - "name": "oozie_zk_secure", - "value": "true" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/priority-one-confs.json b/sahara/plugins/cdh/v5_4_0/resources/priority-one-confs.json deleted file mode 100644 index f7c77953..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/priority-one-confs.json +++ /dev/null @@ -1,45 +0,0 @@ -[ - "dfs_block_size", - "dfs_umaskmode", - "dfs_webhdfs_enabled", - "dfs_permissions", - "dfs_replication", - "io_compression_codecs", - "io_sort_mb", - "dfs_datanode_du_reserved", - "dfs_datanode_failed_volumes_tolerated", - "dfs_name_dir_restore", - "fs_trash_interval", - "dfs_safemode_min_datanodes", - "dfs_safemode_extension", - "dfs_access_time_precision", - "yarn_acl_enable", - "yarn_admin_acl", - "yarn_log_aggregation_enable", - "yarn_log_aggregation_retain_seconds", - "mapreduce_jobhistory_max_age_ms", - "mapreduce_jobhistory_cleaner_interval", - "mapreduce_reduce_memory_mb", - "mapreduce_reduce_java_opts", - "mapreduce_map_memory_mb", - "mapreduce_map_java_opts", - "yarn_nodemanager_container_manager_thread_count", - "yarn_nodemanager_delete_thread_count", - "yarn_nodemanager_heartbeat_interval_ms", - "yarn_nodemanager_localizer_cache_cleanup_interval_ms", - "yarn_nodemanager_localizer_client_thread_count", - "yarn_nodemanager_localizer_cache_target_size_mb", - "yarn_nodemanager_localizer_fetch_thread_count", - "yarn_nodemanager_log_retain_seconds", - "yarn_nodemanager_resource_memory_mb", - "yarn_resourcemanager_client_thread_count", - "yarn_resourcemanager_scheduler_client_thread_count", - "yarn_resourcemanager_admin_client_thread_count", - "yarn_resourcemanager_amliveliness_monitor_interval_ms", - "yarn_am_liveness_monitor_expiry_interval_ms", - "yarn_resourcemanager_am_max_retries", - "yarn_scheduler_minimum_allocation_mb", - "yarn_scheduler_maximum_allocation_mb", - "yarn_app_mapreduce_am_command_opts", - "yarn_app_mapreduce_am_resource_mb" -] diff --git a/sahara/plugins/cdh/v5_4_0/resources/sentry-impala-client-safety.xml b/sahara/plugins/cdh/v5_4_0/resources/sentry-impala-client-safety.xml deleted file mode 100644 index e986433a..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/sentry-impala-client-safety.xml +++ /dev/null @@ -1,16 +0,0 @@ -The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Sentry Server", - "name": "sentry_server_java_opts", - "value": "" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Directory where Sentry Server will place its log files.", - "display_name": "Sentry Server Log Directory", - "name": "sentry_server_log_dir", - "value": "/var/log/sentry" - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Sentry Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Sentry Server in Bytes", - "name": "sentry_server_java_heapsize", - "value": "1073741824" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "RPC port number of Sentry Server.", - "display_name": "Sentry Server RPC Port", - "name": "sentry_service_server_rpc_port", - "value": "8038" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "sentry_server_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Sentry Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "sentry_env_safety_valve", - "value": null - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The minimum log level for Sentry Server logs", - "display_name": "Sentry Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Enables the health test that the Sentry Server's process state is consistent with the role configuration", - "display_name": "Sentry Server Process Health Test", - "name": "sentry_server_scm_health_enabled", - "value": "true" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/sentry-service.json b/sahara/plugins/cdh/v5_4_0/resources/sentry-service.json deleted file mode 100644 index f8c51988..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/sentry-service.json +++ /dev/null @@ -1,158 +0,0 @@ -[ - { - "desc": "Password for Sentry Server database.", - "display_name": "Sentry Server Database Password", - "name": "sentry_server_database_password", - "value": "" - }, - { - "desc": "\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n
\n\n\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n
\n\nEvent trackers are defined in a JSON object like the following:
\n\n\n{\n \"timeToLive\" : [integer],\n \"fields\" : [\n {\n \"type\" : [string],\n \"name\" : [string]\n }\n ]\n}\n\n\n\nWhere:\n
\n\n\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n
\n\n\nThe following is the list of fields that can be used to compare Sentry events:\n
\n\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Port number of Sentry Server database.", - "display_name": "Sentry Server Database Port", - "name": "sentry_server_database_port", - "value": "3306" - }, - { - "desc": "If an end user is in one of these admin groups, that user has administrative privileges on the Sentry Server.", - "display_name": "Admin Groups", - "name": "sentry_service_admin_group", - "value": "hive,impala,hue" - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for navigator.client.properties.", - "display_name": "Sentry Client Advanced Configuration Snippet (Safety Valve) for navigator.client.properties", - "name": "navigator_client_config_safety_valve", - "value": null - }, - { - "desc": "Enable collection of audit events from the service's roles.", - "display_name": "Enable Audit Collection", - "name": "navigator_audit_enabled", - "value": "true" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/solr-gateway.json b/sahara/plugins/cdh/v5_4_0/resources/solr-gateway.json deleted file mode 100644 index ae9e4dc5..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/solr-gateway.json +++ /dev/null @@ -1,32 +0,0 @@ -[ - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The minimum log level for Gateway logs", - "display_name": "Gateway Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "The directory where the client configs will be deployed", - "display_name": "Deploy Directory", - "name": "client_config_root_dir", - "value": "/etc/solr" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Gateway Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", - "display_name": "Alternatives Priority", - "name": "client_config_priority", - "value": "90" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/solr-service.json b/sahara/plugins/cdh/v5_4_0/resources/solr-service.json deleted file mode 100644 index 76c64862..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/solr-service.json +++ /dev/null @@ -1,188 +0,0 @@ -[ - { - "desc": "Choose the authentication mechanism used by Solr.", - "display_name": "Solr Secure Authentication", - "name": "solr_security_authentication", - "value": "simple" - }, - { - "desc": "\nConfigures the rules for event tracking and coalescing. This feature is\nused to define equivalency between different audit events. When\nevents match, according to a set of configurable parameters, only one\nentry in the audit list is generated for all the matching events.\n
\n\n\nTracking works by keeping a reference to events when they first appear,\nand comparing other incoming events against the \"tracked\" events according\nto the rules defined here.\n
\n\nEvent trackers are defined in a JSON object like the following:
\n\n\n{\n \"timeToLive\" : [integer],\n \"fields\" : [\n {\n \"type\" : [string],\n \"name\" : [string]\n }\n ]\n}\n\n\n\nWhere:\n
\n\n\nEach field has an evaluator type associated with it. The evaluator defines\nhow the field data is to be compared. The following evaluators are\navailable:\n
\n\n\nThe following is the list of fields that can be used to compare Sentry events:\n
\n\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The path to the TLS/SSL keystore file containing the server certificate and private key used for TLS/SSL. Used when Solr is acting as a TLS/SSL server. The keystore must be in JKS format.", - "display_name": "Solr TLS/SSL Server JKS Keystore File Location", - "name": "solr_https_keystore_file", - "value": null - }, - { - "desc": "ZooKeeper znode used to store information about this Solr service.", - "display_name": "ZooKeeper Znode", - "name": "zookeeper_znode", - "value": "/solr" - }, - { - "desc": "If Solr does not respond on its web URL within this time interval, the Catalina process is killed.", - "display_name": "Solrd Watchdog Timeout", - "name": "solrd_watchdog_timeout", - "value": "30" - }, - { - "desc": "Enable collection of audit events from the service's roles.", - "display_name": "Enable Audit Collection", - "name": "navigator_audit_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds of the overall Solr Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Solr Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Solr Servers falls below the critical threshold.", - "display_name": "Healthy Solr Server Monitoring Thresholds", - "name": "solr_solr_servers_healthy_thresholds", - "value": "{\"critical\":\"90.0\",\"warning\":\"95.0\"}" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/solr-solr_server.json b/sahara/plugins/cdh/v5_4_0/resources/solr-solr_server.json deleted file mode 100644 index 69811911..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/solr-solr_server.json +++ /dev/null @@ -1,302 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Enable caching of HDFS blocks in Solr. There is one block cache per Solr collection. configured to use off-heap memory, Maximum Off-Heap Memory must be set high enough to account for all block caches.", - "display_name": "HDFS Block Cache", - "name": "solr_hdfs_blockcache_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Admin port of the Solr Server.", - "display_name": "Solr Admin Port", - "name": "solr_admin_port", - "value": "8984" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "solr_server_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Solr Server in Bytes", - "name": "solr_java_heapsize", - "value": "1073741824" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Enables the health test that the Solr Server's process state is consistent with the role configuration", - "display_name": "Solr Server Process Health Test", - "name": "solr_server_scm_health_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds on the duration of the Solr Server API request.", - "display_name": "Solr Server API Liveness Request Duration", - "name": "solr_core_status_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "When computing the overall Solr Server health, consider the host's health.", - "display_name": "Solr Server Host Health Test", - "name": "solr_server_host_health_enabled", - "value": "true" - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "Directory where Solr Server will place its log files.", - "display_name": "Solr Server Log Directory", - "name": "solr_log_dir", - "value": "/var/log/solr" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Solr Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "HTTPS port of Solr Server.", - "display_name": "Solr HTTPS port", - "name": "solr_https_port", - "value": "8985" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Solr Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "SOLR_SERVER_role_env_safety_valve", - "value": null - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather status of Solr Cores from the Solr Server with a simple API request.", - "display_name": "Solr Server API Liveness", - "name": "solr_core_status_collection_health_enabled", - "value": "true" - }, - { - "desc": "Use off-heap memory when caching HDFS blocks in Solr.", - "display_name": "HDFS Block Cache Off-Heap Memory", - "name": "solr_hdfs_blockcache_direct_memory_allocation", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "solr_server_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Solr Server logs. Typically used by log4j or logback.", - "display_name": "Solr Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Directory on local file system where Solr Server keeps the configurations for collections.", - "display_name": "Solr Data Directory", - "name": "solr_data_dir", - "value": "/var/lib/solr" - }, - { - "desc": "Maximum amount of off-heap memory in bytes that may be allocated by the Java process. Passed to Java -XX:MaxDirectMemorySize. If unset, defaults to the size of the heap.", - "display_name": "Java Direct Memory Size of Solr Server in Bytes", - "name": "solr_java_direct_memory_size", - "value": "1073741824" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Number of blocks per cache slab. The size of the cache is 8 KB (the block size) times the number of blocks per slab times the number of slabs.", - "display_name": "HDFS Block Cache Blocks per Slab", - "name": "solr_hdfs_blockcache_blocksperbank", - "value": "16384" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "The maximum number of rolled log files to keep for Solr Server logs. Typically used by log4j or logback.", - "display_name": "Solr Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "solr_server_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "Number of slabs per block cache. The size of the cache is 8 KB (the block size) times the number of blocks per slab times the number of slabs.", - "display_name": "HDFS Block Cache Number of Slabs", - "name": "solr_hdfs_blockcache_slab_count", - "value": "1" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "solr_server_gc_duration_window", - "value": "5" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "solr_server_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Solr Server", - "name": "solr_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "The minimum log level for Solr Server logs", - "display_name": "Solr Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "HTTP port of Solr Server.", - "display_name": "Solr HTTP Port", - "name": "solr_http_port", - "value": "8983" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/spark-gateway.json b/sahara/plugins/cdh/v5_4_0/resources/spark-gateway.json deleted file mode 100644 index fa148de9..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/spark-gateway.json +++ /dev/null @@ -1,62 +0,0 @@ -[ - { - "desc": "Name of class implementing org.apache.spark.serializer.Serializer to use in Spark applications.", - "display_name": "Spark Data Serializer", - "name": "spark_data_serializer", - "value": "org.apache.spark.serializer.KryoSerializer" - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for spark-conf/log4j.properties.", - "display_name": "Spark Client Advanced Configuration Snippet (Safety Valve) for spark-conf/log4j.properties", - "name": "spark-conf/log4j.properties_client_config_safety_valve", - "value": null - }, - { - "desc": "Enable Usage of External Shuffle Service.", - "display_name": "Enable Shuffle Service", - "name": "spark_shuffle_service_enabled", - "value": "true" - }, - { - "desc": "The directory where the client configs will be deployed", - "display_name": "Deploy Directory", - "name": "client_config_root_dir", - "value": "/etc/spark" - }, - { - "desc": "Write Spark application history logs to HDFS.", - "display_name": "Enable History", - "name": "spark_history_enabled", - "value": "true" - }, - { - "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", - "display_name": "Alternatives Priority", - "name": "client_config_priority", - "value": "51" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Which deploy mode to use by default. Can be overridden by users when launching applications.", - "display_name": "Default Application Deploy Mode", - "name": "spark_deploy_mode", - "value": "client" - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for spark-conf/spark-defaults.conf.", - "display_name": "Spark Client Advanced Configuration Snippet (Safety Valve) for spark-conf/spark-defaults.conf", - "name": "spark-conf/spark-defaults.conf_client_config_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for spark-conf/spark-env.sh.", - "display_name": "Spark Client Advanced Configuration Snippet (Safety Valve) for spark-conf/spark-env.sh", - "name": "spark-conf/spark-env.sh_client_config_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/spark-service.json b/sahara/plugins/cdh/v5_4_0/resources/spark-service.json deleted file mode 100644 index 2e529da3..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/spark-service.json +++ /dev/null @@ -1,80 +0,0 @@ -[ - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The location of the Spark jar in HDFS. If left blank, CM will use the Spark jar installed on the cluster nodes.", - "display_name": "Spark Jar Location (HDFS)", - "name": "spark_jar_hdfs_path", - "value": "" - }, - { - "desc": "The group that this service's processes should run as.", - "display_name": "System Group", - "name": "process_groupname", - "value": "spark" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Spark Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "SPARK_ON_YARN_service_env_safety_valve", - "value": null - }, - { - "desc": "Enable whether the Spark communication protocols do authentication using a shared secret.", - "display_name": "Spark Authentication", - "name": "spark_authenticate", - "value": "false" - }, - { - "desc": "The location of Spark application history logs in HDFS. Changing this value will not move existing logs to the new location.", - "display_name": "Spark History Location (HDFS)", - "name": "spark_history_log_dir", - "value": "/user/spark/applicationHistory" - }, - { - "desc": "Name of the YARN (MR2 Included) service that this Spark service instance depends on", - "display_name": "YARN (MR2 Included) Service", - "name": "yarn_service", - "value": null - }, - { - "desc": "The port the Spark Shuffle Service listens for fetch requests.", - "display_name": "Spark Shuffle Service Port", - "name": "spark_shuffle_service_port", - "value": "7337" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The user that this service's processes should run as.", - "display_name": "System User", - "name": "process_username", - "value": "spark" - }, - { - "desc": "Kerberos principal used by all roles of this service.", - "display_name": "Kerberos Principal", - "name": "kerberos_princ_name", - "value": "spark" - }, - { - "desc": "The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/spark-spark_yarn_history_server.json b/sahara/plugins/cdh/v5_4_0/resources/spark-spark_yarn_history_server.json deleted file mode 100644 index ad6340f0..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/spark-spark_yarn_history_server.json +++ /dev/null @@ -1,146 +0,0 @@ -[ - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "History Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "SPARK_YARN_HISTORY_SERVER_role_env_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into config/spark-env.sh for this role only.", - "display_name": "History Server Advanced Configuration Snippet (Safety Valve) for config/spark-env.sh", - "name": "config/spark-env.sh_role_safety_valve", - "value": null - }, - { - "desc": "The port of the history server WebUI", - "display_name": "History Server WebUI Port", - "name": "history_server_web_port", - "value": "18088" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The maximum size, in megabytes, per log file for History Server logs. Typically used by log4j or logback.", - "display_name": "History Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "spark_yarn_history_server_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The log directory for log files of the role History Server.", - "display_name": "History Server Log Directory", - "name": "log_dir", - "value": "/var/log/spark" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "History Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "Maximum size for the Java process heap memory. Passed to Java -Xmx. Measured in bytes.", - "display_name": "Java Heap Size of History Server in Bytes", - "name": "history_server_max_heapsize", - "value": "268435456" - }, - { - "desc": "When computing the overall History Server health, consider the host's health.", - "display_name": "History Server Host Health Test", - "name": "spark_yarn_history_server_host_health_enabled", - "value": "true" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum number of rolled log files to keep for History Server logs. Typically used by log4j or logback.", - "display_name": "History Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Enables the health test that the History Server's process state is consistent with the role configuration", - "display_name": "History Server Process Health Test", - "name": "spark_yarn_history_server_scm_health_enabled", - "value": "true" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "The minimum log level for History Server logs", - "display_name": "History Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/sqoop-service.json b/sahara/plugins/cdh/v5_4_0/resources/sqoop-service.json deleted file mode 100644 index e2f827aa..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/sqoop-service.json +++ /dev/null @@ -1,62 +0,0 @@ -[ - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "When computing the overall SQOOP health, consider Sqoop 2 Server's health", - "display_name": "Sqoop 2 Server Role Health Test", - "name": "sqoop_sqoop_server_health_enabled", - "value": "true" - }, - { - "desc": "The group that this service's processes should run as.", - "display_name": "System Group", - "name": "process_groupname", - "value": "sqoop" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "Sqoop 2 Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "sqoop_env_safety_valve", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The user that this service's processes should run as.", - "display_name": "System User", - "name": "process_username", - "value": "sqoop2" - }, - { - "desc": "Kerberos principal used by all roles of this service.", - "display_name": "Kerberos Principal", - "name": "kerberos_princ_name", - "value": "sqoop2" - }, - { - "desc": "The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "MapReduce jobs are run against this service.", - "display_name": "MapReduce Service", - "name": "mapreduce_yarn_service", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/sqoop-sqoop_server.json b/sahara/plugins/cdh/v5_4_0/resources/sqoop-sqoop_server.json deleted file mode 100644 index a13d6c11..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/sqoop-sqoop_server.json +++ /dev/null @@ -1,272 +0,0 @@ -[ - { - "desc": "Directory where Sqoop 2 Server will place its log files.", - "display_name": "Sqoop 2 Server Log Directory", - "name": "sqoop_log_dir", - "value": "/var/log/sqoop2" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Password for Sqoop repository database. Does not apply if you are using Derby as the database type.", - "display_name": "Sqoop Repository Database Password", - "name": "sqoop_repository_database_password", - "value": "" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Admin port of Sqoop 2 Server. (Note: This configuration only applies from CDH 4.3 onwards.)", - "display_name": "Sqoop 2 Admin Port", - "name": "sqoop_admin_port", - "value": "8005" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Maximum number of clients allowed to connect to the Sqoop 2 Server.", - "display_name": "Maximum Client Connections", - "name": "max_client_connections", - "value": "10" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "For advanced use only, a string to be inserted into sqoop.properties for this role only.", - "display_name": "Sqoop 2 Server Advanced Configuration Snippet (Safety Valve) for sqoop.properties", - "name": "sqoop_config_safety_valve", - "value": null - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The maximum number of rolled log files to keep for Sqoop 2 Server logs. Typically used by log4j or logback.", - "display_name": "Sqoop 2 Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "HTTP port of Sqoop 2 Server. (Note: This configuration only applies from CDH 4.3 onwards.)", - "display_name": "Sqoop 2 HTTP Port", - "name": "sqoop_http_port", - "value": "12000" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Sqoop 2 Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "SQOOP_SERVER_role_env_safety_valve", - "value": null - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "sqoop_server_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of Sqoop 2 Server in Bytes", - "name": "sqoop_java_heapsize", - "value": "1073741824" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Sqoop 2 Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Type of Sqoop repository database.", - "display_name": "Sqoop Repository Database Type", - "name": "sqoop_repository_database_type", - "value": "derby" - }, - { - "desc": "User for Sqoop repository database. Does not apply if you are using Derby as the database type.", - "display_name": "Sqoop Repository Database User", - "name": "sqoop_repository_database_user", - "value": "sa" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "Name of Sqoop repository database. Does not apply if you are using Derby as the database type.", - "display_name": "Sqoop Repository Database Name", - "name": "sqoop_repository_database_name", - "value": "sqoop" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for Sqoop 2 Server logs. Typically used by log4j or logback.", - "display_name": "Sqoop 2 Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "Directory where the Sqoop 2 Server places its metastore data. This is used only when Sqoop Repository Database Type is Derby.", - "display_name": "Sqoop 2 Server Metastore Directory", - "name": "sqoop_metastore_data_dir", - "value": "/var/lib/sqoop2" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Sqoop 2 Server", - "name": "sqoop_java_opts", - "value": "" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "When computing the overall Sqoop 2 Server health, consider the host's health.", - "display_name": "Sqoop 2 Server Host Health Test", - "name": "sqoop_server_host_health_enabled", - "value": "true" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Enables the health test that the Sqoop 2 Server's process state is consistent with the role configuration", - "display_name": "Sqoop 2 Server Process Health Test", - "name": "sqoop_server_scm_health_enabled", - "value": "true" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "Host name of Sqoop repository database. If the port is non-default for your database type, use host:port notation. Does not apply if you are using Derby as the database type.", - "display_name": "Sqoop Repository Database Host", - "name": "sqoop_repository_database_host", - "value": "localhost" - }, - { - "desc": "The minimum log level for Sqoop 2 Server logs", - "display_name": "Sqoop 2 Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/yarn-gateway.json b/sahara/plugins/cdh/v5_4_0/resources/yarn-gateway.json deleted file mode 100644 index 2b06b736..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/yarn-gateway.json +++ /dev/null @@ -1,344 +0,0 @@ -[ - { - "desc": "For MapReduce job outputs that are compressed, specify the compression codec to use. Will be part of generated client configuration.", - "display_name": "Compression Codec of MapReduce Job Output", - "name": "mapred_output_compression_codec", - "value": "org.apache.hadoop.io.compress.DefaultCodec" - }, - { - "desc": "The default number of parallel transfers run by reduce during the copy (shuffle) phase. This number should be between sqrt(nodes*number_of_map_slots_per_node) and nodes*number_of_map_slots_per_node/2. Will be part of generated client configuration.", - "display_name": "Default Number of Parallel Transfers During Shuffle", - "name": "mapred_reduce_parallel_copies", - "value": "10" - }, - { - "desc": "Whether map tasks should attempt to use the optimized native implementation of the map-side output collector. This can improve performance of many jobs that are shuffle-intensive. Experimental in CDH 5.2.", - "display_name": "Enable Optimized Map-side Output Collector", - "name": "mapreduce_enable_native_map_output_collector", - "value": "false" - }, - { - "desc": "The directory where the client configs will be deployed", - "display_name": "Deploy Directory", - "name": "client_config_root_dir", - "value": "/etc/hadoop" - }, - { - "desc": "The number of streams to merge at the same time while sorting files. That is, the number of sort heads to use during the merge sort on the reducer side. This determines the number of open file handles. Merging more files in parallel reduces merge sort iterations and improves run time by eliminating disk I/O. Note that merging more files in parallel uses more memory. If 'io.sort.factor' is set too high or the maximum JVM heap is set too low, excessive garbage collection will occur. The Hadoop default is 10, but Cloudera recommends a higher value. Will be part of generated client configuration.", - "display_name": "I/O Sort Factor", - "name": "io_sort_factor", - "value": "64" - }, - { - "desc": "The priority level that the client configuration will have in the Alternatives system on the hosts. Higher priority levels will cause Alternatives to prefer this configuration over any others.", - "display_name": "Alternatives Priority", - "name": "client_config_priority", - "value": "92" - }, - { - "desc": "Location to store the job history files of running jobs. This is a path on the host where the JobTracker is running.", - "display_name": "Running Job History Location", - "name": "hadoop_job_history_dir", - "value": "/var/log/hadoop-mapreduce/history" - }, - { - "desc": "The number of virtual CPU cores allocated for each map task of a job. This parameter has no effect prior to CDH 4.4.", - "display_name": "Map Task CPU Virtual Cores", - "name": "mapreduce_map_cpu_vcores", - "value": "1" - }, - { - "desc": "If enabled, multiple instances of some reduce tasks may be executed in parallel.", - "display_name": "Reduce Tasks Speculative Execution", - "name": "mapred_reduce_tasks_speculative_execution", - "value": "false" - }, - { - "desc": "The application framework to run jobs with. If not set, jobs will be run with the local job runner.", - "display_name": "Application Framework", - "name": "mapreduce_framework_name", - "value": "yarn" - }, - { - "desc": "Base sleep time between failover attempts. Used only if RM HA is enabled.", - "display_name": "Client Failover Sleep Base Time", - "name": "client_failover_sleep_base", - "value": "100" - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for yarn-site.xml.", - "display_name": "YARN Client Advanced Configuration Snippet (Safety Valve) for yarn-site.xml", - "name": "yarn_client_config_safety_valve", - "value": null - }, - { - "desc": "Size of buffer for read and write operations of SequenceFiles.", - "display_name": "SequenceFile I/O Buffer Size", - "name": "io_file_buffer_size", - "value": "65536" - }, - { - "desc": "Fraction of the number of map tasks in the job which should be completed before reduce tasks are scheduled for the job.", - "display_name": "Number of Map Tasks to Complete Before Reduce Tasks", - "name": "mapred_reduce_slowstart_completed_maps", - "value": "0.8" - }, - { - "desc": "For advanced use only, a string to be inserted into the client configuration for mapred-site.xml.", - "display_name": "MapReduce Client Advanced Configuration Snippet (Safety Valve) for mapred-site.xml", - "name": "mapreduce_client_config_safety_valve", - "value": null - }, - { - "desc": "The maximum heap size, in bytes, of the Java MapReduce ApplicationMaster. This number will be formatted and concatenated with 'ApplicationMaster Java Opts Base' to pass to Hadoop.", - "display_name": "ApplicationMaster Java Maximum Heap Size", - "name": "yarn_app_mapreduce_am_max_heap", - "value": "825955249" - }, - { - "desc": "The replication level for submitted job files.", - "display_name": "Mapreduce Submit Replication", - "name": "mapred_submit_replication", - "value": "10" - }, - { - "desc": "The total amount of memory buffer, in megabytes, to use while sorting files. Note that this memory comes out of the user JVM heap size (meaning total user JVM heap - this amount of memory = total user usable heap space. Note that Cloudera's default differs from Hadoop's default; Cloudera uses a bigger buffer by default because modern machines often have more RAM. The smallest value across all TaskTrackers will be part of generated client configuration.", - "display_name": "I/O Sort Memory Buffer (MiB)", - "name": "io_sort_mb", - "value": "256" - }, - { - "desc": "Whether to enable ubertask optimization, which runs \"sufficiently small\" jobs sequentially within a single JVM. \"Small\" is defined by the mapreduce.job.ubertask.maxmaps, mapreduce.job.ubertask.maxreduces, and mapreduce.job.ubertask.maxbytes settings.", - "display_name": "Enable Ubertask Optimization", - "name": "mapreduce_job_ubertask_enabled", - "value": "false" - }, - { - "desc": "Java command line arguments passed to the MapReduce ApplicationMaster.", - "display_name": "ApplicationMaster Java Opts Base", - "name": "yarn_app_mapreduce_am_command_opts", - "value": "-Djava.net.preferIPv4Stack=true" - }, - { - "desc": "The amount of physical memory, in MiB, allocated for each reduce task of a job. This parameter has no effect prior to CDH 4.4.", - "display_name": "Reduce Task Memory", - "name": "mapreduce_reduce_memory_mb", - "value": "1024" - }, - { - "desc": "The maximum permissible size of the split metainfo file. The JobTracker won't attempt to read split metainfo files bigger than the configured value. No limits if set to -1.", - "display_name": "JobTracker MetaInfo Maxsize", - "name": "mapreduce_jobtracker_split_metainfo_maxsize", - "value": "10000000" - }, - { - "desc": "Compression level for the codec used to compress MapReduce outputs. Default compression is a balance between speed and compression ratio.", - "display_name": "Compression Level of Codecs", - "name": "zlib_compress_level", - "value": "DEFAULT_COMPRESSION" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Gateway Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "A shared directory for temporary files.", - "display_name": "Shared Temp Directories", - "name": "mapreduce_cluster_temp_dir", - "value": null - }, - { - "desc": "These are Java command line arguments. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Client Java Configuration Options", - "name": "mapreduce_client_java_opts", - "value": "-Djava.net.preferIPv4Stack=true" - }, - { - "desc": "The physical memory requirement, in MiB, for the ApplicationMaster.", - "display_name": "ApplicationMaster Memory", - "name": "yarn_app_mapreduce_am_resource_mb", - "value": "1024" - }, - { - "desc": "The maximum Java heap size, in bytes, of the reduce processes. This number will be formatted and concatenated with 'Reduce Task Java Opts Base' to pass to Hadoop.", - "display_name": "Reduce Task Maximum Heap Size", - "name": "mapreduce_reduce_java_opts_max_heap", - "value": "825955249" - }, - { - "desc": "Maximum size in bytes for the Java process heap memory. Passed to Java -Xmx.", - "display_name": "Client Java Heap Size in Bytes", - "name": "mapreduce_client_java_heapsize", - "value": "825955249" - }, - { - "desc": "The number of milliseconds before a task will be terminated if it neither reads an input, writes an output, nor updates its status string.", - "display_name": "Mapreduce Task Timeout", - "name": "mapred_task_timeout", - "value": "600000" - }, - { - "desc": "The virtual CPU cores requirement, for the ApplicationMaster. This parameter has no effect prior to CDH 4.4.", - "display_name": "ApplicationMaster Virtual CPU Cores", - "name": "yarn_app_mapreduce_am_resource_cpu_vcores", - "value": "1" - }, - { - "desc": "The amount of physical memory, in MiB, allocated for each map task of a job.", - "display_name": "Map Task Memory", - "name": "mapreduce_map_memory_mb", - "value": "1024" - }, - { - "desc": "Threshold for number of reduces, beyond which a job is considered too big for ubertask optimization. Note: As of CDH 5, MR2 does not support more than one reduce in an ubertask. (Zero is valid.)", - "display_name": "Ubertask Maximum Reduces", - "name": "mapreduce_job_ubertask_maxreduces", - "value": "1" - }, - { - "desc": "For MapReduce map outputs that are compressed, specify the compression codec to use. Will be part of generated client configuration.", - "display_name": "Compression Codec of MapReduce Map Output", - "name": "mapred_map_output_compression_codec", - "value": "org.apache.hadoop.io.compress.SnappyCodec" - }, - { - "desc": "Classpaths to include for MapReduce applications.", - "display_name": "MR Application Classpath", - "name": "mapreduce_application_classpath", - "value": "$HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$MR2_CLASSPATH" - }, - { - "desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.", - "display_name": "Enable Log Event Capture", - "name": "catch_events", - "value": "true" - }, - { - "desc": "Maximum sleep time between failover attempts. Used only if RM HA is enabled.", - "display_name": "Client Failover Sleep Max Time", - "name": "client_failover_sleep_max", - "value": "2000" - }, - { - "desc": "Limit on the number of counters allowed per job.", - "display_name": "Job Counters Limit", - "name": "mapreduce_job_counters_limit", - "value": "120" - }, - { - "desc": "Compress the output of MapReduce jobs. Will be part of generated client configuration.", - "display_name": "Compress MapReduce Job Output", - "name": "mapred_output_compress", - "value": "false" - }, - { - "desc": "For MapReduce job outputs that are compressed as SequenceFiles, you can select one of these compression type options: NONE, RECORD or BLOCK. Cloudera recommends BLOCK. Will be part of generated client configuration.", - "display_name": "Compression Type of MapReduce Job Output", - "name": "mapred_output_compression_type", - "value": "BLOCK" - }, - { - "desc": "Java opts for the map processes. The following symbol, if present, will be interpolated: @taskid@ is replaced by current TaskID. Any other occurrences of '@' will go unchanged. For example, to enable verbose gc logging to a file named for the taskid in /tmp pass a value of: \"-verbose:gc -Xloggc:/tmp/@taskid@.gc\". The configuration variable 'Map Task Memory' can be used to control the maximum memory of the map processes.", - "display_name": "Map Task Java Opts Base", - "name": "mapreduce_map_java_opts", - "value": "-Djava.net.preferIPv4Stack=true" - }, - { - "desc": "Additional execution environment entries for map and reduce task processes.", - "display_name": "MR Application Environment", - "name": "mapreduce_admin_user_env", - "value": "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH" - }, - { - "desc": "The soft limit in either the buffer or record collection buffers. When this limit is reached, a thread will begin to spill the contents to disk in the background. Note that this does not imply any chunking of data to the spill. A value less than 0.5 is not recommended. The syntax is in decimal units; the default is 80% and is formatted 0.8. Will be part of generated client configuration.", - "display_name": "I/O Sort Spill Percent", - "name": "io_sort_spill_percent", - "value": "0.8" - }, - { - "desc": "The default number of reduce tasks per job. Will be part of generated client configuration.", - "display_name": "Default Number of Reduce Tasks per Job", - "name": "mapred_reduce_tasks", - "value": "1" - }, - { - "desc": "Maximum allowed connections for the shuffle. Set to 0 (zero) to indicate no limit on the number of connections.", - "display_name": "Max Shuffle Connections", - "name": "mapreduce_shuffle_max_connections", - "value": "80" - }, - { - "desc": "The maximum Java heap size, in bytes, of the map processes. This number will be formatted and concatenated with 'Map Task Java Opts Base' to pass to Hadoop.", - "display_name": "Map Task Maximum Heap Size", - "name": "mapreduce_map_java_opts_max_heap", - "value": "825955249" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into the client configuration for hadoop-env.sh", - "display_name": "Gateway Client Environment Advanced Configuration Snippet for hadoop-env.sh (Safety Valve)", - "name": "mapreduce_client_env_safety_valve", - "value": null - }, - { - "desc": "The number of virtual CPU cores for each reduce task of a job.", - "display_name": "Reduce Task CPU Virtual Cores", - "name": "mapreduce_reduce_cpu_vcores", - "value": "1" - }, - { - "desc": "Environment variables for the MapReduce ApplicationMaster. These settings can be overridden in the ApplicationMaster User Environment (yarn.app.mapreduce.am.env).", - "display_name": "ApplicationMaster Environment", - "name": "yarn_app_mapreduce_am_admin_user_env", - "value": "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native:$JAVA_LIBRARY_PATH" - }, - { - "desc": "Java opts for the reduce processes. The following symbol, if present, will be interpolated: @taskid@ is replaced by current TaskID. Any other occurrences of '@' will go unchanged. For example, to enable verbose gc logging to a file named for the taskid in /tmp pass a value of: \"-verbose:gc -Xloggc:/tmp/@taskid@.gc\". The configuration variable 'Reduce Task Memory' can be used to control the maximum memory of the reduce processes.", - "display_name": "Reduce Task Java Opts Base", - "name": "mapreduce_reduce_java_opts", - "value": "-Djava.net.preferIPv4Stack=true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "If enabled, uses compression on the map outputs before they are sent across the network. Will be part of generated client configuration.", - "display_name": "Use Compression on Map Outputs", - "name": "mapred_compress_map_output", - "value": "true" - }, - { - "desc": "The maximum number of application attempts for MapReduce jobs. The value of this parameter overrides ApplicationMaster Maximum Attempts for MapReduce jobs.", - "display_name": "Maximum Number of Attempts for MapReduce Jobs", - "name": "mapreduce_am_max_attempts", - "value": "2" - }, - { - "desc": "The minimum log level for Gateway logs", - "display_name": "Gateway Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "Threshold for number of maps, beyond which a job is considered too big for ubertask optimization.", - "display_name": "Ubertask Maximum Maps", - "name": "mapreduce_job_ubertask_maxmaps", - "value": "9" - }, - { - "desc": "If enabled, multiple instances of some map tasks may be executed in parallel.", - "display_name": "Map Tasks Speculative Execution", - "name": "mapred_map_tasks_speculative_execution", - "value": "false" - }, - { - "desc": "Threshold for number of input bytes, beyond which a job is considered too big for ubertask optimization. If no value is specified, dfs.block.size is used as a default.", - "display_name": "Ubertask Maximum Job Size", - "name": "mapreduce_job_ubertask_maxbytes", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/yarn-jobhistory.json b/sahara/plugins/cdh/v5_4_0/resources/yarn-jobhistory.json deleted file mode 100644 index 790b7178..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/yarn-jobhistory.json +++ /dev/null @@ -1,320 +0,0 @@ -[ - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of JobHistory Server in Bytes", - "name": "mr2_jobhistory_java_heapsize", - "value": "1073741824" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for JobHistory Server", - "name": "mr2_jobhistory_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "Kerberos principal used by the JobHistory Server roles.", - "display_name": "Role-Specific Kerberos Principal", - "name": "kerberos_role_princ_name", - "value": "mapred" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "jobhistory_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Enables the health test that the JobHistory Server's process state is consistent with the role configuration", - "display_name": "JobHistory Server Process Health Test", - "name": "jobhistory_scm_health_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "JobHistory Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "JOBHISTORY_role_env_safety_valve", - "value": null - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "When computing the overall JobHistory Server health, consider the host's health.", - "display_name": "JobHistory Server Host Health Test", - "name": "jobhistory_host_health_enabled", - "value": "true" - }, - { - "desc": "The maximum size, in megabytes, per log file for JobHistory Server logs. Typically used by log4j or logback.", - "display_name": "JobHistory Server Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Directory where JobHistory Server will place its log files.", - "display_name": "JobHistory Server Log Directory", - "name": "mr2_jobhistory_log_dir", - "value": "/var/log/hadoop-mapreduce" - }, - { - "desc": "Time interval for history cleaner to check for files to delete. Files are only deleted if they are older than mapreduce.jobhistory.max-age-ms.", - "display_name": "Job History Files Cleaner Interval", - "name": "mapreduce_jobhistory_cleaner_interval", - "value": "86400000" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "jobhistory_gc_duration_window", - "value": "5" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "For advanced use only, a string to be inserted into yarn-site.xml for this role only.", - "display_name": "JobHistory Server Advanced Configuration Snippet (Safety Valve) for yarn-site.xml", - "name": "jobhistory_config_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "JobHistory Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "The port of the MapReduce JobHistory Server administrative interface. Together with the host name of the JobHistory role forms the address.", - "display_name": "MapReduce JobHistory Server Admin Interface Port", - "name": "mapreduce_jobhistory_admin_address", - "value": "10033" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "The group that the JobHistory Server process should run as.", - "display_name": "System Group", - "name": "history_process_groupname", - "value": "hadoop" - }, - { - "desc": "Job history files older than this time duration will deleted when the history cleaner runs.", - "display_name": "Job History Files Maximum Age", - "name": "mapreduce_jobhistory_max_age_ms", - "value": "604800000" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The minimum log level for JobHistory Server logs", - "display_name": "JobHistory Server Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "The HTTP port of the MapReduce JobHistory Server web application. Together with the host name of the JobHistory role forms the address.", - "display_name": "MapReduce JobHistory Web Application HTTP Port", - "name": "mapreduce_jobhistory_webapp_address", - "value": "19888" - }, - { - "desc": "The root HDFS directory of the staging area for users' MR2 jobs; for example /user. The staging directories are always named after the user.", - "display_name": "MapReduce ApplicationMaster Staging Root Directory", - "name": "yarn_app_mapreduce_am_staging_dir", - "value": "/user" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "jobhistory_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "The port of the MapReduce JobHistory Server. Together with the hostname of the JobHistory role, forms the address. ", - "display_name": "MapReduce JobHistory Server Port", - "name": "mapreduce_jobhistory_address", - "value": "10020" - }, - { - "desc": "For advanced use only, a string to be inserted into mapred-site.xml for this role only.", - "display_name": "JobHistory Server Advanced Configuration Snippet (Safety Valve) for mapred-site.xml", - "name": "jobhistory_mapred_safety_valve", - "value": null - }, - { - "desc": "If enabled, the JobHistory Server binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind JobHistory Server to Wildcard Address", - "name": "yarn_jobhistory_bind_wildcard", - "value": "false" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "The maximum number of rolled log files to keep for JobHistory Server logs. Typically used by log4j or logback.", - "display_name": "JobHistory Server Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "The user that the JobHistory Server process should run as.", - "display_name": "System User", - "name": "history_process_username", - "value": "mapred" - }, - { - "desc": "The HTTPS port of the MapReduce JobHistory Server web application. Together with the host name of the JobHistory role forms the address.", - "display_name": "MapReduce JobHistory Web Application HTTPS Port (SSL)", - "name": "mapreduce_jobhistory_webapp_https_address", - "value": "19890" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "jobhistory_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "jobhistory_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/yarn-nodemanager.json b/sahara/plugins/cdh/v5_4_0/resources/yarn-nodemanager.json deleted file mode 100644 index 3dd6fe32..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/yarn-nodemanager.json +++ /dev/null @@ -1,476 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into yarn-site.xml for this role only.", - "display_name": "NodeManager Advanced Configuration Snippet (Safety Valve) for yarn-site.xml", - "name": "nodemanager_config_safety_valve", - "value": null - }, - { - "desc": "The HTTP Port of the NodeManager web application.", - "display_name": "NodeManager Web Application HTTP Port", - "name": "nodemanager_webserver_port", - "value": "8042" - }, - { - "desc": "If enabled, adds 'org.apache.hadoop.mapred.ShuffleHandler' to the NodeManager auxiliary services. This is required for MapReduce applications.", - "display_name": "Enable Shuffle Auxiliary Service", - "name": "mapreduce_aux_service", - "value": "true" - }, - { - "desc": "Heartbeat interval to ResourceManager", - "display_name": "Heartbeat Interval", - "name": "yarn_nodemanager_heartbeat_interval_ms", - "value": "1000" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Enables the health test that the NodeManager's process state is consistent with the role configuration", - "display_name": "NodeManager Process Health Test", - "name": "nodemanager_scm_health_enabled", - "value": "true" - }, - { - "desc": "The amount of time to wait for the NodeManager to fully start up and connect to the ResourceManager before enforcing the connectivity check.", - "display_name": "NodeManager Connectivity Tolerance at Startup", - "name": "nodemanager_connectivity_tolerance_seconds", - "value": "180" - }, - { - "desc": "List of directories on the local filesystem where a NodeManager stores intermediate data files.", - "display_name": "NodeManager Local Directories", - "name": "yarn_nodemanager_local_dirs", - "value": null - }, - { - "desc": "Enables the health check that verifies the NodeManager is connected to the ResourceManager", - "display_name": "NodeManager Connectivity Health Check", - "name": "nodemanager_connectivity_health_enabled", - "value": "true" - }, - { - "desc": "Number of threads to handle localization requests.", - "display_name": "Localizer Client Thread Count", - "name": "yarn_nodemanager_localizer_client_thread_count", - "value": "5" - }, - { - "desc": "The local filesystem directory in which the NodeManager stores state when recovery is enabled. Recovery is enabled by default.", - "display_name": "NodeManager Recovery Directory", - "name": "yarn_nodemanager_recovery_dir", - "value": "/var/lib/hadoop-yarn/yarn-nm-recovery" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "Number of threads to use for localization fetching.", - "display_name": "Localizer Fetch Thread Count", - "name": "yarn_nodemanager_localizer_fetch_thread_count", - "value": "4" - }, - { - "desc": "Target size of localizer cache in MB, per local directory.", - "display_name": "Localizer Cache Target Size", - "name": "yarn_nodemanager_localizer_cache_target_size_mb", - "value": "10240" - }, - { - "desc": "The address of the NodeManager IPC.", - "display_name": "NodeManager IPC Address", - "name": "yarn_nodemanager_address", - "value": "8041" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's The local filesystem directory in which the NodeManager stores state when recovery is enabled. Recovery is enabled by default.. Specified as a percentage of the capacity on that filesystem. This setting is not used if a The local filesystem directory in which the NodeManager stores state when recovery is enabled. Recovery is enabled by default. Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "NodeManager Recovery Directory Free Space Monitoring Percentage Thresholds", - "name": "nodemanager_recovery_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for NodeManager", - "name": "node_manager_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "List of users explicitly whitelisted to be allowed to run containers. Users with IDs lower than the \"Minimum User Id\" setting may be whitelisted by using this setting.", - "display_name": "Allowed System Users", - "name": "container_executor_allowed_system_users", - "value": "nobody,impala,hive,llama" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of NodeManager in Bytes", - "name": "node_manager_java_heapsize", - "value": "1073741824" - }, - { - "desc": "Directory where NodeManager will place its log files.", - "display_name": "NodeManager Log Directory", - "name": "node_manager_log_dir", - "value": "/var/log/hadoop-yarn" - }, - { - "desc": "The system group that owns the container-executor binary. This does not need to be changed unless the ownership of the binary is explicitly changed.", - "display_name": "Container Executor Group", - "name": "container_executor_group", - "value": "yarn" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "NodeManager Environment Advanced Configuration Snippet (Safety Valve)", - "name": "NODEMANAGER_role_env_safety_valve", - "value": null - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "nodemanager_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "nodemanager_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's The local filesystem directory in which the NodeManager stores state when recovery is enabled. Recovery is enabled by default..", - "display_name": "NodeManager Recovery Directory Free Space Monitoring Absolute Thresholds", - "name": "nodemanager_recovery_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Environment variables that should be forwarded from the NodeManager's environment to the container's.", - "display_name": "Containers Environment Variable", - "name": "yarn_nodemanager_admin_env", - "value": "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "nodemanager_gc_duration_window", - "value": "5" - }, - { - "desc": "The minimum Linux user ID allowed. Used to prevent other super users.", - "display_name": "Minimum User ID", - "name": "container_executor_min_user_id", - "value": "1000" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "NodeManager Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Number of seconds after an application finishes before the NodeManager's DeletionService will delete the application's localized file and log directory. To diagnose YARN application problems, set this property's value large enough (for example, to 600 = 10 minutes) to permit examination of these directories.", - "display_name": "Localized Dir Deletion Delay", - "name": "yarn_nodemanager_delete_debug_delay_sec", - "value": "0" - }, - { - "desc": "HDFS directory where application logs are stored when an application completes.", - "display_name": "Remote App Log Directory", - "name": "yarn_nodemanager_remote_app_log_dir", - "value": "/tmp/logs" - }, - { - "desc": "Time in seconds to retain user logs. Only applicable if log aggregation is disabled.", - "display_name": "Log Retain Duration", - "name": "yarn_nodemanager_log_retain_seconds", - "value": "10800" - }, - { - "desc": "Number of threads used in cleanup.", - "display_name": "Cleanup Thread Count", - "name": "yarn_nodemanager_delete_thread_count", - "value": "4" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "Number of threads container manager uses.", - "display_name": "Container Manager Thread Count", - "name": "yarn_nodemanager_container_manager_thread_count", - "value": "20" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Address where the localizer IPC is.", - "display_name": "Localizer Port", - "name": "yarn_nodemanager_localizer_address", - "value": "8040" - }, - { - "desc": "Address where the localizer IPC is.", - "display_name": "Localizer Cache Cleanup Interval", - "name": "yarn_nodemanager_localizer_cache_cleanup_interval_ms", - "value": "600000" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "Absolute path to the script which is periodically run by the node health monitoring service to determine if the node is healthy or not. If the value of this key is empty or the file does not exist in the location configured here, the node health monitoring service is not started.", - "display_name": "Healthchecker Script Path", - "name": "mapred_healthchecker_script_path", - "value": "" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The HTTPS port of the NodeManager web application.", - "display_name": "NodeManager Web Application HTTPS Port (SSL)", - "name": "nodemanager_webserver_https_port", - "value": "8044" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "When computing the overall NodeManager health, consider the host's health.", - "display_name": "NodeManager Host Health Test", - "name": "nodemanager_host_health_enabled", - "value": "true" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's List of directories on the local filesystem where a NodeManager stores intermediate data files..", - "display_name": "NodeManager Local Directories Free Space Monitoring Absolute Thresholds", - "name": "nodemanager_local_data_directories_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "List of directories on the local filesystem where a NodeManager stores container log files.", - "display_name": "NodeManager Container Log Directories", - "name": "yarn_nodemanager_log_dirs", - "value": "/var/log/hadoop-yarn/container" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Path (rooted in the cgroups hierarchy on the machine) where to place YARN-managed cgroups.", - "display_name": "CGroups Hierarchy", - "name": "linux_container_executor_cgroups_hierarchy", - "value": "/hadoop-yarn" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "nodemanager_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "For advanced use only, a string to be inserted into mapred-site.xml for this role only.", - "display_name": "NodeManager Advanced Configuration Snippet (Safety Valve) for mapred-site.xml", - "name": "nodemanager_mapred_safety_valve", - "value": null - }, - { - "desc": "The maximum size, in megabytes, per log file for NodeManager logs. Typically used by log4j or logback.", - "display_name": "NodeManager Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "The remote log dir will be created at {yarn.nodemanager.remote-app-log-dir}/${user}/{thisParam}", - "display_name": "Remote App Log Directory Suffix", - "name": "yarn_nodemanager_remote_app_log_dir_suffix", - "value": "logs" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "The maximum number of rolled log files to keep for NodeManager logs. Typically used by log4j or logback.", - "display_name": "NodeManager Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "Amount of physical memory, in MiB, that can be allocated for containers.", - "display_name": "Container Memory", - "name": "yarn_nodemanager_resource_memory_mb", - "value": "8192" - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "nodemanager_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Number of virtual CPU cores that can be allocated for containers. This parameter has no effect prior to CDH 4.4.", - "display_name": "Container Virtual CPU Cores", - "name": "yarn_nodemanager_resource_cpu_vcores", - "value": "8" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "Environment variables that containers may override rather than use NodeManager's default.", - "display_name": "Containers Environment Variables Whitelist ", - "name": "yarn_nodemanager_env_whitelist", - "value": "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "Comma-separated list of arguments which are to be passed to node health script when it is being launched.", - "display_name": "Healthchecker Script Arguments", - "name": "mapred_healthchecker_script_args", - "value": "" - }, - { - "desc": "List of users banned from running containers.", - "display_name": "Banned System Users", - "name": "container_executor_banned_users", - "value": "hdfs,yarn,mapred,bin" - }, - { - "desc": "The minimum log level for NodeManager logs", - "display_name": "NodeManager Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's List of directories on the local filesystem where a NodeManager stores intermediate data files.. Specified as a percentage of the capacity on that filesystem. This setting is not used if a List of directories on the local filesystem where a NodeManager stores intermediate data files. Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "NodeManager Local Directories Free Space Monitoring Percentage Thresholds", - "name": "nodemanager_local_data_directories_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/yarn-resourcemanager.json b/sahara/plugins/cdh/v5_4_0/resources/yarn-resourcemanager.json deleted file mode 100644 index 893d4241..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/yarn-resourcemanager.json +++ /dev/null @@ -1,506 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds of the number of file descriptors used. Specified as a percentage of file descriptor limit.", - "display_name": "File Descriptor Monitoring Thresholds", - "name": "resourcemanager_fd_thresholds", - "value": "{\"critical\":\"70.0\",\"warning\":\"50.0\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "An XML string that will be inserted verbatim into the Fair Scheduler allocations file. For CDH5, overrides the configuration set using the Pools configuration UI. For CDH4, this is the only way to configure the Fair Scheduler for YARN.", - "display_name": "Fair Scheduler XML Advanced Configuration Snippet (Safety Valve)", - "name": "resourcemanager_fair_scheduler_configuration", - "value": null - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "For advanced use only, a string to be inserted into nodes_allow.txt for this role only.", - "display_name": "ResourceManager Advanced Configuration Snippet (Safety Valve) for nodes_allow.txt", - "name": "rm_hosts_allow_safety_valve", - "value": null - }, - { - "desc": "The maximum number of application attempts. This is a global setting for all ApplicationMasters. Each ApplicationMaster can specify its individual maximum through the API, but if the individual maximum is more than the global maximum, the ResourceManager overrides it.", - "display_name": "ApplicationMaster Maximum Attempts", - "name": "yarn_resourcemanager_am_max_retries", - "value": "2" - }, - { - "desc": "For advanced use only, a string to be inserted into nodes_exclude.txt for this role only.", - "display_name": "ResourceManager Advanced Configuration Snippet (Safety Valve) for nodes_exclude.txt", - "name": "rm_hosts_exclude_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds on the duration of the metrics request to the web server.", - "display_name": "Web Metric Collection Duration", - "name": "resourcemanager_web_metric_collection_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"10000.0\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The maximum number of completed applications that the ResourceManager keeps.", - "display_name": "Max Completed Applications", - "name": "yarn_resourcemanager_max_completed_applications", - "value": "10000" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "For advanced use only, a string to be inserted into yarn-site.xml for this role only.", - "display_name": "ResourceManager Advanced Configuration Snippet (Safety Valve) for yarn-site.xml", - "name": "resourcemanager_config_safety_valve", - "value": null - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for ResourceManager", - "name": "resource_manager_java_opts", - "value": "-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:-CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled" - }, - { - "desc": "Enables the health test that the ResourceManager's process state is consistent with the role configuration", - "display_name": "ResourceManager Process Health Test", - "name": "resourcemanager_scm_health_enabled", - "value": "true" - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "Directory where ResourceManager will place its log files.", - "display_name": "ResourceManager Log Directory", - "name": "resource_manager_log_dir", - "value": "/var/log/hadoop-yarn" - }, - { - "desc": "The smallest amount of physical memory, in MiB, that can be requested for a container. If using the Capacity or FIFO scheduler (or any scheduler, prior to CDH 5), memory requests will be rounded up to the nearest multiple of this number.", - "display_name": "Container Memory Minimum", - "name": "yarn_scheduler_minimum_allocation_mb", - "value": "1024" - }, - { - "desc": "The number of threads used to handle applications manager requests.", - "display_name": "Client Thread Count", - "name": "yarn_resourcemanager_client_thread_count", - "value": "50" - }, - { - "desc": "The periodic interval that the ResourceManager will check whether containers are still alive.", - "display_name": "Container Monitor Interval", - "name": "yarn_resourcemanager_container_liveness_monitor_interval_ms", - "value": "600000" - }, - { - "desc": "The HTTP port of the ResourceManager web application.", - "display_name": "ResourceManager Web Application HTTP Port", - "name": "resourcemanager_webserver_port", - "value": "8088" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "false" - }, - { - "desc": "The expiry interval to wait until a NodeManager is considered dead.", - "display_name": "NodeManager Monitor Expiry", - "name": "yarn_nm_liveness_monitor_expiry_interval_ms", - "value": "600000" - }, - { - "desc": "The largest amount of physical memory, in MiB, that can be requested for a container.", - "display_name": "Container Memory Maximum", - "name": "yarn_scheduler_maximum_allocation_mb", - "value": "65536" - }, - { - "desc": "The largest number of virtual CPU cores that can be requested for a container. This parameter has no effect prior to CDH 4.4.", - "display_name": "Container Virtual CPU Cores Maximum", - "name": "yarn_scheduler_maximum_allocation_vcores", - "value": "32" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "Enables the health test that the Cloudera Manager Agent can successfully contact and gather metrics from the web server.", - "display_name": "Web Metric Collection", - "name": "resourcemanager_web_metric_collection_enabled", - "value": "true" - }, - { - "desc": "Number of threads to handle resource tracker calls.", - "display_name": "Resource Tracker Thread Count", - "name": "yarn_resourcemanager_resource_tracker_client_thread_count", - "value": "50" - }, - { - "desc": "If using the Fair Scheduler, virtual core requests will be rounded up to the nearest multiple of this number. This parameter has no effect prior to CDH 5.", - "display_name": "Container Virtual CPU Cores Increment", - "name": "yarn_scheduler_increment_allocation_vcores", - "value": "1" - }, - { - "desc": "The address of the scheduler interface in the ResourceManager.", - "display_name": "Scheduler Address", - "name": "yarn_resourcemanager_scheduler_address", - "value": "8030" - }, - { - "desc": "For advanced use only, a string to be inserted into mapred-site.xml for this role only.", - "display_name": "ResourceManager Advanced Configuration Snippet (Safety Valve) for mapred-site.xml", - "name": "resourcemanager_mapred_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The HTTPS port of the ResourceManager web application.", - "display_name": "ResourceManager Web Application HTTPS Port (SSL)", - "name": "resourcemanager_webserver_https_port", - "value": "8090" - }, - { - "desc": "The period to review when computing the moving average of garbage collection time.", - "display_name": "Garbage Collection Duration Monitoring Period", - "name": "resourcemanager_gc_duration_window", - "value": "5" - }, - { - "desc": "For applications that request containers on particular racks, the number of scheduling opportunities since the last container assignment to wait before accepting a placement on another rack. Expressed as a float between 0 and 1, which, as a fraction of the cluster size, is the number of scheduling opportunities to pass up. If not set, this means don't pass up any scheduling opportunities. Requires Fair Scheduler continuous scheduling to be disabled. If continuous scheduling is enabled, yarn.scheduler.fair.locality-delay-rack-ms should be used instead.", - "display_name": "Fair Scheduler Rack Locality Threshold", - "name": "resourcemanager_fair_scheduler_locality_threshold_rack", - "value": null - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "Advanced Configuration Snippet (Safety Valve) for Hadoop Metrics2. Properties will be inserted into hadoop-metrics2.properties.", - "display_name": "Hadoop Metrics2 Advanced Configuration Snippet (Safety Valve)", - "name": "hadoop_metrics2_safety_valve", - "value": null - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The address of the applications manager interface in the ResourceManager.", - "display_name": "ResourceManager Address", - "name": "yarn_resourcemanager_address", - "value": "8032" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "ResourceManager Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "Enables multiple Fair Scheduler container assignments in one heartbeat, which improves cluster throughput when there are many small tasks to run.", - "display_name": "Fair Scheduler Assign Multiple Tasks", - "name": "resourcemanager_fair_scheduler_assign_multiple", - "value": "false" - }, - { - "desc": "The health test thresholds for the weighted average time spent in Java garbage collection. Specified as a percentage of elapsed wall clock time.", - "display_name": "Garbage Collection Duration Thresholds", - "name": "resourcemanager_gc_duration_thresholds", - "value": "{\"critical\":\"60.0\",\"warning\":\"30.0\"}" - }, - { - "desc": "When enabled, any applications that were running on the cluster when the ResourceManager died will be recovered when the ResourceManager next starts. Note: If RM-HA is enabled, then this configuration is always enabled.", - "display_name": "Enable ResourceManager Recovery", - "name": "yarn_resourcemanager_recovery_enabled", - "value": "false" - }, - { - "desc": "The periodic interval that the ResourceManager will check whether ApplicationMasters is still alive.", - "display_name": "ApplicationMaster Monitor Interval", - "name": "yarn_resourcemanager_amliveliness_monitor_interval_ms", - "value": "1000" - }, - { - "desc": "For applications that request containers on particular nodes, the minimum time in milliseconds the Fair Scheduler waits before accepting a placement on another node. Requires Fair Scheduler continuous scheduling to be enabled. If continuous scheduling is disabled, yarn.scheduler.fair.locality.threshold.node should be used instead.", - "display_name": "Fair Scheduler Node Locality Delay", - "name": "yarn_scheduler_fair_locality_delay_node_ms", - "value": "2000" - }, - { - "desc": "Number of threads used to handle the ResourceManager admin interface.", - "display_name": "Admin Client Thread Count", - "name": "yarn_resourcemanager_admin_client_thread_count", - "value": "1" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of ResourceManager in Bytes", - "name": "resource_manager_java_heapsize", - "value": "1073741824" - }, - { - "desc": "The maximum size, in megabytes, per log file for ResourceManager logs. Typically used by log4j or logback.", - "display_name": "ResourceManager Max Log Size", - "name": "max_log_size", - "value": "200" - }, - { - "desc": "When set, a SIGKILL signal is sent to the role process when java.lang.OutOfMemoryError is thrown.", - "display_name": "Kill When Out of Memory", - "name": "oom_sigkill_enabled", - "value": "true" - }, - { - "desc": "When enabled, if a pool's minimum share is not met for some period of time, the Fair Scheduler preempts applications in other pools. Preemption guarantees that production applications are not starved while also allowing the cluster to be used for experimental and research applications. To minimize wasted computation, the Fair Scheduler preempts the most recently launched applications.", - "display_name": "Fair Scheduler Preemption", - "name": "resourcemanager_fair_scheduler_preemption", - "value": "false" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The smallest number of virtual CPU cores that can be requested for a container. If using the Capacity or FIFO scheduler (or any scheduler, prior to CDH 5), virtual core requests will be rounded up to the nearest multiple of this number. This parameter has no effect prior to CDH 4.4.", - "display_name": "Container Virtual CPU Cores Minimum", - "name": "yarn_scheduler_minimum_allocation_vcores", - "value": "1" - }, - { - "desc": "The health test thresholds for unexpected exits encountered within a recent period specified by the unexpected_exits_window configuration for the role.", - "display_name": "Unexpected Exits Thresholds", - "name": "unexpected_exits_thresholds", - "value": "{\"critical\":\"any\",\"warning\":\"never\"}" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "ResourceManager Environment Advanced Configuration Snippet (Safety Valve)", - "name": "RESOURCEMANAGER_role_env_safety_valve", - "value": null - }, - { - "desc": "If using the Fair Scheduler, memory requests will be rounded up to the nearest multiple of this number. This parameter has no effect prior to CDH 5.", - "display_name": "Container Memory Increment", - "name": "yarn_scheduler_increment_allocation_mb", - "value": "512" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "If enabled, the ResourceManager binds to the wildcard address (\"0.0.0.0\") on all of its ports.", - "display_name": "Bind ResourceManager to Wildcard Address", - "name": "yarn_rm_bind_wildcard", - "value": "false" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "For applications that request containers on particular racks, the minimum time in milliseconds the Fair Scheduler waits before accepting a placement on another rack. Requires Fair Scheduler continuous scheduling to be enabled. If continuous scheduling is disabled, yarn.scheduler.fair.locality.threshold.rack should be used instead.", - "display_name": "Fair Scheduler Rack Locality Delay", - "name": "yarn_scheduler_fair_locality_delay_rack_ms", - "value": "4000" - }, - { - "desc": "Enable continuous scheduling in the Fair Scheduler. When enabled, scheduling decisions are decoupled from NodeManager heartbeats, leading to faster resource allocations.", - "display_name": "Enable Fair Scheduler Continuous Scheduling", - "name": "yarn_scheduler_fair_continuous_scheduling_enabled", - "value": "true" - }, - { - "desc": "The directory in which stacks logs are placed. If not set, stacks are logged into a stacks subdirectory of the role's log directory.", - "display_name": "Stacks Collection Directory", - "name": "stacks_collection_directory", - "value": null - }, - { - "desc": "When computing the overall ResourceManager health, consider the host's health.", - "display_name": "ResourceManager Host Health Test", - "name": "resourcemanager_host_health_enabled", - "value": "true" - }, - { - "desc": "The number of threads used to handle requests through the scheduler interface.", - "display_name": "Scheduler Thread Count", - "name": "yarn_resourcemanager_scheduler_client_thread_count", - "value": "50" - }, - { - "desc": "The maximum number of rolled log files to keep for ResourceManager logs. Typically used by log4j or logback.", - "display_name": "ResourceManager Maximum Log File Backups", - "name": "max_log_backup_index", - "value": "10" - }, - { - "desc": "When enabled, the Fair Scheduler will assign shares to individual apps based on their size, rather than providing an equal share to all apps regardless of size.", - "display_name": "Fair Scheduler Size-Based Weight", - "name": "resourcemanager_fair_scheduler_size_based_weight", - "value": "false" - }, - { - "desc": "For applications that request containers on particular nodes, the number of scheduling opportunities since the last container assignment to wait before accepting a placement on another node. Expressed as a float between 0 and 1, which, as a fraction of the cluster size, is the number of scheduling opportunities to pass up. If not set, this means don't pass up any scheduling opportunities. Requires Fair Scheduler continuous scheduling to be disabled. If continuous scheduling is enabled, yarn.scheduler.fair.locality-delay-node-ms should be used instead.", - "display_name": "Fair Scheduler Node Locality Threshold", - "name": "resourcemanager_fair_scheduler_locality_threshold_node", - "value": null - }, - { - "desc": "Number of CPU shares to assign to this role. The greater the number of shares, the larger the share of the host's CPUs that will be given to this role when the host experiences CPU contention. Must be between 2 and 262144. Defaults to 1024 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup CPU Shares", - "name": "rm_cpu_shares", - "value": "1024" - }, - { - "desc": "Path to directory where heap dumps are generated when java.lang.OutOfMemoryError error is thrown. This directory is automatically created if it does not exist. If this directory already exists, role user must have write access to this directory. If this directory is shared among multiple roles, it should have 1777 permissions. The heap dump files are created with 600 permissions and are owned by the role user. The amount of free space in this directory should be greater than the maximum Java Process heap size configured for this role.", - "display_name": "Heap Dump Directory", - "name": "oom_heap_dump_dir", - "value": "/tmp" - }, - { - "desc": "When enabled, ResourceManager has proxy user privileges.", - "display_name": "Enable ResourceManger Proxy User Privileges", - "name": "yarn_resourcemanager_proxy_user_privileges_enabled", - "value": "true" - }, - { - "desc": "The frequency with which stacks are collected.", - "display_name": "Stacks Collection Frequency", - "name": "stacks_collection_frequency", - "value": "5.0" - }, - { - "desc": "When set to true, the Fair Scheduler uses the username as the default pool name, in the event that a pool name is not specified. When set to false, all applications are run in a shared pool, called default.", - "display_name": "Fair Scheduler User As Default Queue", - "name": "resourcemanager_fair_scheduler_user_as_default_queue", - "value": "true" - }, - { - "desc": "The address of the resource tracker interface in the ResourceManager.", - "display_name": "Resource Tracker Address", - "name": "yarn_resourcemanager_resource_tracker_address", - "value": "8031" - }, - { - "desc": "The expiry interval to wait until an ApplicationMaster is considered dead.", - "display_name": "ApplicationMaster Monitor Expiry", - "name": "yarn_am_liveness_monitor_expiry_interval_ms", - "value": "600000" - }, - { - "desc": "The class to use as the resource scheduler. FairScheduler is only supported in CDH 4.2.1 and later.", - "display_name": "Scheduler Class", - "name": "yarn_resourcemanager_scheduler_class", - "value": "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler" - }, - { - "desc": "The minimum log level for ResourceManager logs", - "display_name": "ResourceManager Logging Threshold", - "name": "log_threshold", - "value": "INFO" - }, - { - "desc": "The amount of time allowed after this role is started that failures of health checks that rely on communication with this role will be tolerated.", - "display_name": "Health Check Startup Tolerance", - "name": "resourcemanager_startup_tolerance_minutes", - "value": "5" - }, - { - "desc": "The address of the admin interface in the ResourceManager.", - "display_name": "Administration Address", - "name": "yarn_resourcemanager_admin_address", - "value": "8033" - }, - { - "desc": "The periodic interval that the ResourceManager will check whether NodeManagers are still alive.", - "display_name": "NodeManager Monitor Interval", - "name": "yarn_resourcemanager_nm_liveness_monitor_interval_ms", - "value": "1000" - }, - { - "desc": "Enter an XML string that represents the Capacity Scheduler configuration.", - "display_name": "Capacity Scheduler Configuration", - "name": "resourcemanager_capacity_scheduler_configuration", - "value": "\nThe configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "Controls which applications an admin user can see in the applications list view", - "display_name": "Admin Users Applications List Visibility Settings", - "name": "admin_application_list_settings", - "value": "ALL" - }, - { - "desc": "Whether YARN uses the Linux Container Executor both in secure (Kerberos) and insecure (not Kerberos) environments. Cgroups enforcement only works when the Linux Container Executor is used.", - "display_name": "Always Use Linux Container Executor", - "name": "yarn_service_lce_always", - "value": "false" - }, - { - "desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.", - "display_name": "Enable Log Event Capture", - "name": "catch_events", - "value": "true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when this entity's configuration changes.", - "display_name": "Enable Configuration Change Alerts", - "name": "enable_config_alerts", - "value": "false" - }, - { - "desc": "Comma-delimited list of groups that you want to allow the Llama (AM for Impala) user to impersonate. The default '*' allows all groups. To disable entirely, use a string that doesn't correspond to a group name, such as '_no_group_'.", - "display_name": "Llama Proxy User Groups", - "name": "llama_proxy_user_groups_list", - "value": "*" - }, - { - "desc": "Path to the keystore file containing the server certificate and private key used for encrypted shuffle and encrypted web UIs. Applies to configurations of all daemon roles of this service.", - "display_name": "Hadoop SSL Server Keystore File Location", - "name": "ssl_server_keystore_location", - "value": null - }, - { - "desc": "Controls the aggregate metrics generated for YARN applications. The structure is a JSON list of the attributes to aggregate and the entities to aggregate to. For example, if the attributeName is 'maps_completed' and the aggregationTargets is ['USER'] then the Service Monitor will create the metric 'yarn_application_maps_completed_rate' and, every ten minutes, will record the total maps completed for each user across all their YARN applications. By default it will also record the number of applications submitted ('apps_submitted_rate') for both users and pool. For a full list of the supported attributes see the YARN search page. Note that the valid aggregation targets are USER, YARN_POOL, and YARN (the service), and that these aggregate metrics can be viewed on both the reports and charts search pages.", - "display_name": "YARN Application Aggregates", - "name": "yarn_application_aggregates", - "value": "[\n {\n \"attributeName\": \"maps_total\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"reduces_total\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"cpu_milliseconds\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"mb_millis_maps\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"mb_millis_reduces\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"vcores_millis_maps\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"vcores_millis_reduces\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"file_bytes_read\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"file_bytes_written\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"hdfs_bytes_read\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"hdfs_bytes_written\",\n \"aggregationTargets\": [\"USER\", \"YARN_POOL\", \"YARN\"]\n },\n {\n \"attributeName\": \"cm_cpu_milliseconds\",\n \"aggregationTargets\": [\"USER\"]\n }\n]\n" - }, - { - "desc": "Comma-delimited list of hosts where you want to allow the Llama (AM for Impala) user to impersonate other users. The default '*' allows all hosts. To disable entirely, use a string that doesn't correspond to a host name, such as '_no_host'.", - "display_name": "Llama Proxy User Hosts", - "name": "llama_proxy_user_hosts_list", - "value": "*" - }, - { - "desc": "UNIX user that containers run as when Linux-container-executor is used in nonsecure mode.", - "display_name": "UNIX User for Nonsecure Mode with Linux Container Executor", - "name": "yarn_nodemanager_linux_container_executor_nonsecure_mode_local_user", - "value": "nobody" - }, - { - "desc": "For advanced use only, a string to be inserted into mapred-site.xml. Applies to configurations of all roles in this service except client configuration.", - "display_name": "YARN Service MapReduce Advanced Configuration Snippet (Safety Valve)", - "name": "yarn_service_mapred_safety_valve", - "value": null - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of all roles in this service except client configuration.", - "display_name": "YARN (MR2 Included) Service Environment Advanced Configuration Snippet (Safety Valve)", - "name": "yarn_service_env_safety_valve", - "value": null - }, - { - "desc": "Whether to enable log aggregation", - "display_name": "Enable Log Aggregation", - "name": "yarn_log_aggregation_enable", - "value": "true" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/resources/zookeeper-server.json b/sahara/plugins/cdh/v5_4_0/resources/zookeeper-server.json deleted file mode 100644 index e20e80a3..00000000 --- a/sahara/plugins/cdh/v5_4_0/resources/zookeeper-server.json +++ /dev/null @@ -1,410 +0,0 @@ -[ - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Log Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Log Directory Free Space Monitoring Percentage Thresholds", - "name": "log_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "This file contains the rules which govern how log messages are turned into events by the custom log4j appender that this role loads. It is in JSON format, and is composed of a list of rules. Every log message is evaluated against each of these rules in turn to decide whether or not to send an event for that message.
Each rule has some or all of the following fields:
Example:{\"alert\": false, \"rate\": 10, \"exceptiontype\": \"java.lang.StringIndexOutOfBoundsException\"}
This rule will send events to Cloudera Manager for every StringIndexOutOfBoundsException, up to a maximum of 10 every minute.
", - "display_name": "Rules to Extract Events from Log Files", - "name": "log_event_whitelist", - "value": "{\n \"version\": \"0\",\n \"rules\": [\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"FATAL\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Instead, use .*\"},\n {\"alert\": false, \"rate\": 0, \"threshold\":\"WARN\", \"content\": \".* is deprecated. Use .* instead\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 2, \"exceptiontype\": \".*\"},\n {\"alert\": false, \"rate\": 1, \"periodminutes\": 1, \"threshold\":\"WARN\"}\n ]\n}\n" - }, - { - "desc": "The port to monitor for inter-server communication", - "display_name": "Quorum Port", - "name": "quorumPort", - "value": "3181" - }, - { - "desc": "Whether or not periodic stacks collection is enabled.", - "display_name": "Stacks Collection Enabled", - "name": "stacks_collection_enabled", - "value": "false" - }, - { - "desc": "The port to monitor for leadership election", - "display_name": "Election Port", - "name": "electionPort", - "value": "4181" - }, - { - "desc": "Specifies the name of the user that has read-only privileges when using password file based authentication for JMX access. JMX authentication must be enabled for this setting to take effect.", - "display_name": "Name of User with Read-Only access to the JMX Agent", - "name": "jmx_passwd_file_readonly_user", - "value": "monitorRole" - }, - { - "desc": "The health check thresholds of the weighted average size of the ZooKeeper Server connection count over a recent period. See ZooKeeper Server Connection Count Monitoring Period.", - "display_name": "ZooKeeper Server Connection Count Thresholds", - "name": "zookeeper_server_connection_count_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory. Specified as a percentage of the capacity on that filesystem. This setting is not used if a Heap Dump Directory Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Heap Dump Directory Free Space Monitoring Percentage Thresholds", - "name": "heap_dump_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The disk location that ZooKeeper will use to store its transaction logs.", - "display_name": "Transaction Log Directory", - "name": "dataLogDir", - "value": "/var/lib/zookeeper" - }, - { - "desc": "The port used by the ZooKeeper Server's RMI registry. This is required to enable JMX access through RMI which is required for Cloudera Manager ZooKeeper monitoring. This is added as \"-Dcom.sun.management.jmxremote.port\" to the ZooKeeper Server's JVM command line.", - "display_name": "JMX Remote Port", - "name": "server_jmx_agent_port", - "value": "9010" - }, - { - "desc": "For advanced use only, key-value pairs (one on each line) to be inserted into a role's environment. Applies to configurations of this role except client configuration.", - "display_name": "Server Environment Advanced Configuration Snippet (Safety Valve)", - "name": "SERVER_role_env_safety_valve", - "value": null - }, - { - "desc": "The tolerance window that will be used in the detection of a ZooKeeper server's membership in a quorum. Specified in minutes.", - "display_name": "Quorum Membership Detection Window", - "name": "zookeeper_server_quorum_membership_detection_window", - "value": "3" - }, - { - "desc": "Enables the health test that the Server's process state is consistent with the role configuration", - "display_name": "Server Process Health Test", - "name": "zookeeper_server_scm_health_enabled", - "value": "true" - }, - { - "desc": "The address (IPv4, IPv6, or hostname) to monitor for client connections. This is the address that clients attempt to connect to. This setting is optional, because by default, ZooKeeper binds in such a way that any connection to the client port for any address/interface/NIC on the server will be accepted.", - "display_name": "Client Port Address", - "name": "clientPortAddress", - "value": null - }, - { - "desc": "When computing the overall Server health, consider the host's health.", - "display_name": "Server Host Health Test", - "name": "zookeeper_server_host_health_enabled", - "value": "true" - }, - { - "desc": "The method used to collect stacks. The jstack option involves periodically running the jstack command against the role's daemon process. The servlet method is available for those roles that have an HTTP server endpoint exposing the current stacks traces of all threads. When the servlet method is selected, that HTTP endpoint is periodically scraped.", - "display_name": "Stacks Collection Method", - "name": "stacks_collection_method", - "value": "jstack" - }, - { - "desc": "Specifies the password of the user that has read-only privileges when using password file based authentication for JMX access. JMX authentication must be enabled for this setting to take effect.", - "display_name": "Password of User with Read-Only Access to the JMX agent", - "name": "jmx_passwd_file_readonly_user_password", - "value": "MONITOR" - }, - { - "desc": "The health check thresholds of the weighted average size of the ZooKeeper Server outstanding requests queue over a recent period. See ZooKeeper Server Outstanding Requests Monitoring Period.", - "display_name": "ZooKeeper Server Outstanding Requests Thresholds", - "name": "zookeeper_server_outstanding_requests_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "The period to review when computing the moving average of the outstanding requests queue size. Specified in minutes.", - "display_name": "ZooKeeper Server Outstanding Requests Monitoring Period", - "name": "zookeeper_server_outstanding_requests_window", - "value": "3" - }, - { - "desc": "Maximum size in bytes for the Java Process heap memory. Passed to Java -Xmx.", - "display_name": "Java Heap Size of ZooKeeper Server in Bytes", - "name": "zookeeper_server_java_heapsize", - "value": "1073741824" - }, - { - "desc": "When set, this role's process is automatically (and transparently) restarted in the event of an unexpected failure.", - "display_name": "Automatically Restart Process", - "name": "process_auto_restart", - "value": "true" - }, - { - "desc": "The configured triggers for this role. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific role. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the following JSON formatted trigger configured for a DataNode fires if the DataNode has more than 1500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleName=$ROLENAME and last(fd_open) > 1500) DO health:bad\",\n \"streamThreshold\": 0, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Role Triggers", - "name": "role_triggers", - "value": "[]" - }, - { - "desc": "The minimum session timeout, in milliseconds, that the ZooKeeper Server will allow the client to negotiate", - "display_name": "Minimum Session Timeout", - "name": "minSessionTimeout", - "value": "4000" - }, - { - "desc": "Specifies the name of the user that has read-write privileges when using password file based authentication for JMX access. JMX authentication must be enabled for this setting to take effect.", - "display_name": "Name of User with Read-Write Access to the JMX Agent", - "name": "jmx_passwd_file_readwrite_user", - "value": "controlRole" - }, - { - "desc": "Soft memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process if and only if the host is facing memory pressure. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Soft Limit", - "name": "rm_memory_soft_limit", - "value": "-1" - }, - { - "desc": "When set, generates heap dump file when java.lang.OutOfMemoryError is thrown.", - "display_name": "Dump Heap When Out of Memory", - "name": "oom_heap_dump_enabled", - "value": "false" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's The disk location that ZooKeeper will use to store its database snapshots.. Specified as a percentage of the capacity on that filesystem. This setting is not used if a The disk location that ZooKeeper will use to store its database snapshots. Free Space Monitoring Absolute Thresholds setting is configured.", - "display_name": "Data Directory Free Space Monitoring Percentage Thresholds", - "name": "zookeeper_server_data_directory_free_space_percentage_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"never\"}" - }, - { - "desc": "Hard memory limit to assign to this role, enforced by the Linux kernel. When the limit is reached, the kernel will reclaim pages charged to the process. If reclaiming fails, the kernel may kill the process. Both anonymous as well as page cache pages contribute to the limit. Use a value of -1 B to specify no limit. By default processes not managed by Cloudera Manager will have no limit.", - "display_name": "Cgroup Memory Hard Limit", - "name": "rm_memory_hard_limit", - "value": "-1" - }, - { - "desc": "The disk location that ZooKeeper will use to store its database snapshots.", - "display_name": "Data Directory", - "name": "dataDir", - "value": "/var/lib/zookeeper" - }, - { - "desc": "The health test thresholds on the swap memory usage of the process.", - "display_name": "Process Swap Memory Thresholds", - "name": "process_swap_memory_thresholds", - "value": "{\"critical\":\"never\",\"warning\":\"any\"}" - }, - { - "desc": "The period to review when computing unexpected exits.", - "display_name": "Unexpected Exits Monitoring Period", - "name": "unexpected_exits_window", - "value": "5" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's The disk location that ZooKeeper will use to store its database snapshots..", - "display_name": "Data Directory Free Space Monitoring Absolute Thresholds", - "name": "zookeeper_server_data_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Weight for the read I/O requests issued by this role. The greater the weight, the higher the priority of the requests when the host experiences I/O contention. Must be between 100 and 1000. Defaults to 1000 for processes not managed by Cloudera Manager.", - "display_name": "Cgroup I/O Weight", - "name": "rm_io_weight", - "value": "500" - }, - { - "desc": "The amount of stacks data that is retained. After the retention limit is reached, the oldest data is deleted.", - "display_name": "Stacks Collection Data Retention", - "name": "stacks_collection_data_retention", - "value": "104857600" - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's heap dump directory.", - "display_name": "Heap Dump Directory Free Space Monitoring Absolute Thresholds", - "name": "heap_dump_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "Enables authentication when interacting with the JMX agent on the ZooKeeper server.", - "display_name": "Enable Authenticated Communication with the JMX Agent", - "name": "enable_jmx_authentication", - "value": "false" - }, - { - "desc": "The period to review when computing the moving average of the connection count. Specified in minutes.", - "display_name": "ZooKeeper Server Connection Count Monitoring Period", - "name": "zookeeper_server_connection_count_window", - "value": "3" - }, - { - "desc": "For advanced use only, a string to be inserted into log4j.properties for this role only.", - "display_name": "Server Logging Advanced Configuration Snippet (Safety Valve)", - "name": "log4j_safety_valve", - "value": null - }, - { - "desc": "The maximum session timeout, in milliseconds, that the ZooKeeper Server will allow the client to negotiate", - "display_name": "Maximum Session Timeout", - "name": "maxSessionTimeout", - "value": "40000" - }, - { - "desc": "For advanced use only, a string to be inserted into zoo.cfg for this role only.", - "display_name": "Server Advanced Configuration Snippet (Safety Valve) for zoo.cfg", - "name": "zookeeper_config_safety_valve", - "value": null - }, - { - "desc": "The health test thresholds for monitoring of free space on the filesystem that contains this role's log directory.", - "display_name": "Log Directory Free Space Monitoring Absolute Thresholds", - "name": "log_directory_free_space_absolute_thresholds", - "value": "{\"critical\":\"5.36870912E9\",\"warning\":\"1.073741824E10\"}" - }, - { - "desc": "The maximum number of concurrent connections (at the socket level) that a single client, identified by the IP address, may make to a single member of the ZooKeeper ensemble. This setting is used to prevent certain classes of DoS attacks, including file descriptor exhaustion. To remove the limit on concurrent connections, set this value to 0.", - "display_name": "Maximum Client Connections", - "name": "maxClientCnxns", - "value": "60" - }, - { - "desc": "If configured, overrides the process soft and hard rlimits (also called ulimits) for file descriptors to the configured value.", - "display_name": "Maximum Process File Descriptors", - "name": "rlimit_fds", - "value": null - }, - { - "desc": "These arguments will be passed as part of the Java command line. Commonly, garbage collection flags or extra debugging flags would be passed here.", - "display_name": "Java Configuration Options for Zookeeper Server", - "name": "zk_server_java_opts", - "value": "" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this role reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Health Alerts for this Role", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The port used by the ZooKeeper Server's RMI server to handle JMX RMI requests. This is added as \"-Dcom.sun.management.jmxremote.rmi.port=The configured triggers for this service. This is a JSON formatted list of triggers. These triggers are evaluated as part as the health system. Every trigger expression is parsed, and if the trigger condition is met, the list of actions provided in the trigger expression is executed.
Each trigger has all of the following fields:
triggerName (mandatory) - The name of the trigger. This value must be unique for the specific service. triggerExpression (mandatory) - A tsquery expression representing the trigger. streamThreshold (optional) - The maximum number of streams that can satisfy a condition of a trigger before the condition fires. By default set to 0, and any stream returned causes the condition to fire. enabled (optional) - By default set to 'true'. If set to 'false', the trigger will not be evaluated.expressionEditorConfig (optional) - Metadata for the trigger editor. If present, the trigger should only be edited from the Edit Trigger page; editing the trigger here may lead to inconsistencies.For example, the followig JSON formatted trigger fires if there are more than 10 DataNodes with more than 500 file-descriptors opened:
[{\"triggerName\": \"sample-trigger\",\n \"triggerExpression\": \"IF (SELECT fd_open WHERE roleType = DataNode and last(fd_open) > 500) DO health:bad\",\n \"streamThreshold\": 10, \"enabled\": \"true\"}]See the trigger rules documentation for more details on how to write triggers using tsquery.
The JSON format is evolving and may change in the future and, as a result, backward compatibility is not guaranteed between releases at this time.
", - "display_name": "Service Triggers", - "name": "service_triggers", - "value": "[]" - }, - { - "desc": "The length of time, in milliseconds, of a single tick, which is the basic time unit used by ZooKeeper. A tick is used to regulate heartbeats and timeouts.", - "display_name": "Tick Time", - "name": "tickTime", - "value": "2000" - }, - { - "desc": "When set, each role identifies important log events and forwards them to Cloudera Manager.", - "display_name": "Enable Log Event Capture", - "name": "catch_events", - "value": "true" - }, - { - "desc": "When set, Cloudera Manager will send alerts when the health of this service reaches the threshold specified by the EventServer setting eventserver_health_events_alert_threshold", - "display_name": "Enable Service Level Health Alerts", - "name": "enable_alerts", - "value": "true" - }, - { - "desc": "The health test thresholds of the overall Server health. The check returns \"Concerning\" health if the percentage of \"Healthy\" Servers falls below the warning threshold. The check is unhealthy if the total percentage of \"Healthy\" and \"Concerning\" Servers falls below the critical threshold.", - "display_name": "Healthy Server Monitoring Thresholds", - "name": "zookeeper_servers_healthy_thresholds", - "value": "{\"critical\":\"51.0\",\"warning\":\"99.0\"}" - }, - { - "desc": "For advanced use only, a list of derived configuration properties that will be used by the Service Monitor instead of the default ones.", - "display_name": "Service Monitor Derived Configs Advanced Configuration Snippet (Safety Valve)", - "name": "smon_derived_configs_safety_valve", - "value": null - }, - { - "desc": "Whether the leader accepts client connections.", - "display_name": "Leader Serves", - "name": "leaderServes", - "value": "yes" - }, - { - "desc": "Automatically create data directories at startup, if they do not exist. Enabling this configuration should be used with care as it will suppress any errors in setup of data directories.", - "display_name": "Enable auto-creation of data directories", - "name": "zookeeper_datadir_autocreate", - "value": "false" - } -] \ No newline at end of file diff --git a/sahara/plugins/cdh/v5_4_0/validation.py b/sahara/plugins/cdh/v5_4_0/validation.py deleted file mode 100644 index 92f3e9e1..00000000 --- a/sahara/plugins/cdh/v5_4_0/validation.py +++ /dev/null @@ -1,232 +0,0 @@ -# Copyright (c) 2015 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.i18n import _ -from sahara.plugins.cdh.v5_4_0 import plugin_utils as pu -from sahara.plugins.cdh import validation -from sahara.plugins import exceptions as ex -from sahara.plugins import utils as u - - -class ValidatorV540(validation.Validator): - PU = pu.PluginUtilsV540() - - @classmethod - def validate_cluster_creating(cls, cluster): - super(ValidatorV540, cls).validate_cluster_creating(cluster) - cls._hdfs_ha_validation(cluster) - cls._yarn_ha_validation(cluster) - cls._flume_validation(cluster) - cls._sentry_validation(cluster) - cls._solr_validation(cluster) - cls._sqoop_validation(cluster) - cls._hbase_indexer_validation(cluster) - cls._impala_validation(cluster) - cls._kms_validation(cluster) - - @classmethod - def _hdfs_ha_validation(cls, cluster): - jn_count = cls._get_inst_count(cluster, 'HDFS_JOURNALNODE') - zk_count = cls._get_inst_count(cluster, 'ZOOKEEPER_SERVER') - - require_anti_affinity = cls.PU.c_helper.get_required_anti_affinity( - cluster) - - if jn_count > 0: - if jn_count < 3: - raise ex.InvalidComponentCountException('HDFS_JOURNALNODE', - _('not less than 3'), - jn_count) - if not jn_count % 2: - raise ex.InvalidComponentCountException('HDFS_JOURNALNODE', - _('be odd'), jn_count) - if zk_count < 1: - raise ex.RequiredServiceMissingException('ZOOKEEPER', - required_by='HDFS HA') - if require_anti_affinity: - if 'HDFS_SECONDARYNAMENODE' not in\ - cls._get_anti_affinity(cluster): - raise ex.NameNodeHAConfigurationError( - _('HDFS_SECONDARYNAMENODE should be enabled ' - 'in anti_affinity.')) - if 'HDFS_NAMENODE' not in cls._get_anti_affinity(cluster): - raise ex.NameNodeHAConfigurationError( - _('HDFS_NAMENODE should be enabled in anti_affinity.')) - - @classmethod - def _yarn_ha_validation(cls, cluster): - rm_count = cls._get_inst_count(cluster, 'YARN_RESOURCEMANAGER') - zk_count = cls._get_inst_count(cluster, 'ZOOKEEPER_SERVER') - stdb_rm_count = cls._get_inst_count(cluster, 'YARN_STANDBYRM') - - require_anti_affinity = cls.PU.c_helper.get_required_anti_affinity( - cluster) - - if stdb_rm_count > 1: - raise ex.InvalidComponentCountException( - 'YARN_STANDBYRM', _('0 or 1'), stdb_rm_count) - if stdb_rm_count > 0: - if rm_count < 1: - raise ex.RequiredServiceMissingException( - 'YARN_RESOURCEMANAGER', required_by='RM HA') - if zk_count < 1: - raise ex.RequiredServiceMissingException( - 'ZOOKEEPER', required_by='RM HA') - if require_anti_affinity: - if 'YARN_RESOURCEMANAGER' not in\ - cls._get_anti_affinity(cluster): - raise ex.ResourceManagerHAConfigurationError( - _('YARN_RESOURCEMANAGER should be enabled in ' - 'anti_affinity.')) - if 'YARN_STANDBYRM' not in cls._get_anti_affinity(cluster): - raise ex.ResourceManagerHAConfigurationError( - _('YARN_STANDBYRM should be' - ' enabled in anti_affinity.')) - - @classmethod - def _flume_validation(cls, cluster): - a_count = cls._get_inst_count(cluster, 'FLUME_AGENT') - dn_count = cls._get_inst_count(cluster, 'HDFS_DATANODE') - - if a_count >= 1: - if dn_count < 1: - raise ex.RequiredServiceMissingException( - 'HDFS_DATANODE', required_by='FLUME_AGENT') - - @classmethod - def _sentry_validation(cls, cluster): - - snt_count = cls._get_inst_count(cluster, 'SENTRY_SERVER') - dn_count = cls._get_inst_count(cluster, 'HDFS_DATANODE') - zk_count = cls._get_inst_count(cluster, 'ZOOKEEPER_SERVER') - - if snt_count > 1: - raise ex.InvalidComponentCountException( - 'SENTRY_SERVER', _('0 or 1'), snt_count) - if snt_count == 1: - if dn_count < 1: - raise ex.RequiredServiceMissingException( - 'HDFS_DATANODE', required_by='SENTRY_SERVER') - if zk_count < 1: - raise ex.RequiredServiceMissingException( - 'ZOOKEEPER', required_by='SENTRY_SERVER') - - @classmethod - def _solr_validation(cls, cluster): - slr_count = cls._get_inst_count(cluster, 'SOLR_SERVER') - zk_count = cls._get_inst_count(cluster, 'ZOOKEEPER_SERVER') - dn_count = cls._get_inst_count(cluster, 'HDFS_DATANODE') - - if slr_count >= 1: - if dn_count < 1: - raise ex.RequiredServiceMissingException( - 'HDFS_DATANODE', required_by='SOLR_SERVER') - if zk_count < 1: - raise ex.RequiredServiceMissingException( - 'ZOOKEEPER', required_by='SOLR_SERVER') - - @classmethod - def _sqoop_validation(cls, cluster): - - s2s_count = cls._get_inst_count(cluster, 'SQOOP_SERVER') - dn_count = cls._get_inst_count(cluster, 'HDFS_DATANODE') - hs_count = cls._get_inst_count(cluster, 'YARN_JOBHISTORY') - nm_count = cls._get_inst_count(cluster, 'YARN_NODEMANAGER') - - if s2s_count > 1: - raise ex.InvalidComponentCountException( - 'SQOOP_SERVER', _('0 or 1'), s2s_count) - if s2s_count == 1: - if dn_count < 1: - raise ex.RequiredServiceMissingException( - 'HDFS_DATANODE', required_by='SQOOP_SERVER') - if nm_count < 1: - raise ex.RequiredServiceMissingException( - 'YARN_NODEMANAGER', required_by='SQOOP_SERVER') - if hs_count != 1: - raise ex.RequiredServiceMissingException( - 'YARN_JOBHISTORY', required_by='SQOOP_SERVER') - - @classmethod - def _hbase_indexer_validation(cls, cluster): - - lhbi_count = cls._get_inst_count(cluster, 'HBASE_INDEXER') - zk_count = cls._get_inst_count(cluster, 'ZOOKEEPER_SERVER') - dn_count = cls._get_inst_count(cluster, 'HDFS_DATANODE') - slr_count = cls._get_inst_count(cluster, 'SOLR_SERVER') - hbm_count = cls._get_inst_count(cluster, 'HBASE_MASTER') - - if lhbi_count >= 1: - if dn_count < 1: - raise ex.RequiredServiceMissingException( - 'HDFS_DATANODE', required_by='HBASE_INDEXER') - if zk_count < 1: - raise ex.RequiredServiceMissingException( - 'ZOOKEEPER', required_by='HBASE_INDEXER') - if slr_count < 1: - raise ex.RequiredServiceMissingException( - 'SOLR_SERVER', required_by='HBASE_INDEXER') - if hbm_count < 1: - raise ex.RequiredServiceMissingException( - 'HBASE_MASTER', required_by='HBASE_INDEXER') - - @classmethod - def _impala_validation(cls, cluster): - ics_count = cls._get_inst_count(cluster, 'IMPALA_CATALOGSERVER') - iss_count = cls._get_inst_count(cluster, 'IMPALA_STATESTORE') - id_count = cls._get_inst_count(cluster, 'IMPALAD') - dn_count = cls._get_inst_count(cluster, 'HDFS_DATANODE') - hms_count = cls._get_inst_count(cluster, 'HIVE_METASTORE') - - if ics_count > 1: - raise ex.InvalidComponentCountException('IMPALA_CATALOGSERVER', - _('0 or 1'), ics_count) - if iss_count > 1: - raise ex.InvalidComponentCountException('IMPALA_STATESTORE', - _('0 or 1'), iss_count) - if ics_count == 1: - datanode_ng = u.get_node_groups(cluster, "HDFS_DATANODE") - impalad_ng = u.get_node_groups(cluster, "IMPALAD") - datanodes = set(ng.id for ng in datanode_ng) - impalads = set(ng.id for ng in impalad_ng) - - if datanodes != impalads: - raise ex.InvalidClusterTopology( - _("IMPALAD must be installed on every HDFS_DATANODE")) - - if iss_count != 1: - raise ex.RequiredServiceMissingException( - 'IMPALA_STATESTORE', required_by='IMPALA') - if id_count < 1: - raise ex.RequiredServiceMissingException( - 'IMPALAD', required_by='IMPALA') - if dn_count < 1: - raise ex.RequiredServiceMissingException( - 'HDFS_DATANODE', required_by='IMPALA') - if hms_count < 1: - raise ex.RequiredServiceMissingException( - 'HIVE_METASTORE', required_by='IMPALA') - - @classmethod - def _kms_validation(cls, cluster): - - kms_count = cls._get_inst_count(cluster, 'KMS') - if kms_count > 1: - raise ex.InvalidComponentCountException('KMS', - _('0 or 1'), kms_count) - - @classmethod - def _get_anti_affinity(cls, cluster): - return cluster.anti_affinity diff --git a/sahara/plugins/cdh/v5_4_0/versionhandler.py b/sahara/plugins/cdh/v5_4_0/versionhandler.py deleted file mode 100644 index 27e73525..00000000 --- a/sahara/plugins/cdh/v5_4_0/versionhandler.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright (c) 2015 Intel Corporation -# Copyright (c) 2015 ISPRAS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.plugins.cdh import abstractversionhandler as avm -from sahara.plugins.cdh.v5_4_0 import cloudera_utils -from sahara.plugins.cdh.v5_4_0 import config_helper -from sahara.plugins.cdh.v5_4_0 import deploy -from sahara.plugins.cdh.v5_4_0 import edp_engine -from sahara.plugins.cdh.v5_4_0 import plugin_utils -from sahara.plugins.cdh.v5_4_0 import validation - - -class VersionHandler(avm.BaseVersionHandler): - - def __init__(self): - super(VersionHandler, self).__init__() - self.config_helper = config_helper.ConfigHelperV540() - self.cloudera_utils = cloudera_utils.ClouderaUtilsV540() - self.plugin_utils = plugin_utils.PluginUtilsV540() - self.deploy = deploy - self.edp_engine = edp_engine - self.validation = validation.ValidatorV540() - - def get_node_processes(self): - return { - "CLOUDERA": ['CLOUDERA_MANAGER'], - "HDFS": ['HDFS_NAMENODE', 'HDFS_DATANODE', - 'HDFS_SECONDARYNAMENODE', 'HDFS_JOURNALNODE'], - "YARN": ['YARN_RESOURCEMANAGER', 'YARN_NODEMANAGER', - 'YARN_JOBHISTORY', 'YARN_STANDBYRM'], - "OOZIE": ['OOZIE_SERVER'], - "HIVE": ['HIVE_SERVER2', 'HIVE_METASTORE', 'HIVE_WEBHCAT'], - "HUE": ['HUE_SERVER'], - "SPARK_ON_YARN": ['SPARK_YARN_HISTORY_SERVER'], - "ZOOKEEPER": ['ZOOKEEPER_SERVER'], - "HBASE": ['HBASE_MASTER', 'HBASE_REGIONSERVER'], - "FLUME": ['FLUME_AGENT'], - "IMPALA": ['IMPALA_CATALOGSERVER', 'IMPALA_STATESTORE', 'IMPALAD'], - "KS_INDEXER": ['KEY_VALUE_STORE_INDEXER'], - "SOLR": ['SOLR_SERVER'], - "SQOOP": ['SQOOP_SERVER'], - "SENTRY": ['SENTRY_SERVER'], - "KMS": ['KMS'], - - "YARN_GATEWAY": [], - "RESOURCEMANAGER": [], - "NODEMANAGER": [], - "JOBHISTORY": [], - - "HDFS_GATEWAY": [], - 'DATANODE': [], - 'NAMENODE': [], - 'SECONDARYNAMENODE': [], - 'JOURNALNODE': [], - - 'REGIONSERVER': [], - 'MASTER': [], - - 'HIVEMETASTORE': [], - 'HIVESERVER': [], - 'WEBCAT': [], - - 'CATALOGSERVER': [], - 'STATESTORE': [], - 'IMPALAD': [], - } - - def get_edp_engine(self, cluster, job_type): - oozie_type = self.edp_engine.EdpOozieEngine.get_supported_job_types() - spark_type = self.edp_engine.EdpSparkEngine.get_supported_job_types() - if job_type in oozie_type: - return self.edp_engine.EdpOozieEngine(cluster) - if job_type in spark_type: - return self.edp_engine.EdpSparkEngine(cluster) - return None - - def get_edp_job_types(self): - return (edp_engine.EdpOozieEngine.get_supported_job_types() + - edp_engine.EdpSparkEngine.get_supported_job_types()) diff --git a/sahara/tests/unit/plugins/cdh/test_cloudera_utils.py b/sahara/tests/unit/plugins/cdh/test_cloudera_utils.py index 9395a81d..0d5009b1 100644 --- a/sahara/tests/unit/plugins/cdh/test_cloudera_utils.py +++ b/sahara/tests/unit/plugins/cdh/test_cloudera_utils.py @@ -16,16 +16,16 @@ import mock import testtools -from sahara.plugins.cdh.v5_3_0 import cloudera_utils as cu +from sahara.plugins.cdh import cloudera_utils as cu from sahara.tests.unit import base from sahara.tests.unit.plugins.cdh import utils as ctu -CU = cu.ClouderaUtilsV530() +CU = cu.ClouderaUtils() class ClouderaUtilsTestCase(base.SaharaTestCase): - @mock.patch('sahara.plugins.cdh.v5_3_0.cloudera_utils.ClouderaUtilsV530.' + @mock.patch('sahara.plugins.cdh.cloudera_utils.ClouderaUtils.' 'get_cloudera_cluster') def test_get_service(self, mock_get_cl_cluster): self.assertRaises(ValueError, CU.get_service_by_role, 'NAMENODE') @@ -83,7 +83,7 @@ class ClouderaUtilsTestCase(base.SaharaTestCase): 'yarn01', 'yarn01', 'oozie01', 'hive01', 'hive01', 'hive01', 'hue01', 'spark_on_yarn01', 'zookeeper01', 'hbase01', 'hbase01'] - provider = cu.ClouderaUtilsV530() + provider = cu.ClouderaUtils() cluster = mock.Mock() for (role, resp) in zip(roles, resps): self.assertEqual( diff --git a/sahara/tests/unit/plugins/cdh/test_cloudera_utils_540.py b/sahara/tests/unit/plugins/cdh/test_cloudera_utils_540.py deleted file mode 100644 index e41f78bc..00000000 --- a/sahara/tests/unit/plugins/cdh/test_cloudera_utils_540.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright (c) 2014 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock - -from sahara.plugins.cdh.v5_4_0 import cloudera_utils as cu -from sahara.tests.unit import base -from sahara.tests.unit.plugins.cdh import utils as ctu - - -CU = cu.ClouderaUtilsV540() - - -class ClouderaUtilsTestCase(base.SaharaTestCase): - @mock.patch('sahara.plugins.cdh.v5_4_0.cloudera_utils.ClouderaUtilsV540.' - 'get_cloudera_cluster') - def test_get_service(self, mock_get_cl_cluster): - self.assertRaises(ValueError, CU.get_service_by_role, 'NAMENODE') - - cluster = ctu.get_fake_cluster() - inst = cluster.node_groups[0].instances[0] - mock_get_cl_cluster.return_value = None - - self.assertRaises(ValueError, CU.get_service_by_role, 'spam', - cluster) - self.assertRaises(ValueError, CU.get_service_by_role, 'spam', - instance=inst) - - mock_get_cl_cluster.reset_mock() - - mock_get_service = mock.MagicMock() - mock_get_service.get_service.return_value = mock.Mock() - mock_get_cl_cluster.return_value = mock_get_service - - CU.get_service_by_role('NAMENODE', cluster) - args = ((CU.HDFS_SERVICE_NAME,),) - self.assertEqual(args, mock_get_service.get_service.call_args) - - mock_get_service.reset_mock() - CU.get_service_by_role('JOBHISTORY', instance=inst) - args = ((CU.YARN_SERVICE_NAME,),) - self.assertEqual(args, mock_get_service.get_service.call_args) - - mock_get_service.reset_mock() - CU.get_service_by_role('OOZIE_SERVER', cluster) - args = ((CU.OOZIE_SERVICE_NAME,),) - self.assertEqual(args, mock_get_service.get_service.call_args) - - def test_get_role_name(self): - inst_mock = mock.Mock() - inst_mock.hostname.return_value = 'spam-host' - - self.assertEqual('eggs_spam_host', - CU.pu.get_role_name(inst_mock, 'eggs')) diff --git a/sahara/tests/unit/plugins/cdh/test_utils.py b/sahara/tests/unit/plugins/cdh/test_utils.py index a27964cc..6822805d 100644 --- a/sahara/tests/unit/plugins/cdh/test_utils.py +++ b/sahara/tests/unit/plugins/cdh/test_utils.py @@ -13,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from sahara.plugins.cdh.v5_3_0 import plugin_utils as pu +from sahara.plugins.cdh import plugin_utils as pu from sahara.tests.unit import base from sahara.tests.unit.plugins.cdh import utils as tu -PU = pu.PluginUtilsV530() +PU = pu.AbstractPluginUtils() class UtilsTestCase(base.SaharaTestCase): diff --git a/sahara/tests/unit/plugins/cdh/test_utils_540.py b/sahara/tests/unit/plugins/cdh/test_utils_540.py deleted file mode 100644 index 633bfc02..00000000 --- a/sahara/tests/unit/plugins/cdh/test_utils_540.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) 2014 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.plugins.cdh.v5_4_0 import plugin_utils as pu -from sahara.tests.unit import base -from sahara.tests.unit.plugins.cdh import utils as tu - - -PU = pu.PluginUtilsV540() - - -class UtilsTestCase(base.SaharaTestCase): - def test_get_manager(self): - cluster = tu.get_fake_cluster() - inst = PU.get_manager(cluster) - self.assertEqual('id1', inst.instance_id) - - def test_get_namenode(self): - cluster = tu.get_fake_cluster() - inst = PU.get_namenode(cluster) - self.assertEqual('id2', inst.instance_id) - - def test_get_secondarynamenode(self): - cluster = tu.get_fake_cluster() - inst = PU.get_secondarynamenode(cluster) - self.assertEqual('id2', inst.instance_id) - - def test_get_resourcemanager(self): - cluster = tu.get_fake_cluster() - inst = PU.get_resourcemanager(cluster) - self.assertEqual('id2', inst.instance_id) - - def test_get_datanodes(self): - cluster = tu.get_fake_cluster() - dns = PU.get_datanodes(cluster) - ids = [dn.instance_id for dn in dns] - self.assertEqual(sorted(['id00', 'id01', 'id02']), sorted(ids)) - - def test_get_nodemanagers(self): - cluster = tu.get_fake_cluster() - nms = PU.get_nodemanagers(cluster) - ids = [nm.instance_id for nm in nms] - self.assertEqual(sorted(['id00', 'id01', 'id02']), sorted(ids)) - - def test_get_historyserver(self): - cluster = tu.get_fake_cluster() - inst = PU.get_historyserver(cluster) - self.assertEqual('id2', inst.instance_id) - - def test_get_oozie(self): - cluster = tu.get_fake_cluster() - inst = PU.get_oozie(cluster) - self.assertEqual('id2', inst.instance_id) diff --git a/sahara/tests/unit/plugins/cdh/test_versionfactory.py b/sahara/tests/unit/plugins/cdh/test_versionfactory.py index e73f1748..47127d0d 100644 --- a/sahara/tests/unit/plugins/cdh/test_versionfactory.py +++ b/sahara/tests/unit/plugins/cdh/test_versionfactory.py @@ -40,4 +40,4 @@ class VersionFactoryTestCase(base.SaharaTestCase): self.assertIsInstance(hander, avh.AbstractVersionHandler) def get_support_versions(self): - return ['5', '5.3.0', '5.4.0', '5.5.0', '5.7.0', '5.9.0'] + return ['5.5.0', '5.7.0', '5.9.0'] diff --git a/sahara/tests/unit/plugins/cdh/v5/__init__.py b/sahara/tests/unit/plugins/cdh/v5/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/sahara/tests/unit/plugins/cdh/v5/test_config_helper_5.py b/sahara/tests/unit/plugins/cdh/v5/test_config_helper_5.py deleted file mode 100644 index 183117a3..00000000 --- a/sahara/tests/unit/plugins/cdh/v5/test_config_helper_5.py +++ /dev/null @@ -1,139 +0,0 @@ -# Copyright (c) 2016 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from oslo_serialization import jsonutils as json - -from sahara.plugins.cdh.v5 import config_helper -from sahara.tests.unit import base -from sahara.tests.unit.plugins.cdh import utils as ctu -from sahara.utils import files as f - -c_h = config_helper.ConfigHelperV5() - -path_to_config = 'plugins/cdh/v5/resources/' -json_files = [ - 'hdfs-service.json', - 'hdfs-namenode.json', - 'hdfs-datanode.json', - 'hdfs-secondarynamenode.json', - 'hdfs-gateway.json', - 'yarn-service.json', - 'yarn-resourcemanager.json', - 'yarn-nodemanager.json', - 'yarn-jobhistory.json', - 'yarn-gateway.json', - 'oozie-service.json', - 'oozie-oozie.json', - 'hive-service.json', - 'hive-metastore.json', - 'hive-hiveserver2.json', - 'hive-webhcat.json', - 'hue-service.json', - 'hue-hue.json', - 'spark-service.json', - 'spark-history.json', - 'zookeeper-server.json', - 'zookeeper-service.json', - 'hbase-service.json', - 'hbase-master.json', - 'hbase-regionserver.json' -] - - -class ConfigHelperTestCase(base.SaharaTestCase): - - def test_get_ng_plugin_configs(self): - actual_configs = c_h._get_ng_plugin_configs() - - expected_configs = [] - for json_file in json_files: - expected_configs += json.loads( - f.get_file_text(path_to_config + json_file)) - - # compare names - expected_names = set(i['name'] for i in expected_configs) - actual_names = set(i.to_dict()['name'] for i in actual_configs) - self.assertEqual(expected_names, actual_names) - - def test_get_cdh5_repo_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.CDH5_REPO_URL.default_value, - c_h.get_cdh5_repo_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.CDH5_REPO_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_cdh5_repo_url(cluster)) - - def test_get_cdh5_key_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.CDH5_REPO_KEY_URL.default_value, - c_h.get_cdh5_key_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.CDH5_REPO_KEY_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_cdh5_key_url(cluster)) - - def test_get_cm5_repo_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.CM5_REPO_URL.default_value, - c_h.get_cm5_repo_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.CM5_REPO_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_cm5_repo_url(cluster)) - - def test_get_cm5_key_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.CM5_REPO_KEY_URL.default_value, - c_h.get_cm5_key_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.CM5_REPO_KEY_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_cm5_key_url(cluster)) - - def test_is_swift_enabled(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertTrue(c_h.is_swift_enabled(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.ENABLE_SWIFT.name: False}}) - self.assertFalse(c_h.is_swift_enabled(cluster)) - - def test_get_swift_lib_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.DEFAULT_SWIFT_LIB_URL, - c_h.get_swift_lib_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.SWIFT_LIB_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_swift_lib_url(cluster)) - - def test_is_hbase_common_lib_enabled(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertTrue(c_h.is_hbase_common_lib_enabled(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': - {c_h.ENABLE_HBASE_COMMON_LIB.name: False}}) - self.assertFalse(c_h.is_hbase_common_lib_enabled(cluster)) - - def test_get_extjs_lib_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.DEFAULT_EXTJS_LIB_URL, - c_h.get_extjs_lib_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.EXTJS_LIB_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_extjs_lib_url(cluster)) diff --git a/sahara/tests/unit/plugins/cdh/v5/test_edp_engine.py b/sahara/tests/unit/plugins/cdh/v5/test_edp_engine.py deleted file mode 100644 index ac15ad47..00000000 --- a/sahara/tests/unit/plugins/cdh/v5/test_edp_engine.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright (c) 2015 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock - -from sahara.plugins import base as pb -from sahara.plugins.cdh.v5 import edp_engine -from sahara.plugins import exceptions as ex -from sahara.tests.unit import base as sahara_base -from sahara.tests.unit.plugins.cdh import utils as c_u -from sahara.utils import edp - - -def get_cluster(): - cluster = c_u.get_fake_cluster(plugin_name='CDH', hadoop_version='5.0.0') - return cluster - - -class EdpEngineTestV5(sahara_base.SaharaTestCase): - - def setUp(self): - super(EdpEngineTestV5, self).setUp() - pb.setup_plugins() - - def test_get_hdfs_user(self): - eng = edp_engine.EdpOozieEngine(get_cluster()) - self.assertEqual('hdfs', eng.get_hdfs_user()) - - @mock.patch('sahara.service.edp.hdfs_helper.create_dir_hadoop2') - def test_create_hdfs_dir(self, create_dir_hadoop2): - eng = edp_engine.EdpOozieEngine(get_cluster()) - remote = mock.Mock() - dir_name = mock.Mock() - eng.create_hdfs_dir(remote, dir_name) - create_dir_hadoop2.assert_called_once_with(remote, - dir_name, - eng.get_hdfs_user()) - - def test_get_oozie_server_uri(self): - cluster = get_cluster() - eng = edp_engine.EdpOozieEngine(cluster) - uri = eng.get_oozie_server_uri(cluster) - self.assertEqual("http://1.2.3.5:11000/oozie", uri) - - def test_get_name_node_uri(self): - cluster = get_cluster() - eng = edp_engine.EdpOozieEngine(cluster) - uri = eng.get_name_node_uri(cluster) - self.assertEqual("hdfs://master_inst.novalocal:8020", uri) - - def test_get_resource_manager_uri(self): - cluster = get_cluster() - eng = edp_engine.EdpOozieEngine(cluster) - uri = eng.get_resource_manager_uri(cluster) - self.assertEqual("master_inst.novalocal:8032", uri) - - def test_get_oozie_server(self): - cluster = get_cluster() - eng = edp_engine.EdpOozieEngine(cluster) - actual = eng.get_oozie_server(cluster) - expect = cluster.node_groups[1].instances[0] - self.assertEqual(expect, actual) - - @mock.patch('sahara.service.edp.oozie.engine.' - 'OozieJobEngine.validate_job_execution') - def test_validate_job_execution(self, c): - cluster = get_cluster() - eng = edp_engine.EdpOozieEngine(cluster) - eng.validate_job_execution(cluster, mock.Mock(), mock.Mock()) - - # more than one oozie server - dict.__setitem__(cluster.node_groups[1], 'count', 2) - self.assertRaises(ex.InvalidComponentCountException, - eng.validate_job_execution, cluster, - mock.Mock(), mock.Mock()) - - @mock.patch( - 'sahara.plugins.cdh.confighints_helper.get_possible_hive_config_from', - return_value={}) - def test_get_possible_job_config_hive(self, - get_possible_hive_config_from): - expected_config = {'job_config': {}} - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_HIVE) - get_possible_hive_config_from.assert_called_once_with( - 'plugins/cdh/v5/resources/hive-site.xml') - self.assertEqual(expected_config, actual_config) - - @mock.patch('sahara.plugins.cdh.v5.edp_engine.EdpOozieEngine') - def test_get_possible_job_config_java(self, BaseCDHEdpOozieEngine): - expected_config = {'job_config': {}} - BaseCDHEdpOozieEngine.get_possible_job_config.return_value = ( - expected_config) - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_JAVA) - BaseCDHEdpOozieEngine.get_possible_job_config.assert_called_once_with( - edp.JOB_TYPE_JAVA) - self.assertEqual(expected_config, actual_config) - - @mock.patch( - 'sahara.plugins.cdh.confighints_helper.' - 'get_possible_mapreduce_config_from', - return_value={}) - def test_get_possible_job_config_mapreduce( - self, get_possible_mapreduce_config_from): - expected_config = {'job_config': {}} - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_MAPREDUCE) - get_possible_mapreduce_config_from.assert_called_once_with( - 'plugins/cdh/v5/resources/mapred-site.xml') - self.assertEqual(expected_config, actual_config) - - @mock.patch( - 'sahara.plugins.cdh.confighints_helper.' - 'get_possible_mapreduce_config_from', - return_value={}) - def test_get_possible_job_config_mapreduce_streaming( - self, get_possible_mapreduce_config_from): - expected_config = {'job_config': {}} - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_MAPREDUCE_STREAMING) - get_possible_mapreduce_config_from.assert_called_once_with( - 'plugins/cdh/v5/resources/mapred-site.xml') - self.assertEqual(expected_config, actual_config) - - @mock.patch( - 'sahara.plugins.cdh.confighints_helper.get_possible_pig_config_from', - return_value={}) - def test_get_possible_job_config_pig(self, - get_possible_pig_config_from): - expected_config = {'job_config': {}} - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_PIG) - get_possible_pig_config_from.assert_called_once_with( - 'plugins/cdh/v5/resources/mapred-site.xml') - self.assertEqual(expected_config, actual_config) - - @mock.patch('sahara.plugins.cdh.v5.edp_engine.EdpOozieEngine') - def test_get_possible_job_config_shell(self, BaseCDHEdpOozieEngine): - expected_config = {'job_config': {}} - BaseCDHEdpOozieEngine.get_possible_job_config.return_value = ( - expected_config) - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_SHELL) - BaseCDHEdpOozieEngine.get_possible_job_config.assert_called_once_with( - edp.JOB_TYPE_SHELL) - self.assertEqual(expected_config, actual_config) diff --git a/sahara/tests/unit/plugins/cdh/v5/test_plugin_utils_5.py b/sahara/tests/unit/plugins/cdh/v5/test_plugin_utils_5.py deleted file mode 100644 index 8b914518..00000000 --- a/sahara/tests/unit/plugins/cdh/v5/test_plugin_utils_5.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2015 Intel Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock - -from sahara.plugins.cdh.v5 import plugin_utils as pu -from sahara.tests.unit.plugins.cdh import base_plugin_utils_test - - -class TestPluginUtilsV5(base_plugin_utils_test.TestPluginUtils): - - def setUp(self): - super(TestPluginUtilsV5, self).setUp() - self.plug_utils = pu.PluginUtilsV5() - self.version = "v5" - - @mock.patch('sahara.config.CONF.disable_event_log') - def test_create_hive_hive_directory(self, log_cfg): - cluster = base_plugin_utils_test.get_concrete_cluster() - namenode = cluster.node_groups[1].instances[0] - self.plug_utils.create_hive_hive_directory(cluster) - with namenode.remote() as r: - calls = [mock.call('sudo su - -c "hadoop fs -mkdir -p' - ' /tmp/hive-hive" hdfs'), - mock.call('sudo su - -c "hadoop fs -chown hive' - ' /tmp/hive-hive" hdfs')] - r.execute_command.assert_has_calls(calls, any_order=False) diff --git a/sahara/tests/unit/plugins/cdh/v5/test_validation.py b/sahara/tests/unit/plugins/cdh/v5/test_validation.py deleted file mode 100644 index 8d371274..00000000 --- a/sahara/tests/unit/plugins/cdh/v5/test_validation.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2015 Intel Corpration -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.plugins.cdh.v5 import validation -from sahara.tests.unit.plugins.cdh import base_validation_tests as bvt - - -class ValidationTestCase(bvt.BaseValidationTestCase): - def setUp(self): - super(ValidationTestCase, self).setUp() - self.module = validation.ValidatorV5 diff --git a/sahara/tests/unit/plugins/cdh/v5/test_versionhandler.py b/sahara/tests/unit/plugins/cdh/v5/test_versionhandler.py deleted file mode 100644 index 7bc9ff07..00000000 --- a/sahara/tests/unit/plugins/cdh/v5/test_versionhandler.py +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright (c) 2015 Intel Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock -import six - -from sahara.plugins.cdh.v5 import edp_engine -from sahara.plugins.cdh.v5 import versionhandler -from sahara.tests.unit import base - - -class VersionHandlerTestCase(base.SaharaTestCase): - - plugin_path = "sahara.plugins.cdh.v5." - cloudera_utils_path = plugin_path + "cloudera_utils.ClouderaUtilsV5." - plugin_utils_path = plugin_path + "plugin_utils.PluginUtilsV5." - - def setUp(self): - super(VersionHandlerTestCase, self).setUp() - self.vh = versionhandler.VersionHandler() - - def test_get_node_processes(self): - processes = self.vh.get_node_processes() - for k, v in six.iteritems(processes): - for p in v: - self.assertIsInstance(p, str) - - @mock.patch("sahara.conductor.API.cluster_update") - @mock.patch("sahara.context.ctx") - @mock.patch(plugin_path + "deploy.configure_cluster") - @mock.patch(cloudera_utils_path + "get_cloudera_manager_info", - return_value={"fake_cm_info": "fake"}) - def test_config_cluster(self, get_cm_info, configure_cluster, - ctx, cluster_update): - cluster = mock.Mock() - self.vh.configure_cluster(cluster) - configure_cluster.assert_called_once_with(cluster) - cluster_update.assert_called_once_with( - ctx(), cluster, - {'info': {"fake_cm_info": "fake"}}) - - @mock.patch(plugin_path + "deploy.start_cluster") - def test_start_cluster(self, start_cluster): - cluster = mock.Mock() - self.vh._set_cluster_info = mock.Mock() - self.vh.start_cluster(cluster) - start_cluster.assert_called_once_with(cluster) - self.vh._set_cluster_info.assert_called_once_with(cluster) - - @mock.patch(plugin_path + "deploy.decommission_cluster") - def test_decommission_nodes(self, decommission_cluster): - cluster = mock.Mock() - instances = mock.Mock() - self.vh.decommission_nodes(cluster, instances) - decommission_cluster.assert_called_once_with(cluster, - instances) - - @mock.patch(plugin_path + "deploy.scale_cluster") - def test_scale_cluster(self, scale_cluster): - cluster = mock.Mock() - instances = mock.Mock() - self.vh.scale_cluster(cluster, instances) - scale_cluster.assert_called_once_with(cluster, instances) - - @mock.patch("sahara.conductor.API.cluster_update") - @mock.patch("sahara.context.ctx") - @mock.patch(cloudera_utils_path + "get_cloudera_manager_info", - return_value={}) - @mock.patch(plugin_utils_path + "get_hue") - def test_set_cluster_info(self, get_hue, get_cloudera_manager_info, - ctx, cluster_update): - hue = mock.Mock() - hue.get_ip_or_dns_name.return_value = "1.2.3.4" - get_hue.return_value = hue - cluster = mock.Mock() - self.vh._set_cluster_info(cluster) - info = {'info': {'Hue Dashboard': {'Web UI': 'http://1.2.3.4:8888'}}} - cluster_update.assert_called_once_with(ctx(), cluster, info) - - @mock.patch("sahara.service.edp.job_utils.get_plugin") - def test_get_edp_engine(self, get_plugin): - cluster = mock.Mock() - job_type = 'Java' - ret = self.vh.get_edp_engine(cluster, job_type) - self.assertIsInstance(ret, edp_engine.EdpOozieEngine) - - job_type = 'unsupported' - ret = self.vh.get_edp_engine(cluster, job_type) - self.assertIsNone(ret) - - def test_get_edp_job_types(self): - ret = self.vh.get_edp_job_types() - self.assertEqual(edp_engine.EdpOozieEngine.get_supported_job_types(), - ret) - - @mock.patch(plugin_path + - "edp_engine.EdpOozieEngine.get_possible_job_config", - return_value={'job_config': {}}) - def test_edp_config_hints(self, get_possible_job_config): - job_type = mock.Mock() - ret = self.vh.get_edp_config_hints(job_type) - get_possible_job_config.assert_called_once_with(job_type) - self.assertEqual(ret, {'job_config': {}}) - - @mock.patch(plugin_path + "deploy.get_open_ports", return_value=[1234]) - def test_get_open_ports(self, get_open_ports): - node_group = mock.Mock() - ret = self.vh.get_open_ports(node_group) - get_open_ports.assert_called_once_with(node_group) - self.assertEqual(ret, [1234]) - - @mock.patch(plugin_utils_path + "recommend_configs") - def test_recommend_configs(self, recommend_configs): - cluster = mock.Mock() - scaling = mock.Mock() - self.vh.get_plugin_configs = mock.Mock() - self.vh.recommend_configs(cluster, scaling) - recommend_configs.assert_called_once_with(cluster, - self.vh.get_plugin_configs(), - scaling) diff --git a/sahara/tests/unit/plugins/cdh/v5_3_0/__init__.py b/sahara/tests/unit/plugins/cdh/v5_3_0/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/sahara/tests/unit/plugins/cdh/v5_3_0/test_config_helper_530.py b/sahara/tests/unit/plugins/cdh/v5_3_0/test_config_helper_530.py deleted file mode 100644 index 4151f0fd..00000000 --- a/sahara/tests/unit/plugins/cdh/v5_3_0/test_config_helper_530.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright (c) 2016 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from oslo_serialization import jsonutils as json - -from sahara.plugins.cdh.v5_3_0 import config_helper -from sahara.tests.unit import base -from sahara.tests.unit.plugins.cdh import utils as ctu -from sahara.utils import files as f - -c_h = config_helper.ConfigHelperV530() - -path_to_config = 'plugins/cdh/v5_3_0/resources/' -json_files = [ - 'hdfs-service.json', - 'hdfs-namenode.json', - 'hdfs-datanode.json', - 'hdfs-secondarynamenode.json', - 'hdfs-gateway.json', - 'yarn-service.json', - 'yarn-resourcemanager.json', - 'yarn-nodemanager.json', - 'yarn-jobhistory.json', - 'yarn-gateway.json', - 'oozie-service.json', - 'oozie-oozie_server.json', - 'hive-service.json', - 'hive-hivemetastore.json', - 'hive-hiveserver2.json', - 'hive-webhcat.json', - 'hue-service.json', - 'hue-hue_server.json', - 'spark-service.json', - 'spark-spark_yarn_history_server.json', - 'zookeeper-service.json', - 'zookeeper-server.json', - 'hbase-service.json', - 'hbase-master.json', - 'hbase-regionserver.json', - 'flume-service.json', - 'flume-agent.json', - 'sentry-service.json', - 'sentry-sentry_server.json', - 'solr-service.json', - 'solr-solr_server.json', - 'sqoop-service.json', - 'sqoop-sqoop_server.json', - 'ks_indexer-service.json', - 'ks_indexer-hbase_indexer.json', - 'impala-service.json', - 'impala-catalogserver.json', - 'impala-impalad.json', - 'impala-statestore.json', -] - - -class ConfigHelperTestCase(base.SaharaTestCase): - - def test_get_ng_plugin_configs(self): - actual_configs = c_h._get_ng_plugin_configs() - - expected_configs = [] - for json_file in json_files: - expected_configs += json.loads( - f.get_file_text(path_to_config + json_file)) - - # compare names - expected_names = set(i['name'] for i in expected_configs) - actual_names = set(i.to_dict()['name'] for i in actual_configs) - self.assertEqual(expected_names, actual_names) - - def test_get_cdh5_repo_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.CDH5_REPO_URL.default_value, - c_h.get_cdh5_repo_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.CDH5_REPO_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_cdh5_repo_url(cluster)) - - def test_get_cdh5_key_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.CDH5_REPO_KEY_URL.default_value, - c_h.get_cdh5_key_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.CDH5_REPO_KEY_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_cdh5_key_url(cluster)) - - def test_get_cm5_repo_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.CM5_REPO_URL.default_value, - c_h.get_cm5_repo_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.CM5_REPO_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_cm5_repo_url(cluster)) - - def test_get_cm5_key_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.CM5_REPO_KEY_URL.default_value, - c_h.get_cm5_key_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.CM5_REPO_KEY_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_cm5_key_url(cluster)) - - def test_is_swift_enabled(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertTrue(c_h.is_swift_enabled(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.ENABLE_SWIFT.name: False}}) - self.assertFalse(c_h.is_swift_enabled(cluster)) - - def test_get_swift_lib_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.DEFAULT_SWIFT_LIB_URL, - c_h.get_swift_lib_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.SWIFT_LIB_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_swift_lib_url(cluster)) - - def test_is_hbase_common_lib_enabled(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertTrue(c_h.is_hbase_common_lib_enabled(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': - {c_h.ENABLE_HBASE_COMMON_LIB.name: False}}) - self.assertFalse(c_h.is_hbase_common_lib_enabled(cluster)) - - def test_get_extjs_lib_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.DEFAULT_EXTJS_LIB_URL, - c_h.get_extjs_lib_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.EXTJS_LIB_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_extjs_lib_url(cluster)) diff --git a/sahara/tests/unit/plugins/cdh/v5_3_0/test_edp_engine.py b/sahara/tests/unit/plugins/cdh/v5_3_0/test_edp_engine.py deleted file mode 100644 index bb1c4cd4..00000000 --- a/sahara/tests/unit/plugins/cdh/v5_3_0/test_edp_engine.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright (c) 2015 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock - -from sahara import exceptions as ex -from sahara.plugins import base as pb -from sahara.plugins.cdh.v5_3_0 import edp_engine -from sahara.plugins import exceptions as pl_ex -from sahara.tests.unit import base as sahara_base -from sahara.tests.unit.plugins.cdh import utils as c_u -from sahara.utils import edp - - -def get_cluster(version='5.3.0'): - cluster = c_u.get_fake_cluster(plugin_name='CDH', hadoop_version=version) - return cluster - - -class EdpEngineTestV530(sahara_base.SaharaTestCase): - - def setUp(self): - super(EdpEngineTestV530, self).setUp() - pb.setup_plugins() - - def test_get_hdfs_user(self): - eng = edp_engine.EdpOozieEngine(get_cluster()) - self.assertEqual('hdfs', eng.get_hdfs_user()) - - @mock.patch('sahara.service.edp.hdfs_helper.create_dir_hadoop2') - def test_create_hdfs_dir(self, create_dir_hadoop2): - eng = edp_engine.EdpOozieEngine(get_cluster()) - remote = mock.Mock() - dir_name = mock.Mock() - eng.create_hdfs_dir(remote, dir_name) - create_dir_hadoop2.assert_called_once_with(remote, - dir_name, - eng.get_hdfs_user()) - - def test_get_oozie_server_uri(self): - cluster = get_cluster() - eng = edp_engine.EdpOozieEngine(cluster) - uri = eng.get_oozie_server_uri(cluster) - self.assertEqual("http://1.2.3.5:11000/oozie", uri) - - def test_get_name_node_uri(self): - cluster = get_cluster() - eng = edp_engine.EdpOozieEngine(cluster) - uri = eng.get_name_node_uri(cluster) - self.assertEqual("hdfs://master_inst.novalocal:8020", uri) - - def test_get_resource_manager_uri(self): - cluster = get_cluster() - eng = edp_engine.EdpOozieEngine(cluster) - uri = eng.get_resource_manager_uri(cluster) - self.assertEqual("master_inst.novalocal:8032", uri) - - def test_get_oozie_server(self): - cluster = get_cluster() - eng = edp_engine.EdpOozieEngine(cluster) - actual = eng.get_oozie_server(cluster) - expect = cluster.node_groups[1].instances[0] - self.assertEqual(expect, actual) - - @mock.patch('sahara.service.edp.oozie.engine.' - 'OozieJobEngine.validate_job_execution') - def test_validate_job_execution(self, c): - cluster = get_cluster() - eng = edp_engine.EdpOozieEngine(cluster) - eng.validate_job_execution(cluster, mock.Mock(), mock.Mock()) - - # more than one oozie server - dict.__setitem__(cluster.node_groups[1], 'count', 2) - self.assertRaises(pl_ex.InvalidComponentCountException, - eng.validate_job_execution, cluster, - mock.Mock(), mock.Mock()) - - @mock.patch( - 'sahara.plugins.cdh.confighints_helper.get_possible_hive_config_from', - return_value={}) - def test_get_possible_job_config_hive(self, - get_possible_hive_config_from): - expected_config = {'job_config': {}} - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_HIVE) - get_possible_hive_config_from.assert_called_once_with( - 'plugins/cdh/v5_3_0/resources/hive-site.xml') - self.assertEqual(expected_config, actual_config) - - @mock.patch('sahara.plugins.cdh.v5_3_0.edp_engine.EdpOozieEngine') - def test_get_possible_job_config_java(self, BaseCDHEdpOozieEngine): - expected_config = {'job_config': {}} - BaseCDHEdpOozieEngine.get_possible_job_config.return_value = ( - expected_config) - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_JAVA) - BaseCDHEdpOozieEngine.get_possible_job_config.assert_called_once_with( - edp.JOB_TYPE_JAVA) - self.assertEqual(expected_config, actual_config) - - @mock.patch( - 'sahara.plugins.cdh.confighints_helper.' - 'get_possible_mapreduce_config_from', - return_value={}) - def test_get_possible_job_config_mapreduce( - self, get_possible_mapreduce_config_from): - expected_config = {'job_config': {}} - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_MAPREDUCE) - get_possible_mapreduce_config_from.assert_called_once_with( - 'plugins/cdh/v5_3_0/resources/mapred-site.xml') - self.assertEqual(expected_config, actual_config) - - @mock.patch( - 'sahara.plugins.cdh.confighints_helper.' - 'get_possible_mapreduce_config_from', - return_value={}) - def test_get_possible_job_config_mapreduce_streaming( - self, get_possible_mapreduce_config_from): - expected_config = {'job_config': {}} - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_MAPREDUCE_STREAMING) - get_possible_mapreduce_config_from.assert_called_once_with( - 'plugins/cdh/v5_3_0/resources/mapred-site.xml') - self.assertEqual(expected_config, actual_config) - - @mock.patch( - 'sahara.plugins.cdh.confighints_helper.get_possible_pig_config_from', - return_value={}) - def test_get_possible_job_config_pig(self, - get_possible_pig_config_from): - expected_config = {'job_config': {}} - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_PIG) - get_possible_pig_config_from.assert_called_once_with( - 'plugins/cdh/v5_3_0/resources/mapred-site.xml') - self.assertEqual(expected_config, actual_config) - - @mock.patch('sahara.plugins.cdh.v5_3_0.edp_engine.EdpOozieEngine') - def test_get_possible_job_config_shell(self, BaseCDHEdpOozieEngine): - expected_config = {'job_config': {}} - BaseCDHEdpOozieEngine.get_possible_job_config.return_value = ( - expected_config) - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_SHELL) - BaseCDHEdpOozieEngine.get_possible_job_config.assert_called_once_with( - edp.JOB_TYPE_SHELL) - self.assertEqual(expected_config, actual_config) - - @mock.patch('sahara.plugins.utils.get_config_value_or_default') - @mock.patch('sahara.plugins.utils.get_instance') - @mock.patch('sahara.service.edp.spark.engine.' - 'SparkJobEngine.validate_job_execution') - def test_spark_engine_validate_job_execution(self, - validate_job_execution, - get_instance, - get_config_value_or_default): - # version unsupported - cluster = get_cluster(version='5.0.0') - eng = edp_engine.EdpSparkEngine(cluster) - self.assertRaises(ex.InvalidDataException, - eng.validate_job_execution, cluster, - mock.Mock(), mock.Mock()) - - # none yarn history server - cluster = get_cluster() - eng = edp_engine.EdpSparkEngine(cluster) - self.assertRaises(pl_ex.InvalidComponentCountException, - eng.validate_job_execution, cluster, - mock.Mock(), mock.Mock()) - # valid - cluster = get_cluster() - yarn_history_node_group = mock.Mock() - yarn_history_node_group.node_processes = ['SPARK_YARN_HISTORY_SERVER'] - yarn_history_node_group.count = 1 - list.append(cluster.node_groups, yarn_history_node_group) - eng = edp_engine.EdpSparkEngine(cluster) - eng.validate_job_execution(cluster, mock.Mock(), mock.Mock()) diff --git a/sahara/tests/unit/plugins/cdh/v5_3_0/test_plugin_utils_530.py b/sahara/tests/unit/plugins/cdh/v5_3_0/test_plugin_utils_530.py deleted file mode 100644 index 34ac31db..00000000 --- a/sahara/tests/unit/plugins/cdh/v5_3_0/test_plugin_utils_530.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2015 Intel Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from sahara.plugins.cdh.v5_3_0 import plugin_utils as pu -from sahara.tests.unit.plugins.cdh import base_plugin_utils_test - - -class TestPluginUtilsV530(base_plugin_utils_test.TestPluginUtilsHigherThanV5): - - def setUp(self): - super(TestPluginUtilsV530, self).setUp() - self.plug_utils = pu.PluginUtilsV530() - self.version = "v5_3_0" diff --git a/sahara/tests/unit/plugins/cdh/v5_3_0/test_spark_engine.py b/sahara/tests/unit/plugins/cdh/v5_3_0/test_spark_engine.py deleted file mode 100644 index 47f04612..00000000 --- a/sahara/tests/unit/plugins/cdh/v5_3_0/test_spark_engine.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (c) 2015 ISPRAS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock -import testtools - -from sahara import conductor as cond -from sahara import context -from sahara.plugins import base as pb -from sahara.plugins import exceptions as ex -from sahara.service.edp.spark import engine -from sahara.tests.unit import base -from sahara.utils import edp - - -conductor = cond.API - - -class SparkPluginTest(base.SaharaWithDbTestCase): - def setUp(self): - super(SparkPluginTest, self).setUp() - self.override_config("plugins", ["cdh"]) - pb.setup_plugins() - - def test_plugin_edp_engine_no_spark(self): - cluster_dict = { - 'name': 'cluster', - 'plugin_name': 'cdh', - 'hadoop_version': '5.3.0', - 'default_image_id': 'image'} - - job = mock.Mock() - job.type = edp.JOB_TYPE_SPARK - - cluster = conductor.cluster_create(context.ctx(), cluster_dict) - plugin = pb.PLUGINS.get_plugin(cluster.plugin_name) - edp_engine = plugin.get_edp_engine(cluster, edp.JOB_TYPE_SPARK) - with testtools.ExpectedException( - ex.InvalidComponentCountException, - value_re="Hadoop cluster should contain 1 " - "SPARK_YARN_HISTORY_SERVER component\(s\). Actual " - "SPARK_YARN_HISTORY_SERVER count is 0\nError ID: .*"): - - edp_engine.validate_job_execution(cluster, job, mock.Mock()) - - def test_plugin_edp_engine(self): - cluster_dict = { - 'name': 'cluster', - 'plugin_name': 'cdh', - 'hadoop_version': '5.3.0', - 'default_image_id': 'image'} - - cluster = conductor.cluster_create(context.ctx(), cluster_dict) - plugin = pb.PLUGINS.get_plugin(cluster.plugin_name) - self.assertIsInstance( - plugin.get_edp_engine(cluster, edp.JOB_TYPE_SPARK), - engine.SparkJobEngine) diff --git a/sahara/tests/unit/plugins/cdh/v5_3_0/test_validation.py b/sahara/tests/unit/plugins/cdh/v5_3_0/test_validation.py deleted file mode 100644 index 4a7ae328..00000000 --- a/sahara/tests/unit/plugins/cdh/v5_3_0/test_validation.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright (c) 2015 Intel Corpration -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.plugins.cdh.v5_3_0 import validation -from sahara.plugins import exceptions as ex -from sahara.tests.unit.plugins.cdh import base_validation_tests as bvt - -icte = ex.InvalidClusterTopology -icce = ex.InvalidComponentCountException -rsme = ex.RequiredServiceMissingException - - -class ValidationTestCase(bvt.BaseValidationTestCase): - - def setUp(self): - super(ValidationTestCase, self).setUp() - self.module = validation.ValidatorV530 - - def _get_test_cases(self): - cases = super(ValidationTestCase, self)._get_test_cases() - cases += [ - [None, {'FLUME_AGENT': 1}], - [icce, {'ZOOKEEPER_SERVER': 1, 'SENTRY_SERVER': 2}], - [None, {'ZOOKEEPER_SERVER': 1, 'SENTRY_SERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 0, 'SENTRY_SERVER': 1}], - [None, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 0, 'SOLR_SERVER': 1}], - [None, {'YARN_NODEMANAGER': 1, 'YARN_JOBHISTORY': 1, - 'SQOOP_SERVER': 1}], - [rsme, {'YARN_NODEMANAGER': 0, 'YARN_JOBHISTORY': 1, - 'SQOOP_SERVER': 1}], - [rsme, {'YARN_NODEMANAGER': 1, 'YARN_JOBHISTORY': 0, - 'SQOOP_SERVER': 1}], - # HBASE_MASTER AND HBASE_REGIONSERVER depend circularly - [None, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1, - 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, - 'HBASE_REGIONSERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 0, 'SOLR_SERVER': 1, - 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, - 'HBASE_REGIONSERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 0, - 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, - 'HBASE_REGIONSERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1, - 'HBASE_MASTER': 0, 'HBASE_INDEXER': 1}], - ] - - worker_with_implama = ('worker_ng', 1, ['HDFS_DATANODE', - 'YARN_NODEMANAGER', - 'IMPALAD'], 3) - cases += [ - [None, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1, - 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, - [worker_with_implama]], - [icte, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1}, - []], - [icte, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1}, - [worker_with_implama]], - [rsme, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 0, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1, - 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, - [worker_with_implama]], - [rsme, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 0, 'HIVE_SERVER2': 1, - 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, - [worker_with_implama]] - - ] - return cases diff --git a/sahara/tests/unit/plugins/cdh/v5_3_0/test_versionhandler.py b/sahara/tests/unit/plugins/cdh/v5_3_0/test_versionhandler.py deleted file mode 100644 index 61daf645..00000000 --- a/sahara/tests/unit/plugins/cdh/v5_3_0/test_versionhandler.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright (c) 2015 Intel Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock -import six - -from sahara.plugins.cdh.v5_3_0 import edp_engine -from sahara.plugins.cdh.v5_3_0 import versionhandler -from sahara.tests.unit import base - - -class VersionHandlerTestCase(base.SaharaTestCase): - - plugin_path = "sahara.plugins.cdh.v5_3_0." - cloudera_utils_path = plugin_path + "cloudera_utils.ClouderaUtilsV530." - plugin_utils_path = plugin_path + "plugin_utils.PluginUtilsV530." - - def setUp(self): - super(VersionHandlerTestCase, self).setUp() - self.vh = versionhandler.VersionHandler() - - def test_get_node_processes(self): - processes = self.vh.get_node_processes() - for k, v in six.iteritems(processes): - for p in v: - self.assertIsInstance(p, str) - - @mock.patch("sahara.conductor.API.cluster_update") - @mock.patch("sahara.context.ctx") - @mock.patch(plugin_path + "deploy.configure_cluster") - @mock.patch(cloudera_utils_path + "get_cloudera_manager_info", - return_value={"fake_cm_info": "fake"}) - def test_config_cluster(self, get_cm_info, configure_cluster, - ctx, cluster_update): - cluster = mock.Mock() - self.vh.configure_cluster(cluster) - configure_cluster.assert_called_once_with(cluster) - cluster_update.assert_called_once_with( - ctx(), cluster, - {'info': {"fake_cm_info": "fake"}}) - - @mock.patch(plugin_path + "deploy.start_cluster") - def test_start_cluster(self, start_cluster): - cluster = mock.Mock() - self.vh._set_cluster_info = mock.Mock() - self.vh.start_cluster(cluster) - start_cluster.assert_called_once_with(cluster) - self.vh._set_cluster_info.assert_called_once_with(cluster) - - @mock.patch(plugin_path + "deploy.decommission_cluster") - def test_decommission_nodes(self, decommission_cluster): - cluster = mock.Mock() - instances = mock.Mock() - self.vh.decommission_nodes(cluster, instances) - decommission_cluster.assert_called_once_with(cluster, - instances) - - @mock.patch(plugin_path + "deploy.scale_cluster") - def test_scale_cluster(self, scale_cluster): - cluster = mock.Mock() - instances = mock.Mock() - self.vh.scale_cluster(cluster, instances) - scale_cluster.assert_called_once_with(cluster, instances) - - @mock.patch("sahara.conductor.API.cluster_update") - @mock.patch("sahara.context.ctx") - @mock.patch(cloudera_utils_path + "get_cloudera_manager_info", - return_value={}) - @mock.patch(plugin_utils_path + "get_hue") - def test_set_cluster_info(self, get_hue, get_cloudera_manager_info, - ctx, cluster_update): - hue = mock.Mock() - hue.get_ip_or_dns_name.return_value = "1.2.3.4" - get_hue.return_value = hue - cluster = mock.Mock() - self.vh._set_cluster_info(cluster) - info = {'info': {'Hue Dashboard': {'Web UI': 'http://1.2.3.4:8888'}}} - cluster_update.assert_called_once_with(ctx(), cluster, info) - - @mock.patch("sahara.plugins.utils.get_instance") - @mock.patch("sahara.plugins.utils.get_config_value_or_default") - @mock.patch("sahara.service.edp.job_utils.get_plugin") - def test_get_edp_engine(self, get_plugin, get_config_value_or_default, - get_instance): - cluster = mock.Mock() - job_type = 'Java' - ret = self.vh.get_edp_engine(cluster, job_type) - self.assertIsInstance(ret, edp_engine.EdpOozieEngine) - - job_type = 'Spark' - ret = self.vh.get_edp_engine(cluster, job_type) - self.assertIsInstance(ret, edp_engine.EdpSparkEngine) - - job_type = 'unsupported' - ret = self.vh.get_edp_engine(cluster, job_type) - self.assertIsNone(ret) - - def test_get_edp_job_types(self): - ret = self.vh.get_edp_job_types() - expect = edp_engine.EdpOozieEngine.get_supported_job_types() + \ - edp_engine.EdpSparkEngine.get_supported_job_types() - self.assertEqual(expect, ret) - - @mock.patch(plugin_path + - "edp_engine.EdpOozieEngine.get_possible_job_config", - return_value={'job_config': {}}) - def test_edp_config_hints(self, get_possible_job_config): - job_type = mock.Mock() - ret = self.vh.get_edp_config_hints(job_type) - get_possible_job_config.assert_called_once_with(job_type) - self.assertEqual(ret, {'job_config': {}}) - - @mock.patch(plugin_path + "deploy.get_open_ports", return_value=[1234]) - def test_get_open_ports(self, get_open_ports): - node_group = mock.Mock() - ret = self.vh.get_open_ports(node_group) - get_open_ports.assert_called_once_with(node_group) - self.assertEqual(ret, [1234]) - - @mock.patch(plugin_utils_path + "recommend_configs") - def test_recommend_configs(self, recommend_configs): - cluster = mock.Mock() - scaling = mock.Mock() - self.vh.get_plugin_configs = mock.Mock() - self.vh.recommend_configs(cluster, scaling) - recommend_configs.assert_called_once_with(cluster, - self.vh.get_plugin_configs(), - scaling) diff --git a/sahara/tests/unit/plugins/cdh/v5_4_0/__init__.py b/sahara/tests/unit/plugins/cdh/v5_4_0/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/sahara/tests/unit/plugins/cdh/v5_4_0/test_config_helper_540.py b/sahara/tests/unit/plugins/cdh/v5_4_0/test_config_helper_540.py deleted file mode 100644 index f4e0ccf8..00000000 --- a/sahara/tests/unit/plugins/cdh/v5_4_0/test_config_helper_540.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright (c) 2016 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from oslo_serialization import jsonutils as json - -from sahara.plugins.cdh.v5_4_0 import config_helper -from sahara.tests.unit import base -from sahara.tests.unit.plugins.cdh import utils as ctu -from sahara.utils import files as f - -c_h = config_helper.ConfigHelperV540() - -path_to_config = 'plugins/cdh/v5_4_0/resources/' -json_files = [ - 'hdfs-service.json', - 'hdfs-namenode.json', - 'hdfs-datanode.json', - 'hdfs-secondarynamenode.json', - 'hdfs-gateway.json', - 'hdfs-journalnode.json', - 'yarn-service.json', - 'yarn-resourcemanager.json', - 'yarn-nodemanager.json', - 'yarn-jobhistory.json', - 'yarn-gateway.json', - 'oozie-service.json', - 'oozie-oozie_server.json', - 'hive-service.json', - 'hive-hivemetastore.json', - 'hive-hiveserver2.json', - 'hive-webhcat.json', - 'hue-service.json', - 'hue-hue_server.json', - 'spark-service.json', - 'spark-spark_yarn_history_server.json', - 'zookeeper-service.json', - 'zookeeper-server.json', - 'hbase-service.json', - 'hbase-master.json', - 'hbase-regionserver.json', - 'flume-service.json', - 'flume-agent.json', - 'sentry-service.json', - 'sentry-sentry_server.json', - 'solr-service.json', - 'solr-solr_server.json', - 'sqoop-service.json', - 'sqoop-sqoop_server.json', - 'ks_indexer-service.json', - 'ks_indexer-hbase_indexer.json', - 'impala-service.json', - 'impala-catalogserver.json', - 'impala-impalad.json', - 'impala-statestore.json', - 'kms-service.json', - 'kms-kms.json' -] - - -class ConfigHelperTestCase(base.SaharaTestCase): - - def test_get_ng_plugin_configs(self): - actual_configs = c_h._get_ng_plugin_configs() - - expected_configs = [] - for json_file in json_files: - expected_configs += json.loads( - f.get_file_text(path_to_config + json_file)) - - # compare names - expected_names = set(i['name'] for i in expected_configs) - actual_names = set(i.to_dict()['name'] for i in actual_configs) - self.assertEqual(expected_names, actual_names) - - def test_get_cdh5_repo_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.CDH5_REPO_URL.default_value, - c_h.get_cdh5_repo_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.CDH5_REPO_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_cdh5_repo_url(cluster)) - - def test_get_cdh5_key_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.CDH5_REPO_KEY_URL.default_value, - c_h.get_cdh5_key_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.CDH5_REPO_KEY_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_cdh5_key_url(cluster)) - - def test_get_cm5_repo_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.CM5_REPO_URL.default_value, - c_h.get_cm5_repo_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.CM5_REPO_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_cm5_repo_url(cluster)) - - def test_get_cm5_key_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.CM5_REPO_KEY_URL.default_value, - c_h.get_cm5_key_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.CM5_REPO_KEY_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_cm5_key_url(cluster)) - - def test_is_swift_enabled(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertTrue(c_h.is_swift_enabled(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.ENABLE_SWIFT.name: False}}) - self.assertFalse(c_h.is_swift_enabled(cluster)) - - def test_get_swift_lib_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.DEFAULT_SWIFT_LIB_URL, - c_h.get_swift_lib_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.SWIFT_LIB_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_swift_lib_url(cluster)) - - def test_is_hbase_common_lib_enabled(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertTrue(c_h.is_hbase_common_lib_enabled(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': - {c_h.ENABLE_HBASE_COMMON_LIB.name: False}}) - self.assertFalse(c_h.is_hbase_common_lib_enabled(cluster)) - - def test_get_extjs_lib_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.DEFAULT_EXTJS_LIB_URL, - c_h.get_extjs_lib_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.EXTJS_LIB_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_extjs_lib_url(cluster)) - - def test_get_kms_key_url(self): - cluster = ctu.get_fake_cluster(cluster_configs={}) - self.assertEqual(c_h.KMS_REPO_KEY_URL.default_value, - c_h.get_kms_key_url(cluster)) - - cluster = ctu.get_fake_cluster( - cluster_configs={'general': {c_h.KMS_REPO_KEY_URL.name: 'spam'}}) - self.assertEqual('spam', c_h.get_kms_key_url(cluster)) diff --git a/sahara/tests/unit/plugins/cdh/v5_4_0/test_edp_engine.py b/sahara/tests/unit/plugins/cdh/v5_4_0/test_edp_engine.py deleted file mode 100644 index a21e8891..00000000 --- a/sahara/tests/unit/plugins/cdh/v5_4_0/test_edp_engine.py +++ /dev/null @@ -1,200 +0,0 @@ -# Copyright (c) 2015 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock - -from sahara import exceptions as ex -from sahara.plugins import base as pb -from sahara.plugins.cdh.v5_4_0 import edp_engine -from sahara.plugins import exceptions as pl_ex -from sahara.tests.unit import base as sahara_base -from sahara.tests.unit.plugins.cdh import utils as c_u -from sahara.utils import edp - - -def get_cluster(version='5.4.0'): - cluster = c_u.get_fake_cluster(plugin_name='CDH', hadoop_version=version) - return cluster - - -class EdpEngineTestV540(sahara_base.SaharaTestCase): - - def setUp(self): - super(EdpEngineTestV540, self).setUp() - pb.setup_plugins() - - def test_get_hdfs_user(self): - eng = edp_engine.EdpOozieEngine(get_cluster()) - self.assertEqual('hdfs', eng.get_hdfs_user()) - - @mock.patch('sahara.service.edp.hdfs_helper.create_dir_hadoop2') - def test_create_hdfs_dir(self, create_dir_hadoop2): - eng = edp_engine.EdpOozieEngine(get_cluster()) - remote = mock.Mock() - dir_name = mock.Mock() - eng.create_hdfs_dir(remote, dir_name) - create_dir_hadoop2.assert_called_once_with(remote, - dir_name, - eng.get_hdfs_user()) - - def test_get_oozie_server_uri(self): - cluster = get_cluster() - eng = edp_engine.EdpOozieEngine(cluster) - uri = eng.get_oozie_server_uri(cluster) - self.assertEqual("http://1.2.3.5:11000/oozie", uri) - - def test_get_name_node_uri(self): - cluster = get_cluster() - eng = edp_engine.EdpOozieEngine(cluster) - uri = eng.get_name_node_uri(cluster) - self.assertEqual("hdfs://master_inst.novalocal:8020", uri) - - # has HDFS_JOURNALNODE - cluster = get_cluster() - jns_node_group = mock.MagicMock() - jns_node_group.node_processes = ['HDFS_JOURNALNODE'] - jns_node_group.instances = [mock.Mock()] - list.append(cluster.node_groups, jns_node_group) - uri = eng.get_name_node_uri(cluster) - self.assertEqual("hdfs://nameservice01", uri) - - def test_get_resource_manager_uri(self): - cluster = get_cluster() - eng = edp_engine.EdpOozieEngine(cluster) - uri = eng.get_resource_manager_uri(cluster) - self.assertEqual("master_inst.novalocal:8032", uri) - - def test_get_oozie_server(self): - cluster = get_cluster() - eng = edp_engine.EdpOozieEngine(cluster) - actual = eng.get_oozie_server(cluster) - expect = cluster.node_groups[1].instances[0] - self.assertEqual(expect, actual) - - @mock.patch('sahara.service.edp.oozie.engine.' - 'OozieJobEngine.validate_job_execution') - def test_validate_job_execution(self, c): - cluster = get_cluster() - eng = edp_engine.EdpOozieEngine(cluster) - eng.validate_job_execution(cluster, mock.Mock(), mock.Mock()) - - # more than one oozie server - dict.__setitem__(cluster.node_groups[1], 'count', 2) - self.assertRaises(pl_ex.InvalidComponentCountException, - eng.validate_job_execution, cluster, - mock.Mock(), mock.Mock()) - - @mock.patch( - 'sahara.plugins.cdh.confighints_helper.get_possible_hive_config_from', - return_value={}) - def test_get_possible_job_config_hive(self, - get_possible_hive_config_from): - expected_config = {'job_config': {}} - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_HIVE) - get_possible_hive_config_from.assert_called_once_with( - 'plugins/cdh/v5_4_0/resources/hive-site.xml') - self.assertEqual(expected_config, actual_config) - - @mock.patch('sahara.plugins.cdh.v5_4_0.edp_engine.EdpOozieEngine') - def test_get_possible_job_config_java(self, BaseCDHEdpOozieEngine): - expected_config = {'job_config': {}} - BaseCDHEdpOozieEngine.get_possible_job_config.return_value = ( - expected_config) - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_JAVA) - BaseCDHEdpOozieEngine.get_possible_job_config.assert_called_once_with( - edp.JOB_TYPE_JAVA) - self.assertEqual(expected_config, actual_config) - - @mock.patch( - 'sahara.plugins.cdh.confighints_helper.' - 'get_possible_mapreduce_config_from', - return_value={}) - def test_get_possible_job_config_mapreduce( - self, get_possible_mapreduce_config_from): - expected_config = {'job_config': {}} - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_MAPREDUCE) - get_possible_mapreduce_config_from.assert_called_once_with( - 'plugins/cdh/v5_4_0/resources/mapred-site.xml') - self.assertEqual(expected_config, actual_config) - - @mock.patch( - 'sahara.plugins.cdh.confighints_helper.' - 'get_possible_mapreduce_config_from', - return_value={}) - def test_get_possible_job_config_mapreduce_streaming( - self, get_possible_mapreduce_config_from): - expected_config = {'job_config': {}} - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_MAPREDUCE_STREAMING) - get_possible_mapreduce_config_from.assert_called_once_with( - 'plugins/cdh/v5_4_0/resources/mapred-site.xml') - self.assertEqual(expected_config, actual_config) - - @mock.patch( - 'sahara.plugins.cdh.confighints_helper.get_possible_pig_config_from', - return_value={}) - def test_get_possible_job_config_pig(self, - get_possible_pig_config_from): - expected_config = {'job_config': {}} - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_PIG) - get_possible_pig_config_from.assert_called_once_with( - 'plugins/cdh/v5_4_0/resources/mapred-site.xml') - self.assertEqual(expected_config, actual_config) - - @mock.patch('sahara.plugins.cdh.v5_4_0.edp_engine.EdpOozieEngine') - def test_get_possible_job_config_shell(self, BaseCDHEdpOozieEngine): - expected_config = {'job_config': {}} - BaseCDHEdpOozieEngine.get_possible_job_config.return_value = ( - expected_config) - actual_config = edp_engine.EdpOozieEngine.get_possible_job_config( - edp.JOB_TYPE_SHELL) - BaseCDHEdpOozieEngine.get_possible_job_config.assert_called_once_with( - edp.JOB_TYPE_SHELL) - self.assertEqual(expected_config, actual_config) - - @mock.patch('sahara.plugins.utils.get_config_value_or_default') - @mock.patch('sahara.plugins.utils.get_instance') - @mock.patch('sahara.service.edp.spark.engine.' - 'SparkJobEngine.validate_job_execution') - def test_spark_engine_validate_job_execution(self, - validate_job_execution, - get_instance, - get_config_value_or_default): - # version unsupported - cluster = get_cluster(version='5.3.0') - eng = edp_engine.EdpSparkEngine(cluster) - self.assertRaises(ex.InvalidDataException, - eng.validate_job_execution, cluster, - mock.Mock(), mock.Mock()) - - # none yarn history server - cluster = get_cluster() - eng = edp_engine.EdpSparkEngine(cluster) - self.assertRaises(pl_ex.InvalidComponentCountException, - eng.validate_job_execution, cluster, - mock.Mock(), mock.Mock()) - - # valid - cluster = get_cluster() - yarn_history_node_group = mock.Mock() - yarn_history_node_group.node_processes = ['SPARK_YARN_HISTORY_SERVER'] - yarn_history_node_group.count = 1 - list.append(cluster.node_groups, yarn_history_node_group) - eng = edp_engine.EdpSparkEngine(cluster) - eng.validate_job_execution(cluster, mock.Mock(), mock.Mock()) diff --git a/sahara/tests/unit/plugins/cdh/v5_4_0/test_plugin_utils_540.py b/sahara/tests/unit/plugins/cdh/v5_4_0/test_plugin_utils_540.py deleted file mode 100644 index be38600b..00000000 --- a/sahara/tests/unit/plugins/cdh/v5_4_0/test_plugin_utils_540.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2015 Intel Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.plugins.cdh.v5_4_0 import plugin_utils as pu -from sahara.tests.unit.plugins.cdh import base_plugin_utils_test - - -class TestPluginUtilsV540(base_plugin_utils_test.TestPluginUtilsHigherThanV5): - - def setUp(self): - super(TestPluginUtilsV540, self).setUp() - self.plug_utils = pu.PluginUtilsV540() - self.version = "v5_4_0" diff --git a/sahara/tests/unit/plugins/cdh/v5_4_0/test_spark_engine.py b/sahara/tests/unit/plugins/cdh/v5_4_0/test_spark_engine.py deleted file mode 100644 index efab241e..00000000 --- a/sahara/tests/unit/plugins/cdh/v5_4_0/test_spark_engine.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (c) 2015 ISPRAS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock -import testtools - -from sahara import conductor as cond -from sahara import context -from sahara.plugins import base as pb -from sahara.plugins import exceptions as ex -from sahara.service.edp.spark import engine -from sahara.tests.unit import base -from sahara.utils import edp - - -conductor = cond.API - - -class SparkPluginTest(base.SaharaWithDbTestCase): - def setUp(self): - super(SparkPluginTest, self).setUp() - self.override_config("plugins", ["cdh"]) - pb.setup_plugins() - - def test_plugin_edp_engine_no_spark(self): - cluster_dict = { - 'name': 'cluster', - 'plugin_name': 'cdh', - 'hadoop_version': '5.4.0', - 'default_image_id': 'image'} - - job = mock.Mock() - job.type = edp.JOB_TYPE_SPARK - - cluster = conductor.cluster_create(context.ctx(), cluster_dict) - plugin = pb.PLUGINS.get_plugin(cluster.plugin_name) - edp_engine = plugin.get_edp_engine(cluster, edp.JOB_TYPE_SPARK) - with testtools.ExpectedException( - ex.InvalidComponentCountException, - value_re="Hadoop cluster should contain 1 " - "SPARK_YARN_HISTORY_SERVER component\(s\). Actual " - "SPARK_YARN_HISTORY_SERVER count is 0\nError ID: .*"): - - edp_engine.validate_job_execution(cluster, job, mock.Mock()) - - def test_plugin_edp_engine(self): - cluster_dict = { - 'name': 'cluster', - 'plugin_name': 'cdh', - 'hadoop_version': '5.4.0', - 'default_image_id': 'image'} - - cluster = conductor.cluster_create(context.ctx(), cluster_dict) - plugin = pb.PLUGINS.get_plugin(cluster.plugin_name) - self.assertIsInstance( - plugin.get_edp_engine(cluster, edp.JOB_TYPE_SPARK), - engine.SparkJobEngine) diff --git a/sahara/tests/unit/plugins/cdh/v5_4_0/test_validation.py b/sahara/tests/unit/plugins/cdh/v5_4_0/test_validation.py deleted file mode 100644 index 63e85dc5..00000000 --- a/sahara/tests/unit/plugins/cdh/v5_4_0/test_validation.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (c) 2015 Intel Corpration -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.plugins.cdh.v5_4_0 import validation -from sahara.plugins import exceptions as ex -from sahara.tests.unit.plugins.cdh import base_validation_tests as bvt - -icte = ex.InvalidClusterTopology -nnce = ex.NameNodeHAConfigurationError -icce = ex.InvalidComponentCountException -rsme = ex.RequiredServiceMissingException -rmce = ex.ResourceManagerHAConfigurationError - - -class ValidationTestCase(bvt.BaseValidationTestCase): - - def setUp(self): - super(ValidationTestCase, self).setUp() - self.module = validation.ValidatorV540 - - def _get_test_cases(self): - cases = super(ValidationTestCase, self)._get_test_cases() - - disable_anti_affinity = {'cluster_configs': {'general': { - 'Require Anti Affinity': False}}} - cases += [ - [None, {'HDFS_JOURNALNODE': 3, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [rsme, {'HDFS_JOURNALNODE': 3, - 'ZOOKEEPER_SERVER': 0}, [], disable_anti_affinity], - [icce, {'HDFS_JOURNALNODE': 4, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [icce, {'HDFS_JOURNALNODE': 2, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [nnce, {'HDFS_JOURNALNODE': 3, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['HDFS_SECONDARYNAMENODE']}], - [nnce, {'HDFS_JOURNALNODE': 3, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['HDFS_NAMENODE']}], - [None, {'HDFS_JOURNALNODE': 3, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['HDFS_NAMENODE', 'HDFS_SECONDARYNAMENODE']}], - [None, {'YARN_STANDBYRM': 1, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [icce, {'YARN_STANDBYRM': 2, - 'ZOOKEEPER_SERVER': 1}, [], disable_anti_affinity], - [rsme, {'YARN_STANDBYRM': 1, - 'ZOOKEEPER_SERVER': 0}, [], disable_anti_affinity], - [rmce, {'YARN_STANDBYRM': 1, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['YARN_RESOURCEMANAGER']}], - [rmce, {'YARN_STANDBYRM': 1, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['YARN_STANDBYRM']}], - [None, {'YARN_STANDBYRM': 1, 'ZOOKEEPER_SERVER': 1}, [], - {'anti_affinity': ['YARN_STANDBYRM', 'YARN_RESOURCEMANAGER']}], - ] - - cases += [ - [None, {'FLUME_AGENT': 1}], - [icce, {'ZOOKEEPER_SERVER': 1, 'SENTRY_SERVER': 2}], - [None, {'ZOOKEEPER_SERVER': 1, 'SENTRY_SERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 0, 'SENTRY_SERVER': 1}], - [None, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 0, 'SOLR_SERVER': 1}], - [None, {'YARN_NODEMANAGER': 1, 'YARN_JOBHISTORY': 1, - 'SQOOP_SERVER': 1}], - [rsme, {'YARN_NODEMANAGER': 0, 'YARN_JOBHISTORY': 1, - 'SQOOP_SERVER': 1}], - [rsme, {'YARN_NODEMANAGER': 1, 'YARN_JOBHISTORY': 0, - 'SQOOP_SERVER': 1}], - # HBASE_MASTER AND HBASE_REGIONSERVER depend circularly - [None, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1, - 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, - 'HBASE_REGIONSERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 0, 'SOLR_SERVER': 1, - 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, - 'HBASE_REGIONSERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 0, - 'HBASE_MASTER': 1, 'HBASE_INDEXER': 1, - 'HBASE_REGIONSERVER': 1}], - [rsme, {'ZOOKEEPER_SERVER': 1, 'SOLR_SERVER': 1, - 'HBASE_MASTER': 0, 'HBASE_INDEXER': 1}], - ] - - worker_with_implama = ('worker_ng', 1, ['HDFS_DATANODE', - 'YARN_NODEMANAGER', - 'IMPALAD'], 3) - cases += [ - [None, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1, - 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, - [worker_with_implama]], - [icte, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1}, - []], - [icte, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1}, - [worker_with_implama]], - [rsme, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 0, - 'HIVE_METASTORE': 1, 'HIVE_SERVER2': 1, - 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, - [worker_with_implama]], - [rsme, {'IMPALA_CATALOGSERVER': 1, 'IMPALA_STATESTORE': 1, - 'HIVE_METASTORE': 0, 'HIVE_SERVER2': 1, - 'HDFS_DATANODE': 0, 'YARN_NODEMANAGER': 0}, - [worker_with_implama]] - - ] - - cases += [ - [None, {'KMS': 1}], - [icce, {'KMS': 2}] - ] - return cases diff --git a/sahara/tests/unit/plugins/cdh/v5_4_0/test_versionhandler.py b/sahara/tests/unit/plugins/cdh/v5_4_0/test_versionhandler.py deleted file mode 100644 index 3e0c9589..00000000 --- a/sahara/tests/unit/plugins/cdh/v5_4_0/test_versionhandler.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright (c) 2015 Intel Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock -import six - -from sahara.plugins.cdh.v5_4_0 import edp_engine -from sahara.plugins.cdh.v5_4_0 import versionhandler -from sahara.tests.unit import base - - -class VersionHandlerTestCase(base.SaharaTestCase): - - plugin_path = "sahara.plugins.cdh.v5_4_0." - cloudera_utils_path = plugin_path + "cloudera_utils.ClouderaUtilsV540." - plugin_utils_path = plugin_path + "plugin_utils.PluginUtilsV540." - - def setUp(self): - super(VersionHandlerTestCase, self).setUp() - self.vh = versionhandler.VersionHandler() - - def test_get_node_processes(self): - processes = self.vh.get_node_processes() - for k, v in six.iteritems(processes): - for p in v: - self.assertIsInstance(p, str) - - @mock.patch("sahara.conductor.API.cluster_update") - @mock.patch("sahara.context.ctx") - @mock.patch(plugin_path + "deploy.configure_cluster") - @mock.patch(cloudera_utils_path + "get_cloudera_manager_info", - return_value={"fake_cm_info": "fake"}) - def test_config_cluster(self, get_cm_info, configure_cluster, - ctx, cluster_update): - cluster = mock.Mock() - self.vh.configure_cluster(cluster) - configure_cluster.assert_called_once_with(cluster) - cluster_update.assert_called_once_with( - ctx(), cluster, - {'info': {"fake_cm_info": "fake"}}) - - @mock.patch(plugin_path + "deploy.start_cluster") - def test_start_cluster(self, start_cluster): - cluster = mock.Mock() - self.vh._set_cluster_info = mock.Mock() - self.vh.start_cluster(cluster) - start_cluster.assert_called_once_with(cluster) - self.vh._set_cluster_info.assert_called_once_with(cluster) - - @mock.patch(plugin_path + "deploy.decommission_cluster") - def test_decommission_nodes(self, decommission_cluster): - cluster = mock.Mock() - instances = mock.Mock() - self.vh.decommission_nodes(cluster, instances) - decommission_cluster.assert_called_once_with(cluster, - instances) - - @mock.patch(plugin_path + "deploy.scale_cluster") - def test_scale_cluster(self, scale_cluster): - cluster = mock.Mock() - instances = mock.Mock() - self.vh.scale_cluster(cluster, instances) - scale_cluster.assert_called_once_with(cluster, instances) - - @mock.patch("sahara.conductor.API.cluster_update") - @mock.patch("sahara.context.ctx") - @mock.patch(cloudera_utils_path + "get_cloudera_manager_info", - return_value={}) - @mock.patch(plugin_utils_path + "get_hue") - def test_set_cluster_info(self, get_hue, get_cloudera_manager_info, - ctx, cluster_update): - hue = mock.Mock() - hue.get_ip_or_dns_name.return_value = "1.2.3.4" - get_hue.return_value = hue - cluster = mock.Mock() - self.vh._set_cluster_info(cluster) - info = {'info': {'Hue Dashboard': {'Web UI': 'http://1.2.3.4:8888'}}} - cluster_update.assert_called_once_with(ctx(), cluster, info) - - @mock.patch("sahara.plugins.utils.get_instance") - @mock.patch("sahara.plugins.utils.get_config_value_or_default") - @mock.patch("sahara.service.edp.job_utils.get_plugin") - def test_get_edp_engine(self, get_plugin, get_config_value_or_default, - get_instance): - cluster = mock.Mock() - job_type = 'Java' - ret = self.vh.get_edp_engine(cluster, job_type) - self.assertIsInstance(ret, edp_engine.EdpOozieEngine) - - job_type = 'Spark' - ret = self.vh.get_edp_engine(cluster, job_type) - self.assertIsInstance(ret, edp_engine.EdpSparkEngine) - - job_type = 'unsupported' - ret = self.vh.get_edp_engine(cluster, job_type) - self.assertIsNone(ret) - - def test_get_edp_job_types(self): - ret = self.vh.get_edp_job_types() - expect = edp_engine.EdpOozieEngine.get_supported_job_types() + \ - edp_engine.EdpSparkEngine.get_supported_job_types() - self.assertEqual(expect, ret) - - @mock.patch(plugin_path + - "edp_engine.EdpOozieEngine.get_possible_job_config", - return_value={'job_config': {}}) - def test_edp_config_hints(self, get_possible_job_config): - job_type = mock.Mock() - ret = self.vh.get_edp_config_hints(job_type) - get_possible_job_config.assert_called_once_with(job_type) - self.assertEqual(ret, {'job_config': {}}) - - @mock.patch(plugin_path + "deploy.get_open_ports", return_value=[1234]) - def test_get_open_ports(self, get_open_ports): - node_group = mock.Mock() - ret = self.vh.get_open_ports(node_group) - get_open_ports.assert_called_once_with(node_group) - self.assertEqual(ret, [1234]) - - @mock.patch(plugin_utils_path + "recommend_configs") - def test_recommend_configs(self, recommend_configs): - cluster = mock.Mock() - scaling = mock.Mock() - self.vh.get_plugin_configs = mock.Mock() - self.vh.recommend_configs(cluster, scaling) - recommend_configs.assert_called_once_with(cluster, - self.vh.get_plugin_configs(), - scaling) diff --git a/sahara/tests/unit/service/edp/spark/test_spark_cloudera_v5_3_0.py b/sahara/tests/unit/service/edp/spark/test_spark_cloudera_v5_3_0.py deleted file mode 100644 index 7d17f427..00000000 --- a/sahara/tests/unit/service/edp/spark/test_spark_cloudera_v5_3_0.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) 2015 ISPRAS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.plugins.cdh.v5_3_0 import edp_engine -from sahara.tests.unit.service.edp.spark import base as tests - - -class TestClouderaPlugin(tests.TestSpark): - def setUp(self): - super(TestClouderaPlugin, self).setUp() - self.master_host = "SPARK_YARN_HISTORY_SERVER" - self.engine_class = edp_engine.EdpSparkEngine - self.spark_user = "sudo -u spark " - self.spark_submit = "spark-submit" - self.master = "yarn-cluster" - self.deploy_mode = "cluster" diff --git a/sahara/tests/unit/service/edp/spark/test_spark_cloudera_v5_4_0.py b/sahara/tests/unit/service/edp/spark/test_spark_cloudera_v5_4_0.py deleted file mode 100644 index bcca7bb3..00000000 --- a/sahara/tests/unit/service/edp/spark/test_spark_cloudera_v5_4_0.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) 2015 ISPRAS -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from sahara.plugins.cdh.v5_4_0 import edp_engine -from sahara.tests.unit.service.edp.spark import base as tests - - -class TestClouderaPlugin(tests.TestSpark): - def setUp(self): - super(TestClouderaPlugin, self).setUp() - self.master_host = "SPARK_YARN_HISTORY_SERVER" - self.engine_class = edp_engine.EdpSparkEngine - self.spark_user = "sudo -u spark " - self.spark_submit = "spark-submit" - self.master = "yarn-cluster" - self.deploy_mode = "cluster"