diff --git a/MANIFEST.in b/MANIFEST.in index 57e76f5f4c..ed34d3e9f4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -20,6 +20,7 @@ include sahara/plugins/vanilla/v1_2_1/resources/*.sh include sahara/plugins/vanilla/v1_2_1/resources/*.sql include sahara/plugins/vanilla/v1_2_1/resources/*.xml include sahara/plugins/vanilla/v2_4_1/resources/*.xml +include sahara/plugins/vanilla/v2_6_0/resources/*.xml include sahara/plugins/hdp/versions/version_1_3_2/resources/*.template include sahara/plugins/hdp/versions/version_1_3_2/resources/*.json include sahara/plugins/hdp/versions/version_1_3_2/resources/*.sh diff --git a/sahara/plugins/vanilla/hadoop2/run_scripts.py b/sahara/plugins/vanilla/hadoop2/run_scripts.py index a83e494daf..3929d67ae0 100644 --- a/sahara/plugins/vanilla/hadoop2/run_scripts.py +++ b/sahara/plugins/vanilla/hadoop2/run_scripts.py @@ -170,9 +170,9 @@ def _check_datanodes_count(remote, count): LOG.debug("Checking datanode count") exit_code, stdout = remote.execute_command( - 'sudo su -lc "hadoop dfsadmin -report" hadoop | ' - 'grep \'Datanodes available:\' | ' - 'awk \'{print $3}\'') + 'sudo su -lc "hdfs dfsadmin -report" hadoop | ' + 'grep \'Live datanodes\|Datanodes available:\' | ' + 'grep -o \'[0-9]\+\' | head -n 1') LOG.debug("Datanode count='%s'" % stdout.rstrip()) return exit_code == 0 and stdout and int(stdout) == count diff --git a/sahara/plugins/vanilla/v2_6_0/__init__.py b/sahara/plugins/vanilla/v2_6_0/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sahara/plugins/vanilla/v2_6_0/config_helper.py b/sahara/plugins/vanilla/v2_6_0/config_helper.py new file mode 100644 index 0000000000..9f77d9e98d --- /dev/null +++ b/sahara/plugins/vanilla/v2_6_0/config_helper.py @@ -0,0 +1,94 @@ +# Copyright (c) 2014 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo.config import cfg + +from sahara.openstack.common import log as logging +from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper +from sahara.utils import xmlutils as x + +CONF = cfg.CONF +CONF.import_opt("enable_data_locality", "sahara.topology.topology_helper") + +LOG = logging.getLogger(__name__) + +CORE_DEFAULT = x.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_6_0/resources/core-default.xml') + +HDFS_DEFAULT = x.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_6_0/resources/hdfs-default.xml') + +MAPRED_DEFAULT = x.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_6_0/resources/mapred-default.xml') + +YARN_DEFAULT = x.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_6_0/resources/yarn-default.xml') + +OOZIE_DEFAULT = x.load_hadoop_xml_defaults( + 'plugins/vanilla/v2_6_0/resources/oozie-default.xml') + +XML_CONFS = { + "Hadoop": [CORE_DEFAULT], + "HDFS": [HDFS_DEFAULT], + "YARN": [YARN_DEFAULT], + "MapReduce": [MAPRED_DEFAULT], + "JobFlow": [OOZIE_DEFAULT] +} + +ENV_CONFS = { + "YARN": { + 'ResourceManager Heap Size': 1024, + 'NodeManager Heap Size': 1024 + }, + "HDFS": { + 'NameNode Heap Size': 1024, + 'SecondaryNameNode Heap Size': 1024, + 'DataNode Heap Size': 1024 + }, + "MapReduce": { + 'JobHistoryServer Heap Size': 1024 + }, + "JobFlow": { + 'Oozie Heap Size': 1024 + } +} + + +# Initialise plugin Hadoop configurations +PLUGIN_XML_CONFIGS = c_helper.init_xml_configs(XML_CONFS) +PLUGIN_ENV_CONFIGS = c_helper.init_env_configs(ENV_CONFS) + + +def _init_all_configs(): + configs = [] + configs.extend(PLUGIN_XML_CONFIGS) + configs.extend(PLUGIN_ENV_CONFIGS) + configs.extend(c_helper.PLUGIN_GENERAL_CONFIGS) + return configs + + +PLUGIN_CONFIGS = _init_all_configs() + + +def get_plugin_configs(): + return PLUGIN_CONFIGS + + +def get_xml_configs(): + return PLUGIN_XML_CONFIGS + + +def get_env_configs(): + return ENV_CONFS diff --git a/sahara/plugins/vanilla/v2_6_0/resources/README.rst b/sahara/plugins/vanilla/v2_6_0/resources/README.rst new file mode 100644 index 0000000000..3953c20277 --- /dev/null +++ b/sahara/plugins/vanilla/v2_6_0/resources/README.rst @@ -0,0 +1,26 @@ +Apache Hadoop Configurations for Sahara +======================================== + +This directory contains default XML configuration files: + +* core-default.xml +* hdfs-default.xml +* mapred-default.xml +* yarn-default.xml +* oozie-default.xml + +These files are applied for Sahara's plugin of Apache Hadoop version 2.6.0 + + +Files were taken from here: + +* `core-default.xml `_ +* `hdfs-default.xml `_ +* `yarn-default.xml `_ +* `mapred-default.xml `_ +* `oozie-default.xml `_ + +XML configs are used to expose default Hadoop configurations to the users +through Sahara's REST API. It allows users to override some config values which +will be pushed to the provisioned VMs running Hadoop services as part of +appropriate xml config. diff --git a/sahara/plugins/vanilla/v2_6_0/resources/core-default.xml b/sahara/plugins/vanilla/v2_6_0/resources/core-default.xml new file mode 100644 index 0000000000..4ba2e5ad61 --- /dev/null +++ b/sahara/plugins/vanilla/v2_6_0/resources/core-default.xml @@ -0,0 +1,1649 @@ + + + + + + + + + + + + + + + hadoop.common.configuration.version + 0.23.0 + version of this configuration file + + + + hadoop.tmp.dir + /tmp/hadoop-${user.name} + A base for other temporary directories. + + + + io.native.lib.available + true + Should native hadoop libraries, if present, be used. + + + + hadoop.http.filter.initializers + org.apache.hadoop.http.lib.StaticUserWebFilter + A comma separated list of class names. Each class in the list + must extend org.apache.hadoop.http.FilterInitializer. The corresponding + Filter will be initialized. Then, the Filter will be applied to all user + facing jsp and servlet web pages. The ordering of the list defines the + ordering of the filters. + + + + + + hadoop.security.authorization + false + Is service-level authorization enabled? + + + + hadoop.security.instrumentation.requires.admin + false + + Indicates if administrator ACLs are required to access + instrumentation servlets (JMX, METRICS, CONF, STACKS). + + + + + hadoop.security.authentication + simple + Possible values are simple (no authentication), and kerberos + + + + + hadoop.security.group.mapping + org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback + + Class for user to group mapping (get groups for a given user) for ACL. + The default implementation, + org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback, + will determine if the Java Native Interface (JNI) is available. If JNI is + available the implementation will use the API within hadoop to resolve a + list of groups for a user. If JNI is not available then the shell + implementation, ShellBasedUnixGroupsMapping, is used. This implementation + shells out to the Linux/Unix environment with the + bash -c groups command to resolve a list of groups for a user. + + + + + + + hadoop.security.groups.cache.secs + 300 + + This is the config controlling the validity of the entries in the cache + containing the user->group mapping. When this duration has expired, + then the implementation of the group mapping provider is invoked to get + the groups of the user and then cached back. + + + + + hadoop.security.groups.negative-cache.secs + 30 + + Expiration time for entries in the the negative user-to-group mapping + caching, in seconds. This is useful when invalid users are retrying + frequently. It is suggested to set a small value for this expiration, since + a transient error in group lookup could temporarily lock out a legitimate + user. + + Set this to zero or negative value to disable negative user-to-group caching. + + + + + hadoop.security.groups.cache.warn.after.ms + 5000 + + If looking up a single user to group takes longer than this amount of + milliseconds, we will log a warning message. + + + + + hadoop.security.group.mapping.ldap.url + + + The URL of the LDAP server to use for resolving user groups when using + the LdapGroupsMapping user to group mapping. + + + + + hadoop.security.group.mapping.ldap.ssl + false + + Whether or not to use SSL when connecting to the LDAP server. + + + + + hadoop.security.group.mapping.ldap.ssl.keystore + + + File path to the SSL keystore that contains the SSL certificate required + by the LDAP server. + + + + + hadoop.security.group.mapping.ldap.ssl.keystore.password.file + + + The path to a file containing the password of the LDAP SSL keystore. + + IMPORTANT: This file should be readable only by the Unix user running + the daemons. + + + + + hadoop.security.group.mapping.ldap.bind.user + + + The distinguished name of the user to bind as when connecting to the LDAP + server. This may be left blank if the LDAP server supports anonymous binds. + + + + + hadoop.security.group.mapping.ldap.bind.password.file + + + The path to a file containing the password of the bind user. + + IMPORTANT: This file should be readable only by the Unix user running + the daemons. + + + + + hadoop.security.group.mapping.ldap.base + + + The search base for the LDAP connection. This is a distinguished name, + and will typically be the root of the LDAP directory. + + + + + hadoop.security.group.mapping.ldap.search.filter.user + (&(objectClass=user)(sAMAccountName={0})) + + An additional filter to use when searching for LDAP users. The default will + usually be appropriate for Active Directory installations. If connecting to + an LDAP server with a non-AD schema, this should be replaced with + (&(objectClass=inetOrgPerson)(uid={0}). {0} is a special string used to + denote where the username fits into the filter. + + + + + hadoop.security.group.mapping.ldap.search.filter.group + (objectClass=group) + + An additional filter to use when searching for LDAP groups. This should be + changed when resolving groups against a non-Active Directory installation. + posixGroups are currently not a supported group class. + + + + + hadoop.security.group.mapping.ldap.search.attr.member + member + + The attribute of the group object that identifies the users that are + members of the group. The default will usually be appropriate for + any LDAP installation. + + + + + hadoop.security.group.mapping.ldap.search.attr.group.name + cn + + The attribute of the group object that identifies the group name. The + default will usually be appropriate for all LDAP systems. + + + + + hadoop.security.group.mapping.ldap.directory.search.timeout + 10000 + + The attribute applied to the LDAP SearchControl properties to set a + maximum time limit when searching and awaiting a result. + Set to 0 if infinite wait period is desired. + Default is 10 seconds. Units in milliseconds. + + + + + hadoop.security.service.user.name.key + + + For those cases where the same RPC protocol is implemented by multiple + servers, this configuration is required for specifying the principal + name to use for the service when the client wishes to make an RPC call. + + + + + + hadoop.security.uid.cache.secs + 14400 + + This is the config controlling the validity of the entries in the cache + containing the userId to userName and groupId to groupName used by + NativeIO getFstat(). + + + + + hadoop.rpc.protection + authentication + A comma-separated list of protection values for secured sasl + connections. Possible values are authentication, integrity and privacy. + authentication means authentication only and no integrity or privacy; + integrity implies authentication and integrity are enabled; and privacy + implies all of authentication, integrity and privacy are enabled. + hadoop.security.saslproperties.resolver.class can be used to override + the hadoop.rpc.protection for a connection at the server side. + + + + + hadoop.security.saslproperties.resolver.class + + SaslPropertiesResolver used to resolve the QOP used for a + connection. If not specified, the full set of values specified in + hadoop.rpc.protection is used while determining the QOP used for the + connection. If a class is specified, then the QOP values returned by + the class will be used while determining the QOP used for the connection. + + + + + hadoop.work.around.non.threadsafe.getpwuid + false + Some operating systems or authentication modules are known to + have broken implementations of getpwuid_r and getpwgid_r, such that these + calls are not thread-safe. Symptoms of this problem include JVM crashes + with a stack trace inside these functions. If your system exhibits this + issue, enable this configuration parameter to include a lock around the + calls as a workaround. + + An incomplete list of some systems known to have this issue is available + at http://wiki.apache.org/hadoop/KnownBrokenPwuidImplementations + + + + + hadoop.kerberos.kinit.command + kinit + Used to periodically renew Kerberos credentials when provided + to Hadoop. The default setting assumes that kinit is in the PATH of users + running the Hadoop client. Change this to the absolute path to kinit if this + is not the case. + + + + + hadoop.security.auth_to_local + + Maps kerberos principals to local user names + + + + + io.file.buffer.size + 4096 + The size of buffer for use in sequence files. + The size of this buffer should probably be a multiple of hardware + page size (4096 on Intel x86), and it determines how much data is + buffered during read and write operations. + + + + io.bytes.per.checksum + 512 + The number of bytes per checksum. Must not be larger than + io.file.buffer.size. + + + + io.skip.checksum.errors + false + If true, when a checksum error is encountered while + reading a sequence file, entries are skipped, instead of throwing an + exception. + + + + io.compression.codecs + + A comma-separated list of the compression codec classes that can + be used for compression/decompression. In addition to any classes specified + with this property (which take precedence), codec classes on the classpath + are discovered using a Java ServiceLoader. + + + + io.compression.codec.bzip2.library + system-native + The native-code library to be used for compression and + decompression by the bzip2 codec. This library could be specified + either by by name or the full pathname. In the former case, the + library is located by the dynamic linker, usually searching the + directories specified in the environment variable LD_LIBRARY_PATH. + + The value of "system-native" indicates that the default system + library should be used. To indicate that the algorithm should + operate entirely in Java, specify "java-builtin". + + + + io.serializations + org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization + A list of serialization classes that can be used for + obtaining serializers and deserializers. + + + + io.seqfile.local.dir + ${hadoop.tmp.dir}/io/local + The local directory where sequence file stores intermediate + data files during merge. May be a comma-separated list of + directories on different devices in order to spread disk i/o. + Directories that do not exist are ignored. + + + + + io.map.index.skip + 0 + Number of index entries to skip between each entry. + Zero by default. Setting this to values larger than zero can + facilitate opening large MapFiles using less memory. + + + + io.map.index.interval + 128 + + MapFile consist of two files - data file (tuples) and index file + (keys). For every io.map.index.interval records written in the + data file, an entry (record-key, data-file-position) is written + in the index file. This is to allow for doing binary search later + within the index file to look up records by their keys and get their + closest positions in the data file. + + + + + + + fs.defaultFS + file:/// + The name of the default file system. A URI whose + scheme and authority determine the FileSystem implementation. The + uri's scheme determines the config property (fs.SCHEME.impl) naming + the FileSystem implementation class. The uri's authority is used to + determine the host, port, etc. for a filesystem. + + + + fs.default.name + file:/// + Deprecated. Use (fs.defaultFS) property + instead + + + + fs.trash.interval + 0 + Number of minutes after which the checkpoint + gets deleted. If zero, the trash feature is disabled. + This option may be configured both on the server and the + client. If trash is disabled server side then the client + side configuration is checked. If trash is enabled on the + server side then the value configured on the server is + used and the client configuration value is ignored. + + + + + fs.trash.checkpoint.interval + 0 + Number of minutes between trash checkpoints. + Should be smaller or equal to fs.trash.interval. If zero, + the value is set to the value of fs.trash.interval. + Every time the checkpointer runs it creates a new checkpoint + out of current and removes checkpoints created more than + fs.trash.interval minutes ago. + + + + + fs.AbstractFileSystem.file.impl + org.apache.hadoop.fs.local.LocalFs + The AbstractFileSystem for file: uris. + + + + fs.AbstractFileSystem.har.impl + org.apache.hadoop.fs.HarFs + The AbstractFileSystem for har: uris. + + + + fs.AbstractFileSystem.hdfs.impl + org.apache.hadoop.fs.Hdfs + The FileSystem for hdfs: uris. + + + + fs.AbstractFileSystem.viewfs.impl + org.apache.hadoop.fs.viewfs.ViewFs + The AbstractFileSystem for view file system for viewfs: uris + (ie client side mount table:). + + + + fs.ftp.host + 0.0.0.0 + FTP filesystem connects to this server + + + + fs.ftp.host.port + 21 + + FTP filesystem connects to fs.ftp.host on this port + + + + + fs.df.interval + 60000 + Disk usage statistics refresh interval in msec. + + + + fs.du.interval + 600000 + File space usage statistics refresh interval in msec. + + + + fs.s3.block.size + 67108864 + Block size to use when writing files to S3. + + + + fs.s3.buffer.dir + ${hadoop.tmp.dir}/s3 + Determines where on the local filesystem the S3 filesystem + should store files before sending them to S3 + (or after retrieving them from S3). + + + + + fs.s3.maxRetries + 4 + The maximum number of retries for reading or writing files to S3, + before we signal failure to the application. + + + + + fs.s3.sleepTimeSeconds + 10 + The number of seconds to sleep between each S3 retry. + + + + + fs.swift.impl + org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem + The implementation class of the OpenStack Swift Filesystem + + + + fs.automatic.close + true + By default, FileSystem instances are automatically closed at program + exit using a JVM shutdown hook. Setting this property to false disables this + behavior. This is an advanced option that should only be used by server applications + requiring a more carefully orchestrated shutdown sequence. + + + + + fs.s3n.block.size + 67108864 + Block size to use when reading files using the native S3 + filesystem (s3n: URIs). + + + + fs.s3n.multipart.uploads.enabled + false + Setting this property to true enables multiple uploads to + native S3 filesystem. When uploading a file, it is split into blocks + if the size is larger than fs.s3n.multipart.uploads.block.size. + + + + + fs.s3n.multipart.uploads.block.size + 67108864 + The block size for multipart uploads to native S3 filesystem. + Default size is 64MB. + + + + + fs.s3n.multipart.copy.block.size + 5368709120 + The block size for multipart copy in native S3 filesystem. + Default size is 5GB. + + + + + fs.s3n.server-side-encryption-algorithm + + Specify a server-side encryption algorithm for S3. + The default is NULL, and the only other currently allowable value is AES256. + + + + + fs.s3a.access.key + AWS access key ID. Omit for Role-based authentication. + + + + fs.s3a.secret.key + AWS secret key. Omit for Role-based authentication. + + + + fs.s3a.connection.maximum + 15 + Controls the maximum number of simultaneous connections to S3. + + + + fs.s3a.connection.ssl.enabled + true + Enables or disables SSL connections to S3. + + + + fs.s3a.attempts.maximum + 10 + How many times we should retry commands on transient errors. + + + + fs.s3a.connection.timeout + 5000 + Socket connection timeout in seconds. + + + + fs.s3a.paging.maximum + 5000 + How many keys to request from S3 when doing + directory listings at a time. + + + + fs.s3a.multipart.size + 104857600 + How big (in bytes) to split upload or copy operations up into. + + + + fs.s3a.multipart.threshold + 2147483647 + Threshold before uploads or copies use parallel multipart operations. + + + + fs.s3a.acl.default + Set a canned ACL for newly created and copied objects. Value may be private, + public-read, public-read-write, authenticated-read, log-delivery-write, + bucket-owner-read, or bucket-owner-full-control. + + + + fs.s3a.multipart.purge + false + True if you want to purge existing multipart uploads that may not have been + completed/aborted correctly + + + + fs.s3a.multipart.purge.age + 86400 + Minimum age in seconds of multipart uploads to purge + + + + fs.s3a.buffer.dir + ${hadoop.tmp.dir}/s3a + Comma separated list of directories that will be used to buffer file + uploads to. + + + + fs.s3a.impl + org.apache.hadoop.fs.s3a.S3AFileSystem + The implementation class of the S3A Filesystem + + + + io.seqfile.compress.blocksize + 1000000 + The minimum block size for compression in block compressed + SequenceFiles. + + + + + io.seqfile.lazydecompress + true + Should values of block-compressed SequenceFiles be decompressed + only when necessary. + + + + + io.seqfile.sorter.recordlimit + 1000000 + The limit on number of records to be kept in memory in a spill + in SequenceFiles.Sorter + + + + + io.mapfile.bloom.size + 1048576 + The size of BloomFilter-s used in BloomMapFile. Each time this many + keys is appended the next BloomFilter will be created (inside a DynamicBloomFilter). + Larger values minimize the number of filters, which slightly increases the performance, + but may waste too much space if the total number of keys is usually much smaller + than this number. + + + + + io.mapfile.bloom.error.rate + 0.005 + The rate of false positives in BloomFilter-s used in BloomMapFile. + As this value decreases, the size of BloomFilter-s increases exponentially. This + value is the probability of encountering false positives (default is 0.5%). + + + + + hadoop.util.hash.type + murmur + The default implementation of Hash. Currently this can take one of the + two values: 'murmur' to select MurmurHash and 'jenkins' to select JenkinsHash. + + + + + + + + ipc.client.idlethreshold + 4000 + Defines the threshold number of connections after which + connections will be inspected for idleness. + + + + + ipc.client.kill.max + 10 + Defines the maximum number of clients to disconnect in one go. + + + + + ipc.client.connection.maxidletime + 10000 + The maximum time in msec after which a client will bring down the + connection to the server. + + + + + ipc.client.connect.max.retries + 10 + Indicates the number of retries a client will make to establish + a server connection. + + + + + ipc.client.connect.retry.interval + 1000 + Indicates the number of milliseconds a client will wait for + before retrying to establish a server connection. + + + + + ipc.client.connect.timeout + 20000 + Indicates the number of milliseconds a client will wait for the + socket to establish a server connection. + + + + + ipc.client.connect.max.retries.on.timeouts + 45 + Indicates the number of retries a client will make on socket timeout + to establish a server connection. + + + + + ipc.server.listen.queue.size + 128 + Indicates the length of the listen queue for servers accepting + client connections. + + + + + + + hadoop.security.impersonation.provider.class + + A class which implements ImpersonationProvider interface, used to + authorize whether one user can impersonate a specific user. + If not specified, the DefaultImpersonationProvider will be used. + If a class is specified, then that class will be used to determine + the impersonation capability. + + + + + hadoop.rpc.socket.factory.class.default + org.apache.hadoop.net.StandardSocketFactory + Default SocketFactory to use. This parameter is expected to be + formatted as "package.FactoryClassName". + + + + + hadoop.rpc.socket.factory.class.ClientProtocol + + SocketFactory to use to connect to a DFS. If null or empty, use + hadoop.rpc.socket.class.default. This socket factory is also used by + DFSClient to create sockets to DataNodes. + + + + + + + hadoop.socks.server + + Address (host:port) of the SOCKS server to be used by the + SocksSocketFactory. + + + + + + net.topology.node.switch.mapping.impl + org.apache.hadoop.net.ScriptBasedMapping + The default implementation of the DNSToSwitchMapping. It + invokes a script specified in net.topology.script.file.name to resolve + node names. If the value for net.topology.script.file.name is not set, the + default value of DEFAULT_RACK is returned for all node names. + + + + + net.topology.impl + org.apache.hadoop.net.NetworkTopology + The default implementation of NetworkTopology which is classic three layer one. + + + + + net.topology.script.file.name + + The script name that should be invoked to resolve DNS names to + NetworkTopology names. Example: the script would take host.foo.bar as an + argument, and return /rack1 as the output. + + + + + net.topology.script.number.args + 100 + The max number of args that the script configured with + net.topology.script.file.name should be run with. Each arg is an + IP address. + + + + + net.topology.table.file.name + + The file name for a topology file, which is used when the + net.topology.node.switch.mapping.impl property is set to + org.apache.hadoop.net.TableMapping. The file format is a two column text + file, with columns separated by whitespace. The first column is a DNS or + IP address and the second column specifies the rack where the address maps. + If no entry corresponding to a host in the cluster is found, then + /default-rack is assumed. + + + + + + file.stream-buffer-size + 4096 + The size of buffer to stream files. + The size of this buffer should probably be a multiple of hardware + page size (4096 on Intel x86), and it determines how much data is + buffered during read and write operations. + + + + file.bytes-per-checksum + 512 + The number of bytes per checksum. Must not be larger than + file.stream-buffer-size + + + + file.client-write-packet-size + 65536 + Packet size for clients to write + + + + file.blocksize + 67108864 + Block size + + + + file.replication + 1 + Replication factor + + + + + + s3.stream-buffer-size + 4096 + The size of buffer to stream files. + The size of this buffer should probably be a multiple of hardware + page size (4096 on Intel x86), and it determines how much data is + buffered during read and write operations. + + + + s3.bytes-per-checksum + 512 + The number of bytes per checksum. Must not be larger than + s3.stream-buffer-size + + + + s3.client-write-packet-size + 65536 + Packet size for clients to write + + + + s3.blocksize + 67108864 + Block size + + + + s3.replication + 3 + Replication factor + + + + + + s3native.stream-buffer-size + 4096 + The size of buffer to stream files. + The size of this buffer should probably be a multiple of hardware + page size (4096 on Intel x86), and it determines how much data is + buffered during read and write operations. + + + + s3native.bytes-per-checksum + 512 + The number of bytes per checksum. Must not be larger than + s3native.stream-buffer-size + + + + s3native.client-write-packet-size + 65536 + Packet size for clients to write + + + + s3native.blocksize + 67108864 + Block size + + + + s3native.replication + 3 + Replication factor + + + + + ftp.stream-buffer-size + 4096 + The size of buffer to stream files. + The size of this buffer should probably be a multiple of hardware + page size (4096 on Intel x86), and it determines how much data is + buffered during read and write operations. + + + + ftp.bytes-per-checksum + 512 + The number of bytes per checksum. Must not be larger than + ftp.stream-buffer-size + + + + ftp.client-write-packet-size + 65536 + Packet size for clients to write + + + + ftp.blocksize + 67108864 + Block size + + + + ftp.replication + 3 + Replication factor + + + + + + tfile.io.chunk.size + 1048576 + + Value chunk size in bytes. Default to + 1MB. Values of the length less than the chunk size is + guaranteed to have known value length in read time (See also + TFile.Reader.Scanner.Entry.isValueLengthKnown()). + + + + + tfile.fs.output.buffer.size + 262144 + + Buffer size used for FSDataOutputStream in bytes. + + + + + tfile.fs.input.buffer.size + 262144 + + Buffer size used for FSDataInputStream in bytes. + + + + + + + hadoop.http.authentication.type + simple + + Defines authentication used for Oozie HTTP endpoint. + Supported values are: simple | kerberos | #AUTHENTICATION_HANDLER_CLASSNAME# + + + + + hadoop.http.authentication.token.validity + 36000 + + Indicates how long (in seconds) an authentication token is valid before it has + to be renewed. + + + + + hadoop.http.authentication.signature.secret.file + ${user.home}/hadoop-http-auth-signature-secret + + The signature secret for signing the authentication tokens. + The same secret should be used for JT/NN/DN/TT configurations. + + + + + hadoop.http.authentication.cookie.domain + + + The domain to use for the HTTP cookie that stores the authentication token. + In order to authentiation to work correctly across all Hadoop nodes web-consoles + the domain must be correctly set. + IMPORTANT: when using IP addresses, browsers ignore cookies with domain settings. + For this setting to work properly all nodes in the cluster must be configured + to generate URLs with hostname.domain names on it. + + + + + hadoop.http.authentication.simple.anonymous.allowed + true + + Indicates if anonymous requests are allowed when using 'simple' authentication. + + + + + hadoop.http.authentication.kerberos.principal + HTTP/_HOST@LOCALHOST + + Indicates the Kerberos principal to be used for HTTP endpoint. + The principal MUST start with 'HTTP/' as per Kerberos HTTP SPNEGO specification. + + + + + hadoop.http.authentication.kerberos.keytab + ${user.home}/hadoop.keytab + + Location of the keytab file with the credentials for the principal. + Referring to the same keytab file Oozie uses for its Kerberos credentials for Hadoop. + + + + + dfs.ha.fencing.methods + + + List of fencing methods to use for service fencing. May contain + builtin methods (eg shell and sshfence) or user-defined method. + + + + + dfs.ha.fencing.ssh.connect-timeout + 30000 + + SSH connection timeout, in milliseconds, to use with the builtin + sshfence fencer. + + + + + dfs.ha.fencing.ssh.private-key-files + + + The SSH private key files to use with the builtin sshfence fencer. + + + + + + + + The user name to filter as, on static web filters + while rendering content. An example use is the HDFS + web UI (user to be used for browsing files). + + hadoop.http.staticuser.user + dr.who + + + + ha.zookeeper.quorum + + A list of ZooKeeper server addresses, separated by commas, that are + to be used by the ZKFailoverController in automatic failover. + + + + + ha.zookeeper.session-timeout.ms + 5000 + + The session timeout to use when the ZKFC connects to ZooKeeper. + Setting this value to a lower value implies that server crashes + will be detected more quickly, but risks triggering failover too + aggressively in the case of a transient error or network blip. + + + + + ha.zookeeper.parent-znode + /hadoop-ha + + The ZooKeeper znode under which the ZK failover controller stores + its information. Note that the nameservice ID is automatically + appended to this znode, so it is not normally necessary to + configure this, even in a federated environment. + + + + + ha.zookeeper.acl + world:anyone:rwcda + + A comma-separated list of ZooKeeper ACLs to apply to the znodes + used by automatic failover. These ACLs are specified in the same + format as used by the ZooKeeper CLI. + + If the ACL itself contains secrets, you may instead specify a + path to a file, prefixed with the '@' symbol, and the value of + this configuration will be loaded from within. + + + + + ha.zookeeper.auth + + + A comma-separated list of ZooKeeper authentications to add when + connecting to ZooKeeper. These are specified in the same format + as used by the "addauth" command in the ZK CLI. It is + important that the authentications specified here are sufficient + to access znodes with the ACL specified in ha.zookeeper.acl. + + If the auths contain secrets, you may instead specify a + path to a file, prefixed with the '@' symbol, and the value of + this configuration will be loaded from within. + + + + + + + hadoop.ssl.keystores.factory.class + org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory + + The keystores factory to use for retrieving certificates. + + + + + hadoop.ssl.require.client.cert + false + Whether client certificates are required + + + + hadoop.ssl.hostname.verifier + DEFAULT + + The hostname verifier to provide for HttpsURLConnections. + Valid values are: DEFAULT, STRICT, STRICT_I6, DEFAULT_AND_LOCALHOST and + ALLOW_ALL + + + + + hadoop.ssl.server.conf + ssl-server.xml + + Resource file from which ssl server keystore information will be extracted. + This file is looked up in the classpath, typically it should be in Hadoop + conf/ directory. + + + + + hadoop.ssl.client.conf + ssl-client.xml + + Resource file from which ssl client keystore information will be extracted + This file is looked up in the classpath, typically it should be in Hadoop + conf/ directory. + + + + + hadoop.ssl.enabled + false + + Deprecated. Use dfs.http.policy and yarn.http.policy instead. + + + + + hadoop.ssl.enabled.protocols + TLSv1 + + Protocols supported by the ssl. + + + + + hadoop.jetty.logs.serve.aliases + true + + Enable/Disable aliases serving from jetty + + + + + fs.permissions.umask-mode + 022 + + The umask used when creating files and directories. + Can be in octal or in symbolic. Examples are: + "022" (octal for u=rwx,g=r-x,o=r-x in symbolic), + or "u=rwx,g=rwx,o=" (symbolic for 007 in octal). + + + + + + + ha.health-monitor.connect-retry-interval.ms + 1000 + + How often to retry connecting to the service. + + + + + ha.health-monitor.check-interval.ms + 1000 + + How often to check the service. + + + + + ha.health-monitor.sleep-after-disconnect.ms + 1000 + + How long to sleep after an unexpected RPC error. + + + + + ha.health-monitor.rpc-timeout.ms + 45000 + + Timeout for the actual monitorHealth() calls. + + + + + ha.failover-controller.new-active.rpc-timeout.ms + 60000 + + Timeout that the FC waits for the new active to become active + + + + + ha.failover-controller.graceful-fence.rpc-timeout.ms + 5000 + + Timeout that the FC waits for the old active to go to standby + + + + + ha.failover-controller.graceful-fence.connection.retries + 1 + + FC connection retries for graceful fencing + + + + + ha.failover-controller.cli-check.rpc-timeout.ms + 20000 + + Timeout that the CLI (manual) FC waits for monitorHealth, getServiceState + + + + + ipc.client.fallback-to-simple-auth-allowed + false + + When a client is configured to attempt a secure connection, but attempts to + connect to an insecure server, that server may instruct the client to + switch to SASL SIMPLE (unsecure) authentication. This setting controls + whether or not the client will accept this instruction from the server. + When false (the default), the client will not allow the fallback to SIMPLE + authentication, and will abort the connection. + + + + + fs.client.resolve.remote.symlinks + true + + Whether to resolve symlinks when accessing a remote Hadoop filesystem. + Setting this to false causes an exception to be thrown upon encountering + a symlink. This setting does not apply to local filesystems, which + automatically resolve local symlinks. + + + + + nfs.exports.allowed.hosts + * rw + + By default, the export can be mounted by any client. The value string + contains machine name and access privilege, separated by whitespace + characters. The machine name format can be a single host, a Java regular + expression, or an IPv4 address. The access privilege uses rw or ro to + specify read/write or read-only access of the machines to exports. If the + access privilege is not provided, the default is read-only. Entries are separated by ";". + For example: "192.168.0.0/22 rw ; host.*\.example\.com ; host1.test.org ro;". + Only the NFS gateway needs to restart after this property is updated. + + + + + hadoop.user.group.static.mapping.overrides + dr.who=; + + Static mapping of user to groups. This will override the groups if + available in the system for the specified user. In otherwords, groups + look-up will not happen for these users, instead groups mapped in this + configuration will be used. + Mapping should be in this format. + user1=group1,group2;user2=;user3=group2; + Default, "dr.who=;" will consider "dr.who" as user without groups. + + + + + rpc.metrics.quantile.enable + false + + Setting this property to true and rpc.metrics.percentiles.intervals + to a comma-separated list of the granularity in seconds, the + 50/75/90/95/99th percentile latency for rpc queue/processing time in + milliseconds are added to rpc metrics. + + + + + rpc.metrics.percentiles.intervals + + + A comma-separated list of the granularity in seconds for the metrics which + describe the 50/75/90/95/99th percentile latency for rpc queue/processing + time. The metrics are outputted if rpc.metrics.quantile.enable is set to + true. + + + + + hadoop.security.crypto.codec.classes.EXAMPLECIPHERSUITE + + + The prefix for a given crypto codec, contains a comma-separated + list of implementation classes for a given crypto codec (eg EXAMPLECIPHERSUITE). + The first implementation will be used if available, others are fallbacks. + + + + + hadoop.security.crypto.codec.classes.aes.ctr.nopadding + org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec,org.apache.hadoop.crypto.JceAesCtrCryptoCodec + + Comma-separated list of crypto codec implementations for AES/CTR/NoPadding. + The first implementation will be used if available, others are fallbacks. + + + + + hadoop.security.crypto.cipher.suite + AES/CTR/NoPadding + + Cipher suite for crypto codec. + + + + + hadoop.security.crypto.jce.provider + + + The JCE provider name used in CryptoCodec. + + + + + hadoop.security.crypto.buffer.size + 8192 + + The buffer size used by CryptoInputStream and CryptoOutputStream. + + + + + hadoop.security.java.secure.random.algorithm + SHA1PRNG + + The java secure random algorithm. + + + + + hadoop.security.secure.random.impl + + + Implementation of secure random. + + + + + hadoop.security.random.device.file.path + /dev/urandom + + OS security random device file path. + + + + + fs.har.impl.disable.cache + true + Don't cache 'har' filesystem instances. + + + + + hadoop.security.kms.client.authentication.retry-count + 1 + + Number of time to retry connecting to KMS on authentication failure + + + + hadoop.security.kms.client.encrypted.key.cache.size + 500 + + Size of the EncryptedKeyVersion cache Queue for each key + + + + hadoop.security.kms.client.encrypted.key.cache.low-watermark + 0.3f + + If size of the EncryptedKeyVersion cache Queue falls below the + low watermark, this cache queue will be scheduled for a refill + + + + hadoop.security.kms.client.encrypted.key.cache.num.refill.threads + 2 + + Number of threads to use for refilling depleted EncryptedKeyVersion + cache Queues + + + + hadoop.security.kms.client.encrypted.key.cache.expiry + 43200000 + + Cache expiry time for a Key, after which the cache Queue for this + key will be dropped. Default = 12hrs + + + + diff --git a/sahara/plugins/vanilla/v2_6_0/resources/hdfs-default.xml b/sahara/plugins/vanilla/v2_6_0/resources/hdfs-default.xml new file mode 100644 index 0000000000..1678ee4985 --- /dev/null +++ b/sahara/plugins/vanilla/v2_6_0/resources/hdfs-default.xml @@ -0,0 +1,2227 @@ + + + + + + + + + + + + + hadoop.hdfs.configuration.version + 1 + version of this configuration file + + + + dfs.namenode.logging.level + info + + The logging level for dfs namenode. Other values are "dir" (trace + namespace mutations), "block" (trace block under/over replications + and block creations/deletions), or "all". + + + + + dfs.namenode.rpc-address + + + RPC address that handles all clients requests. In the case of HA/Federation where multiple namenodes exist, + the name service id is added to the name e.g. dfs.namenode.rpc-address.ns1 + dfs.namenode.rpc-address.EXAMPLENAMESERVICE + The value of this property will take the form of nn-host1:rpc-port. + + + + + dfs.namenode.rpc-bind-host + + + The actual address the RPC server will bind to. If this optional address is + set, it overrides only the hostname portion of dfs.namenode.rpc-address. + It can also be specified per name node or name service for HA/Federation. + This is useful for making the name node listen on all interfaces by + setting it to 0.0.0.0. + + + + + dfs.namenode.servicerpc-address + + + RPC address for HDFS Services communication. BackupNode, Datanodes and all other services should be + connecting to this address if it is configured. In the case of HA/Federation where multiple namenodes exist, + the name service id is added to the name e.g. dfs.namenode.servicerpc-address.ns1 + dfs.namenode.rpc-address.EXAMPLENAMESERVICE + The value of this property will take the form of nn-host1:rpc-port. + If the value of this property is unset the value of dfs.namenode.rpc-address will be used as the default. + + + + + dfs.namenode.servicerpc-bind-host + + + The actual address the service RPC server will bind to. If this optional address is + set, it overrides only the hostname portion of dfs.namenode.servicerpc-address. + It can also be specified per name node or name service for HA/Federation. + This is useful for making the name node listen on all interfaces by + setting it to 0.0.0.0. + + + + + dfs.namenode.secondary.http-address + 0.0.0.0:50090 + + The secondary namenode http server address and port. + + + + + dfs.namenode.secondary.https-address + 0.0.0.0:50091 + + The secondary namenode HTTPS server address and port. + + + + + dfs.datanode.address + 0.0.0.0:50010 + + The datanode server address and port for data transfer. + + + + + dfs.datanode.http.address + 0.0.0.0:50075 + + The datanode http server address and port. + + + + + dfs.datanode.ipc.address + 0.0.0.0:50020 + + The datanode ipc server address and port. + + + + + dfs.datanode.handler.count + 10 + The number of server threads for the datanode. + + + + dfs.namenode.http-address + 0.0.0.0:50070 + + The address and the base port where the dfs namenode web ui will listen on. + + + + + dfs.namenode.http-bind-host + + + The actual adress the HTTP server will bind to. If this optional address + is set, it overrides only the hostname portion of dfs.namenode.http-address. + It can also be specified per name node or name service for HA/Federation. + This is useful for making the name node HTTP server listen on all + interfaces by setting it to 0.0.0.0. + + + + + dfs.https.enable + false + + Deprecated. Use "dfs.http.policy" instead. + + + + + dfs.http.policy + HTTP_ONLY + Decide if HTTPS(SSL) is supported on HDFS + This configures the HTTP endpoint for HDFS daemons: + The following values are supported: + - HTTP_ONLY : Service is provided only on http + - HTTPS_ONLY : Service is provided only on https + - HTTP_AND_HTTPS : Service is provided both on http and https + + + + + dfs.client.https.need-auth + false + Whether SSL client certificate authentication is required + + + + + dfs.client.cached.conn.retry + 3 + The number of times the HDFS client will pull a socket from the + cache. Once this number is exceeded, the client will try to create a new + socket. + + + + + + dfs.https.server.keystore.resource + ssl-server.xml + Resource file from which ssl server keystore + information will be extracted + + + + + dfs.client.https.keystore.resource + ssl-client.xml + Resource file from which ssl client keystore + information will be extracted + + + + + dfs.datanode.https.address + 0.0.0.0:50475 + The datanode secure http server address and port. + + + + dfs.namenode.https-address + 0.0.0.0:50470 + The namenode secure http server address and port. + + + + dfs.namenode.https-bind-host + + + The actual adress the HTTPS server will bind to. If this optional address + is set, it overrides only the hostname portion of dfs.namenode.https-address. + It can also be specified per name node or name service for HA/Federation. + This is useful for making the name node HTTPS server listen on all + interfaces by setting it to 0.0.0.0. + + + + + dfs.datanode.dns.interface + default + The name of the Network Interface from which a data node should + report its IP address. + + + + + dfs.datanode.dns.nameserver + default + The host name or IP address of the name server (DNS) + which a DataNode should use to determine the host name used by the + NameNode for communication and display purposes. + + + + + dfs.namenode.backup.address + 0.0.0.0:50100 + + The backup node server address and port. + If the port is 0 then the server will start on a free port. + + + + + dfs.namenode.backup.http-address + 0.0.0.0:50105 + + The backup node http server address and port. + If the port is 0 then the server will start on a free port. + + + + + dfs.namenode.replication.considerLoad + true + Decide if chooseTarget considers the target's load or not + + + + dfs.default.chunk.view.size + 32768 + The number of bytes to view for a file on the browser. + + + + + dfs.datanode.du.reserved + 0 + Reserved space in bytes per volume. Always leave this much space free for non dfs use. + + + + + dfs.namenode.name.dir + file://${hadoop.tmp.dir}/dfs/name + Determines where on the local filesystem the DFS name node + should store the name table(fsimage). If this is a comma-delimited list + of directories then the name table is replicated in all of the + directories, for redundancy. + + + + dfs.namenode.name.dir.restore + false + Set to true to enable NameNode to attempt recovering a + previously failed dfs.namenode.name.dir. When enabled, a recovery of any + failed directory is attempted during checkpoint. + + + + dfs.namenode.fs-limits.max-component-length + 255 + Defines the maximum number of bytes in UTF-8 encoding in each + component of a path. A value of 0 will disable the check. + + + + dfs.namenode.fs-limits.max-directory-items + 1048576 + Defines the maximum number of items that a directory may + contain. A value of 0 will disable the check. + + + + dfs.namenode.fs-limits.min-block-size + 1048576 + Minimum block size in bytes, enforced by the Namenode at create + time. This prevents the accidental creation of files with tiny block + sizes (and thus many blocks), which can degrade + performance. + + + + dfs.namenode.fs-limits.max-blocks-per-file + 1048576 + Maximum number of blocks per file, enforced by the Namenode on + write. This prevents the creation of extremely large files which can + degrade performance. + + + + dfs.namenode.edits.dir + ${dfs.namenode.name.dir} + Determines where on the local filesystem the DFS name node + should store the transaction (edits) file. If this is a comma-delimited list + of directories then the transaction file is replicated in all of the + directories, for redundancy. Default value is same as dfs.namenode.name.dir + + + + + dfs.namenode.shared.edits.dir + + A directory on shared storage between the multiple namenodes + in an HA cluster. This directory will be written by the active and read + by the standby in order to keep the namespaces synchronized. This directory + does not need to be listed in dfs.namenode.edits.dir above. It should be + left empty in a non-HA cluster. + + + + + dfs.namenode.edits.journal-plugin.qjournal + org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager + + + + dfs.permissions.enabled + true + + If "true", enable permission checking in HDFS. + If "false", permission checking is turned off, + but all other behavior is unchanged. + Switching from one parameter value to the other does not change the mode, + owner or group of files or directories. + + + + + dfs.permissions.superusergroup + supergroup + The name of the group of super-users. + + + + + dfs.namenode.acls.enabled + false + + Set to true to enable support for HDFS ACLs (Access Control Lists). By + default, ACLs are disabled. When ACLs are disabled, the NameNode rejects + all RPCs related to setting or getting ACLs. + + + + + dfs.namenode.lazypersist.file.scrub.interval.sec + 300 + + The NameNode periodically scans the namespace for LazyPersist files with + missing blocks and unlinks them from the namespace. This configuration key + controls the interval between successive scans. Set it to a negative value + to disable this behavior. + + + + dfs.block.access.token.enable + false + + If "true", access tokens are used as capabilities for accessing datanodes. + If "false", no access tokens are checked on accessing datanodes. + + + + + dfs.block.access.key.update.interval + 600 + + Interval in minutes at which namenode updates its access keys. + + + + + dfs.block.access.token.lifetime + 600 + The lifetime of access tokens in minutes. + + + + dfs.datanode.data.dir + file://${hadoop.tmp.dir}/dfs/data + Determines where on the local filesystem an DFS data node + should store its blocks. If this is a comma-delimited + list of directories, then data will be stored in all named + directories, typically on different devices. + Directories that do not exist are ignored. + + + + + dfs.datanode.data.dir.perm + 700 + Permissions for the directories on on the local filesystem where + the DFS data node store its blocks. The permissions can either be octal or + symbolic. + + + + dfs.replication + 3 + Default block replication. + The actual number of replications can be specified when the file is created. + The default is used if replication is not specified in create time. + + + + + dfs.replication.max + 512 + Maximal block replication. + + + + + dfs.namenode.replication.min + 1 + Minimal block replication. + + + + + dfs.blocksize + 134217728 + + The default block size for new files, in bytes. + You can use the following suffix (case insensitive): + k(kilo), m(mega), g(giga), t(tera), p(peta), e(exa) to specify the size (such as 128k, 512m, 1g, etc.), + Or provide complete size in bytes (such as 134217728 for 128 MB). + + + + + dfs.client.block.write.retries + 3 + The number of retries for writing blocks to the data nodes, + before we signal failure to the application. + + + + + dfs.client.block.write.replace-datanode-on-failure.enable + true + + If there is a datanode/network failure in the write pipeline, + DFSClient will try to remove the failed datanode from the pipeline + and then continue writing with the remaining datanodes. As a result, + the number of datanodes in the pipeline is decreased. The feature is + to add new datanodes to the pipeline. + + This is a site-wide property to enable/disable the feature. + + When the cluster size is extremely small, e.g. 3 nodes or less, cluster + administrators may want to set the policy to NEVER in the default + configuration file or disable this feature. Otherwise, users may + experience an unusually high rate of pipeline failures since it is + impossible to find new datanodes for replacement. + + See also dfs.client.block.write.replace-datanode-on-failure.policy + + + + + dfs.client.block.write.replace-datanode-on-failure.policy + DEFAULT + + This property is used only if the value of + dfs.client.block.write.replace-datanode-on-failure.enable is true. + + ALWAYS: always add a new datanode when an existing datanode is removed. + + NEVER: never add a new datanode. + + DEFAULT: + Let r be the replication number. + Let n be the number of existing datanodes. + Add a new datanode only if r is greater than or equal to 3 and either + (1) floor(r/2) is greater than or equal to n; or + (2) r is greater than n and the block is hflushed/appended. + + + + + dfs.client.block.write.replace-datanode-on-failure.best-effort + false + + This property is used only if the value of + dfs.client.block.write.replace-datanode-on-failure.enable is true. + + Best effort means that the client will try to replace a failed datanode + in write pipeline (provided that the policy is satisfied), however, it + continues the write operation in case that the datanode replacement also + fails. + + Suppose the datanode replacement fails. + false: An exception should be thrown so that the write will fail. + true : The write should be resumed with the remaining datandoes. + + Note that setting this property to true allows writing to a pipeline + with a smaller number of datanodes. As a result, it increases the + probability of data loss. + + + + + dfs.blockreport.intervalMsec + 21600000 + Determines block reporting interval in milliseconds. + + + + dfs.blockreport.initialDelay 0 + Delay for first block report in seconds. + + + + dfs.blockreport.split.threshold + 1000000 + If the number of blocks on the DataNode is below this + threshold then it will send block reports for all Storage Directories + in a single message. + + If the number of blocks exceeds this threshold then the DataNode will + send block reports for each Storage Directory in separate messages. + + Set to zero to always split. + + + + + dfs.datanode.directoryscan.interval + 21600 + Interval in seconds for Datanode to scan data directories and + reconcile the difference between blocks in memory and on the disk. + + + + + dfs.datanode.directoryscan.threads + 1 + How many threads should the threadpool used to compile reports + for volumes in parallel have. + + + + + dfs.heartbeat.interval + 3 + Determines datanode heartbeat interval in seconds. + + + + dfs.namenode.handler.count + 10 + The number of server threads for the namenode. + + + + dfs.namenode.safemode.threshold-pct + 0.999f + + Specifies the percentage of blocks that should satisfy + the minimal replication requirement defined by dfs.namenode.replication.min. + Values less than or equal to 0 mean not to wait for any particular + percentage of blocks before exiting safemode. + Values greater than 1 will make safe mode permanent. + + + + + dfs.namenode.safemode.min.datanodes + 0 + + Specifies the number of datanodes that must be considered alive + before the name node exits safemode. + Values less than or equal to 0 mean not to take the number of live + datanodes into account when deciding whether to remain in safe mode + during startup. + Values greater than the number of datanodes in the cluster + will make safe mode permanent. + + + + + dfs.namenode.safemode.extension + 30000 + + Determines extension of safe mode in milliseconds + after the threshold level is reached. + + + + + dfs.namenode.resource.check.interval + 5000 + + The interval in milliseconds at which the NameNode resource checker runs. + The checker calculates the number of the NameNode storage volumes whose + available spaces are more than dfs.namenode.resource.du.reserved, and + enters safemode if the number becomes lower than the minimum value + specified by dfs.namenode.resource.checked.volumes.minimum. + + + + + dfs.namenode.resource.du.reserved + 104857600 + + The amount of space to reserve/require for a NameNode storage directory + in bytes. The default is 100MB. + + + + + dfs.namenode.resource.checked.volumes + + + A list of local directories for the NameNode resource checker to check in + addition to the local edits directories. + + + + + dfs.namenode.resource.checked.volumes.minimum + 1 + + The minimum number of redundant NameNode storage volumes required. + + + + + dfs.datanode.balance.bandwidthPerSec + 1048576 + + Specifies the maximum amount of bandwidth that each datanode + can utilize for the balancing purpose in term of + the number of bytes per second. + + + + + dfs.hosts + + Names a file that contains a list of hosts that are + permitted to connect to the namenode. The full pathname of the file + must be specified. If the value is empty, all hosts are + permitted. + + + + dfs.hosts.exclude + + Names a file that contains a list of hosts that are + not permitted to connect to the namenode. The full pathname of the + file must be specified. If the value is empty, no hosts are + excluded. + + + + dfs.namenode.max.objects + 0 + The maximum number of files, directories and blocks + dfs supports. A value of zero indicates no limit to the number + of objects that dfs supports. + + + + + dfs.namenode.datanode.registration.ip-hostname-check + true + + If true (the default), then the namenode requires that a connecting + datanode's address must be resolved to a hostname. If necessary, a reverse + DNS lookup is performed. All attempts to register a datanode from an + unresolvable address are rejected. + + It is recommended that this setting be left on to prevent accidental + registration of datanodes listed by hostname in the excludes file during a + DNS outage. Only set this to false in environments where there is no + infrastructure to support reverse DNS lookup. + + + + + dfs.namenode.decommission.interval + 30 + Namenode periodicity in seconds to check if decommission is + complete. + + + + dfs.namenode.decommission.nodes.per.interval + 5 + The number of nodes namenode checks if decommission is complete + in each dfs.namenode.decommission.interval. + + + + dfs.namenode.replication.interval + 3 + The periodicity in seconds with which the namenode computes + repliaction work for datanodes. + + + + dfs.namenode.accesstime.precision + 3600000 + The access time for HDFS file is precise upto this value. + The default value is 1 hour. Setting a value of 0 disables + access times for HDFS. + + + + + dfs.datanode.plugins + + Comma-separated list of datanode plug-ins to be activated. + + + + + dfs.namenode.plugins + + Comma-separated list of namenode plug-ins to be activated. + + + + + dfs.stream-buffer-size + 4096 + The size of buffer to stream files. + The size of this buffer should probably be a multiple of hardware + page size (4096 on Intel x86), and it determines how much data is + buffered during read and write operations. + + + + dfs.bytes-per-checksum + 512 + The number of bytes per checksum. Must not be larger than + dfs.stream-buffer-size + + + + dfs.client-write-packet-size + 65536 + Packet size for clients to write + + + + dfs.client.write.exclude.nodes.cache.expiry.interval.millis + 600000 + The maximum period to keep a DN in the excluded nodes list + at a client. After this period, in milliseconds, the previously excluded node(s) will + be removed automatically from the cache and will be considered good for block allocations + again. Useful to lower or raise in situations where you keep a file open for very long + periods (such as a Write-Ahead-Log (WAL) file) to make the writer tolerant to cluster maintenance + restarts. Defaults to 10 minutes. + + + + dfs.namenode.checkpoint.dir + file://${hadoop.tmp.dir}/dfs/namesecondary + Determines where on the local filesystem the DFS secondary + name node should store the temporary images to merge. + If this is a comma-delimited list of directories then the image is + replicated in all of the directories for redundancy. + + + + + dfs.namenode.checkpoint.edits.dir + ${dfs.namenode.checkpoint.dir} + Determines where on the local filesystem the DFS secondary + name node should store the temporary edits to merge. + If this is a comma-delimited list of directoires then teh edits is + replicated in all of the directoires for redundancy. + Default value is same as dfs.namenode.checkpoint.dir + + + + + dfs.namenode.checkpoint.period + 3600 + The number of seconds between two periodic checkpoints. + + + + + dfs.namenode.checkpoint.txns + 1000000 + The Secondary NameNode or CheckpointNode will create a checkpoint + of the namespace every 'dfs.namenode.checkpoint.txns' transactions, regardless + of whether 'dfs.namenode.checkpoint.period' has expired. + + + + + dfs.namenode.checkpoint.check.period + 60 + The SecondaryNameNode and CheckpointNode will poll the NameNode + every 'dfs.namenode.checkpoint.check.period' seconds to query the number + of uncheckpointed transactions. + + + + + dfs.namenode.checkpoint.max-retries + 3 + The SecondaryNameNode retries failed checkpointing. If the + failure occurs while loading fsimage or replaying edits, the number of + retries is limited by this variable. + + + + + dfs.namenode.num.checkpoints.retained + 2 + The number of image checkpoint files that will be retained by + the NameNode and Secondary NameNode in their storage directories. All edit + logs necessary to recover an up-to-date namespace from the oldest retained + checkpoint will also be retained. + + + + + dfs.namenode.num.extra.edits.retained + 1000000 + The number of extra transactions which should be retained + beyond what is minimally necessary for a NN restart. This can be useful for + audit purposes or for an HA setup where a remote Standby Node may have + been offline for some time and need to have a longer backlog of retained + edits in order to start again. + Typically each edit is on the order of a few hundred bytes, so the default + of 1 million edits should be on the order of hundreds of MBs or low GBs. + + NOTE: Fewer extra edits may be retained than value specified for this setting + if doing so would mean that more segments would be retained than the number + configured by dfs.namenode.max.extra.edits.segments.retained. + + + + + dfs.namenode.max.extra.edits.segments.retained + 10000 + The maximum number of extra edit log segments which should be retained + beyond what is minimally necessary for a NN restart. When used in conjunction with + dfs.namenode.num.extra.edits.retained, this configuration property serves to cap + the number of extra edits files to a reasonable value. + + + + + dfs.namenode.delegation.key.update-interval + 86400000 + The update interval for master key for delegation tokens + in the namenode in milliseconds. + + + + + dfs.namenode.delegation.token.max-lifetime + 604800000 + The maximum lifetime in milliseconds for which a delegation + token is valid. + + + + + dfs.namenode.delegation.token.renew-interval + 86400000 + The renewal interval for delegation token in milliseconds. + + + + + dfs.datanode.failed.volumes.tolerated + 0 + The number of volumes that are allowed to + fail before a datanode stops offering service. By default + any volume failure will cause a datanode to shutdown. + + + + + dfs.image.compress + false + Should the dfs image be compressed? + + + + + dfs.image.compression.codec + org.apache.hadoop.io.compress.DefaultCodec + If the dfs image is compressed, how should they be compressed? + This has to be a codec defined in io.compression.codecs. + + + + + dfs.image.transfer.timeout + 60000 + + Socket timeout for image transfer in milliseconds. This timeout and the related + dfs.image.transfer.bandwidthPerSec parameter should be configured such + that normal image transfer can complete successfully. + This timeout prevents client hangs when the sender fails during + image transfer. This is socket timeout during image tranfer. + + + + + dfs.image.transfer.bandwidthPerSec + 0 + + Maximum bandwidth used for image transfer in bytes per second. + This can help keep normal namenode operations responsive during + checkpointing. The maximum bandwidth and timeout in + dfs.image.transfer.timeout should be set such that normal image + transfers can complete successfully. + A default value of 0 indicates that throttling is disabled. + + + + + dfs.image.transfer.chunksize + 65536 + + Chunksize in bytes to upload the checkpoint. + Chunked streaming is used to avoid internal buffering of contents + of image file of huge size. + + + + + dfs.namenode.support.allow.format + true + Does HDFS namenode allow itself to be formatted? + You may consider setting this to false for any production + cluster, to avoid any possibility of formatting a running DFS. + + + + + dfs.datanode.max.transfer.threads + 4096 + + Specifies the maximum number of threads to use for transferring data + in and out of the DN. + + + + + dfs.datanode.readahead.bytes + 4193404 + + While reading block files, if the Hadoop native libraries are available, + the datanode can use the posix_fadvise system call to explicitly + page data into the operating system buffer cache ahead of the current + reader's position. This can improve performance especially when + disks are highly contended. + + This configuration specifies the number of bytes ahead of the current + read position which the datanode will attempt to read ahead. This + feature may be disabled by configuring this property to 0. + + If the native libraries are not available, this configuration has no + effect. + + + + + dfs.datanode.drop.cache.behind.reads + false + + In some workloads, the data read from HDFS is known to be significantly + large enough that it is unlikely to be useful to cache it in the + operating system buffer cache. In this case, the DataNode may be + configured to automatically purge all data from the buffer cache + after it is delivered to the client. This behavior is automatically + disabled for workloads which read only short sections of a block + (e.g HBase random-IO workloads). + + This may improve performance for some workloads by freeing buffer + cache spage usage for more cacheable data. + + If the Hadoop native libraries are not available, this configuration + has no effect. + + + + + dfs.datanode.drop.cache.behind.writes + false + + In some workloads, the data written to HDFS is known to be significantly + large enough that it is unlikely to be useful to cache it in the + operating system buffer cache. In this case, the DataNode may be + configured to automatically purge all data from the buffer cache + after it is written to disk. + + This may improve performance for some workloads by freeing buffer + cache spage usage for more cacheable data. + + If the Hadoop native libraries are not available, this configuration + has no effect. + + + + + dfs.datanode.sync.behind.writes + false + + If this configuration is enabled, the datanode will instruct the + operating system to enqueue all written data to the disk immediately + after it is written. This differs from the usual OS policy which + may wait for up to 30 seconds before triggering writeback. + + This may improve performance for some workloads by smoothing the + IO profile for data written to disk. + + If the Hadoop native libraries are not available, this configuration + has no effect. + + + + + dfs.client.failover.max.attempts + 15 + + Expert only. The number of client failover attempts that should be + made before the failover is considered failed. + + + + + dfs.client.failover.sleep.base.millis + 500 + + Expert only. The time to wait, in milliseconds, between failover + attempts increases exponentially as a function of the number of + attempts made so far, with a random factor of +/- 50%. This option + specifies the base value used in the failover calculation. The + first failover will retry immediately. The 2nd failover attempt + will delay at least dfs.client.failover.sleep.base.millis + milliseconds. And so on. + + + + + dfs.client.failover.sleep.max.millis + 15000 + + Expert only. The time to wait, in milliseconds, between failover + attempts increases exponentially as a function of the number of + attempts made so far, with a random factor of +/- 50%. This option + specifies the maximum value to wait between failovers. + Specifically, the time between two failover attempts will not + exceed +/- 50% of dfs.client.failover.sleep.max.millis + milliseconds. + + + + + dfs.client.failover.connection.retries + 0 + + Expert only. Indicates the number of retries a failover IPC client + will make to establish a server connection. + + + + + dfs.client.failover.connection.retries.on.timeouts + 0 + + Expert only. The number of retry attempts a failover IPC client + will make on socket timeout when establishing a server connection. + + + + + dfs.client.datanode-restart.timeout + 30 + + Expert only. The time to wait, in seconds, from reception of an + datanode shutdown notification for quick restart, until declaring + the datanode dead and invoking the normal recovery mechanisms. + The notification is sent by a datanode when it is being shutdown + using the shutdownDatanode admin command with the upgrade option. + + + + + dfs.nameservices + + + Comma-separated list of nameservices. + + + + + dfs.nameservice.id + + + The ID of this nameservice. If the nameservice ID is not + configured or more than one nameservice is configured for + dfs.nameservices it is determined automatically by + matching the local node's address with the configured address. + + + + + dfs.internal.nameservices + + + Comma-separated list of nameservices that belong to this cluster. + Datanode will report to all the nameservices in this list. By default + this is set to the value of dfs.nameservices. + + + + + dfs.ha.namenodes.EXAMPLENAMESERVICE + + + The prefix for a given nameservice, contains a comma-separated + list of namenodes for a given nameservice (eg EXAMPLENAMESERVICE). + + + + + dfs.ha.namenode.id + + + The ID of this namenode. If the namenode ID is not configured it + is determined automatically by matching the local node's address + with the configured address. + + + + + dfs.ha.log-roll.period + 120 + + How often, in seconds, the StandbyNode should ask the active to + roll edit logs. Since the StandbyNode only reads from finalized + log segments, the StandbyNode will only be as up-to-date as how + often the logs are rolled. Note that failover triggers a log roll + so the StandbyNode will be up to date before it becomes active. + + + + + dfs.ha.tail-edits.period + 60 + + How often, in seconds, the StandbyNode should check for new + finalized log segments in the shared edits log. + + + + + dfs.ha.automatic-failover.enabled + false + + Whether automatic failover is enabled. See the HDFS High + Availability documentation for details on automatic HA + configuration. + + + + + dfs.support.append + true + + Does HDFS allow appends to files? + + + + + dfs.client.use.datanode.hostname + false + Whether clients should use datanode hostnames when + connecting to datanodes. + + + + + dfs.datanode.use.datanode.hostname + false + Whether datanodes should use datanode hostnames when + connecting to other datanodes for data transfer. + + + + + dfs.client.local.interfaces + + A comma separated list of network interface names to use + for data transfer between the client and datanodes. When creating + a connection to read from or write to a datanode, the client + chooses one of the specified interfaces at random and binds its + socket to the IP of that interface. Individual names may be + specified as either an interface name (eg "eth0"), a subinterface + name (eg "eth0:0"), or an IP address (which may be specified using + CIDR notation to match a range of IPs). + + + + + dfs.datanode.shared.file.descriptor.paths + /dev/shm,/tmp + + A comma-separated list of paths to use when creating file descriptors that + will be shared between the DataNode and the DFSClient. Typically we use + /dev/shm, so that the file descriptors will not be written to disk. + Systems that don't have /dev/shm will fall back to /tmp by default. + + + + + dfs.short.circuit.shared.memory.watcher.interrupt.check.ms + 60000 + + The length of time in milliseconds that the short-circuit shared memory + watcher will go between checking for java interruptions sent from other + threads. This is provided mainly for unit tests. + + + + + dfs.namenode.kerberos.internal.spnego.principal + ${dfs.web.authentication.kerberos.principal} + + + + dfs.secondary.namenode.kerberos.internal.spnego.principal + ${dfs.web.authentication.kerberos.principal} + + + + dfs.namenode.avoid.read.stale.datanode + false + + Indicate whether or not to avoid reading from "stale" datanodes whose + heartbeat messages have not been received by the namenode + for more than a specified time interval. Stale datanodes will be + moved to the end of the node list returned for reading. See + dfs.namenode.avoid.write.stale.datanode for a similar setting for writes. + + + + + dfs.namenode.avoid.write.stale.datanode + false + + Indicate whether or not to avoid writing to "stale" datanodes whose + heartbeat messages have not been received by the namenode + for more than a specified time interval. Writes will avoid using + stale datanodes unless more than a configured ratio + (dfs.namenode.write.stale.datanode.ratio) of datanodes are marked as + stale. See dfs.namenode.avoid.read.stale.datanode for a similar setting + for reads. + + + + + dfs.namenode.stale.datanode.interval + 30000 + + Default time interval for marking a datanode as "stale", i.e., if + the namenode has not received heartbeat msg from a datanode for + more than this time interval, the datanode will be marked and treated + as "stale" by default. The stale interval cannot be too small since + otherwise this may cause too frequent change of stale states. + We thus set a minimum stale interval value (the default value is 3 times + of heartbeat interval) and guarantee that the stale interval cannot be less + than the minimum value. A stale data node is avoided during lease/block + recovery. It can be conditionally avoided for reads (see + dfs.namenode.avoid.read.stale.datanode) and for writes (see + dfs.namenode.avoid.write.stale.datanode). + + + + + dfs.namenode.write.stale.datanode.ratio + 0.5f + + When the ratio of number stale datanodes to total datanodes marked + is greater than this ratio, stop avoiding writing to stale nodes so + as to prevent causing hotspots. + + + + + dfs.namenode.invalidate.work.pct.per.iteration + 0.32f + + *Note*: Advanced property. Change with caution. + This determines the percentage amount of block + invalidations (deletes) to do over a single DN heartbeat + deletion command. The final deletion count is determined by applying this + percentage to the number of live nodes in the system. + The resultant number is the number of blocks from the deletion list + chosen for proper invalidation over a single heartbeat of a single DN. + Value should be a positive, non-zero percentage in float notation (X.Yf), + with 1.0f meaning 100%. + + + + + dfs.namenode.replication.work.multiplier.per.iteration + 2 + + *Note*: Advanced property. Change with caution. + This determines the total amount of block transfers to begin in + parallel at a DN, for replication, when such a command list is being + sent over a DN heartbeat by the NN. The actual number is obtained by + multiplying this multiplier with the total number of live nodes in the + cluster. The result number is the number of blocks to begin transfers + immediately for, per DN heartbeat. This number can be any positive, + non-zero integer. + + + + + nfs.server.port + 2049 + + Specify the port number used by Hadoop NFS. + + + + + nfs.mountd.port + 4242 + + Specify the port number used by Hadoop mount daemon. + + + + + nfs.dump.dir + /tmp/.hdfs-nfs + + This directory is used to temporarily save out-of-order writes before + writing to HDFS. For each file, the out-of-order writes are dumped after + they are accumulated to exceed certain threshold (e.g., 1MB) in memory. + One needs to make sure the directory has enough space. + + + + + nfs.rtmax + 1048576 + This is the maximum size in bytes of a READ request + supported by the NFS gateway. If you change this, make sure you + also update the nfs mount's rsize(add rsize= # of bytes to the + mount directive). + + + + + nfs.wtmax + 1048576 + This is the maximum size in bytes of a WRITE request + supported by the NFS gateway. If you change this, make sure you + also update the nfs mount's wsize(add wsize= # of bytes to the + mount directive). + + + + + nfs.keytab.file + + + *Note*: Advanced property. Change with caution. + This is the path to the keytab file for the hdfs-nfs gateway. + This is required when the cluster is kerberized. + + + + + nfs.kerberos.principal + + + *Note*: Advanced property. Change with caution. + This is the name of the kerberos principal. This is required when + the cluster is kerberized.It must be of this format: + nfs-gateway-user/nfs-gateway-host@kerberos-realm + + + + + nfs.allow.insecure.ports + true + + When set to false, client connections originating from unprivileged ports + (those above 1023) will be rejected. This is to ensure that clients + connecting to this NFS Gateway must have had root privilege on the machine + where they're connecting from. + + + + + dfs.webhdfs.enabled + true + + Enable WebHDFS (REST API) in Namenodes and Datanodes. + + + + + hadoop.fuse.connection.timeout + 300 + + The minimum number of seconds that we'll cache libhdfs connection objects + in fuse_dfs. Lower values will result in lower memory consumption; higher + values may speed up access by avoiding the overhead of creating new + connection objects. + + + + + hadoop.fuse.timer.period + 5 + + The number of seconds between cache expiry checks in fuse_dfs. Lower values + will result in fuse_dfs noticing changes to Kerberos ticket caches more + quickly. + + + + + dfs.metrics.percentiles.intervals + + + Comma-delimited set of integers denoting the desired rollover intervals + (in seconds) for percentile latency metrics on the Namenode and Datanode. + By default, percentile latency metrics are disabled. + + + + + dfs.encrypt.data.transfer + false + + Whether or not actual block data that is read/written from/to HDFS should + be encrypted on the wire. This only needs to be set on the NN and DNs, + clients will deduce this automatically. It is possible to override this setting + per connection by specifying custom logic via dfs.trustedchannel.resolver.class. + + + + + dfs.encrypt.data.transfer.algorithm + + + This value may be set to either "3des" or "rc4". If nothing is set, then + the configured JCE default on the system is used (usually 3DES.) It is + widely believed that 3DES is more cryptographically secure, but RC4 is + substantially faster. + + Note that if AES is supported by both the client and server then this + encryption algorithm will only be used to initially transfer keys for AES. + (See dfs.encrypt.data.transfer.cipher.suites.) + + + + + dfs.encrypt.data.transfer.cipher.suites + + + This value may be either undefined or AES/CTR/NoPadding. If defined, then + dfs.encrypt.data.transfer uses the specified cipher suite for data + encryption. If not defined, then only the algorithm specified in + dfs.encrypt.data.transfer.algorithm is used. By default, the property is + not defined. + + + + + dfs.encrypt.data.transfer.cipher.key.bitlength + 128 + + The key bitlength negotiated by dfsclient and datanode for encryption. + This value may be set to either 128, 192 or 256. + + + + + dfs.trustedchannel.resolver.class + + + TrustedChannelResolver is used to determine whether a channel + is trusted for plain data transfer. The TrustedChannelResolver is + invoked on both client and server side. If the resolver indicates + that the channel is trusted, then the data transfer will not be + encrypted even if dfs.encrypt.data.transfer is set to true. The + default implementation returns false indicating that the channel + is not trusted. + + + + + dfs.data.transfer.protection + + + A comma-separated list of SASL protection values used for secured + connections to the DataNode when reading or writing block data. Possible + values are authentication, integrity and privacy. authentication means + authentication only and no integrity or privacy; integrity implies + authentication and integrity are enabled; and privacy implies all of + authentication, integrity and privacy are enabled. If + dfs.encrypt.data.transfer is set to true, then it supersedes the setting for + dfs.data.transfer.protection and enforces that all connections must use a + specialized encrypted SASL handshake. This property is ignored for + connections to a DataNode listening on a privileged port. In this case, it + is assumed that the use of a privileged port establishes sufficient trust. + + + + + dfs.data.transfer.saslproperties.resolver.class + + + SaslPropertiesResolver used to resolve the QOP used for a connection to the + DataNode when reading or writing block data. If not specified, the value of + hadoop.security.saslproperties.resolver.class is used as the default value. + + + + + dfs.datanode.hdfs-blocks-metadata.enabled + false + + Boolean which enables backend datanode-side support for the experimental DistributedFileSystem#getFileVBlockStorageLocations API. + + + + + dfs.client.file-block-storage-locations.num-threads + 10 + + Number of threads used for making parallel RPCs in DistributedFileSystem#getFileBlockStorageLocations(). + + + + + dfs.client.file-block-storage-locations.timeout.millis + 1000 + + Timeout (in milliseconds) for the parallel RPCs made in DistributedFileSystem#getFileBlockStorageLocations(). + + + + + dfs.journalnode.rpc-address + 0.0.0.0:8485 + + The JournalNode RPC server address and port. + + + + + dfs.journalnode.http-address + 0.0.0.0:8480 + + The address and port the JournalNode HTTP server listens on. + If the port is 0 then the server will start on a free port. + + + + + dfs.journalnode.https-address + 0.0.0.0:8481 + + The address and port the JournalNode HTTPS server listens on. + If the port is 0 then the server will start on a free port. + + + + + dfs.namenode.audit.loggers + default + + List of classes implementing audit loggers that will receive audit events. + These should be implementations of org.apache.hadoop.hdfs.server.namenode.AuditLogger. + The special value "default" can be used to reference the default audit + logger, which uses the configured log system. Installing custom audit loggers + may affect the performance and stability of the NameNode. Refer to the custom + logger's documentation for more details. + + + + + dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold + 10737418240 + + Only used when the dfs.datanode.fsdataset.volume.choosing.policy is set to + org.apache.hadoop.hdfs.server.datanode.fsdataset.AvailableSpaceVolumeChoosingPolicy. + This setting controls how much DN volumes are allowed to differ in terms of + bytes of free disk space before they are considered imbalanced. If the free + space of all the volumes are within this range of each other, the volumes + will be considered balanced and block assignments will be done on a pure + round robin basis. + + + + + dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction + 0.75f + + Only used when the dfs.datanode.fsdataset.volume.choosing.policy is set to + org.apache.hadoop.hdfs.server.datanode.fsdataset.AvailableSpaceVolumeChoosingPolicy. + This setting controls what percentage of new block allocations will be sent + to volumes with more available disk space than others. This setting should + be in the range 0.0 - 1.0, though in practice 0.5 - 1.0, since there should + be no reason to prefer that volumes with less available disk space receive + more block allocations. + + + + + dfs.namenode.edits.noeditlogchannelflush + false + + Specifies whether to flush edit log file channel. When set, expensive + FileChannel#force calls are skipped and synchronous disk writes are + enabled instead by opening the edit log file with RandomAccessFile("rws") + flags. This can significantly improve the performance of edit log writes + on the Windows platform. + Note that the behavior of the "rws" flags is platform and hardware specific + and might not provide the same level of guarantees as FileChannel#force. + For example, the write will skip the disk-cache on SAS and SCSI devices + while it might not on SATA devices. This is an expert level setting, + change with caution. + + + + + dfs.client.cache.drop.behind.writes + + + Just like dfs.datanode.drop.cache.behind.writes, this setting causes the + page cache to be dropped behind HDFS writes, potentially freeing up more + memory for other uses. Unlike dfs.datanode.drop.cache.behind.writes, this + is a client-side setting rather than a setting for the entire datanode. + If present, this setting will override the DataNode default. + + If the native libraries are not available to the DataNode, this + configuration has no effect. + + + + + dfs.client.cache.drop.behind.reads + + + Just like dfs.datanode.drop.cache.behind.reads, this setting causes the + page cache to be dropped behind HDFS reads, potentially freeing up more + memory for other uses. Unlike dfs.datanode.drop.cache.behind.reads, this + is a client-side setting rather than a setting for the entire datanode. If + present, this setting will override the DataNode default. + + If the native libraries are not available to the DataNode, this + configuration has no effect. + + + + + dfs.client.cache.readahead + + + When using remote reads, this setting causes the datanode to + read ahead in the block file using posix_fadvise, potentially decreasing + I/O wait times. Unlike dfs.datanode.readahead.bytes, this is a client-side + setting rather than a setting for the entire datanode. If present, this + setting will override the DataNode default. + + When using local reads, this setting determines how much readahead we do in + BlockReaderLocal. + + If the native libraries are not available to the DataNode, this + configuration has no effect. + + + + + dfs.namenode.enable.retrycache + true + + This enables the retry cache on the namenode. Namenode tracks for + non-idempotent requests the corresponding response. If a client retries the + request, the response from the retry cache is sent. Such operations + are tagged with annotation @AtMostOnce in namenode protocols. It is + recommended that this flag be set to true. Setting it to false, will result + in clients getting failure responses to retried request. This flag must + be enabled in HA setup for transparent fail-overs. + + The entries in the cache have expiration time configurable + using dfs.namenode.retrycache.expirytime.millis. + + + + + dfs.namenode.retrycache.expirytime.millis + 600000 + + The time for which retry cache entries are retained. + + + + + dfs.namenode.retrycache.heap.percent + 0.03f + + This parameter configures the heap size allocated for retry cache + (excluding the response cached). This corresponds to approximately + 4096 entries for every 64MB of namenode process java heap size. + Assuming retry cache entry expiration time (configured using + dfs.namenode.retrycache.expirytime.millis) of 10 minutes, this + enables retry cache to support 7 operations per second sustained + for 10 minutes. As the heap size is increased, the operation rate + linearly increases. + + + + + dfs.client.mmap.enabled + true + + If this is set to false, the client won't attempt to perform memory-mapped reads. + + + + + dfs.client.mmap.cache.size + 256 + + When zero-copy reads are used, the DFSClient keeps a cache of recently used + memory mapped regions. This parameter controls the maximum number of + entries that we will keep in that cache. + + The larger this number is, the more file descriptors we will potentially + use for memory-mapped files. mmaped files also use virtual address space. + You may need to increase your ulimit virtual address space limits before + increasing the client mmap cache size. + + Note that you can still do zero-copy reads when this size is set to 0. + + + + + dfs.client.mmap.cache.timeout.ms + 3600000 + + The minimum length of time that we will keep an mmap entry in the cache + between uses. If an entry is in the cache longer than this, and nobody + uses it, it will be removed by a background thread. + + + + + dfs.client.mmap.retry.timeout.ms + 300000 + + The minimum amount of time that we will wait before retrying a failed mmap + operation. + + + + + dfs.client.short.circuit.replica.stale.threshold.ms + 1800000 + + The maximum amount of time that we will consider a short-circuit replica to + be valid, if there is no communication from the DataNode. After this time + has elapsed, we will re-fetch the short-circuit replica even if it is in + the cache. + + + + + dfs.namenode.path.based.cache.block.map.allocation.percent + 0.25 + + The percentage of the Java heap which we will allocate to the cached blocks + map. The cached blocks map is a hash map which uses chained hashing. + Smaller maps may be accessed more slowly if the number of cached blocks is + large; larger maps will consume more memory. + + + + + dfs.datanode.max.locked.memory + 0 + + The amount of memory in bytes to use for caching of block replicas in + memory on the datanode. The datanode's maximum locked memory soft ulimit + (RLIMIT_MEMLOCK) must be set to at least this value, else the datanode + will abort on startup. + + By default, this parameter is set to 0, which disables in-memory caching. + + If the native libraries are not available to the DataNode, this + configuration has no effect. + + + + + dfs.namenode.list.cache.directives.num.responses + 100 + + This value controls the number of cache directives that the NameNode will + send over the wire in response to a listDirectives RPC. + + + + + dfs.namenode.list.cache.pools.num.responses + 100 + + This value controls the number of cache pools that the NameNode will + send over the wire in response to a listPools RPC. + + + + + dfs.namenode.path.based.cache.refresh.interval.ms + 30000 + + The amount of milliseconds between subsequent path cache rescans. Path + cache rescans are when we calculate which blocks should be cached, and on + what datanodes. + + By default, this parameter is set to 30 seconds. + + + + + dfs.namenode.path.based.cache.retry.interval.ms + 30000 + + When the NameNode needs to uncache something that is cached, or cache + something that is not cached, it must direct the DataNodes to do so by + sending a DNA_CACHE or DNA_UNCACHE command in response to a DataNode + heartbeat. This parameter controls how frequently the NameNode will + resend these commands. + + + + + dfs.datanode.fsdatasetcache.max.threads.per.volume + 4 + + The maximum number of threads per volume to use for caching new data + on the datanode. These threads consume both I/O and CPU. This can affect + normal datanode operations. + + + + + dfs.cachereport.intervalMsec + 10000 + + Determines cache reporting interval in milliseconds. After this amount of + time, the DataNode sends a full report of its cache state to the NameNode. + The NameNode uses the cache report to update its map of cached blocks to + DataNode locations. + + This configuration has no effect if in-memory caching has been disabled by + setting dfs.datanode.max.locked.memory to 0 (which is the default). + + If the native libraries are not available to the DataNode, this + configuration has no effect. + + + + + dfs.namenode.edit.log.autoroll.multiplier.threshold + 2.0 + + Determines when an active namenode will roll its own edit log. + The actual threshold (in number of edits) is determined by multiplying + this value by dfs.namenode.checkpoint.txns. + + This prevents extremely large edit files from accumulating on the active + namenode, which can cause timeouts during namenode startup and pose an + administrative hassle. This behavior is intended as a failsafe for when + the standby or secondary namenode fail to roll the edit log by the normal + checkpoint threshold. + + + + + dfs.namenode.edit.log.autoroll.check.interval.ms + 300000 + + How often an active namenode will check if it needs to roll its edit log, + in milliseconds. + + + + + dfs.webhdfs.user.provider.user.pattern + ^[A-Za-z_][A-Za-z0-9._-]*[$]?$ + + Valid pattern for user and group names for webhdfs, it must be a valid java regex. + + + + + dfs.client.context + default + + The name of the DFSClient context that we should use. Clients that share + a context share a socket cache and short-circuit cache, among other things. + You should only change this if you don't want to share with another set of + threads. + + + + + dfs.client.read.shortcircuit + false + + This configuration parameter turns on short-circuit local reads. + + + + + dfs.domain.socket.path + + + Optional. This is a path to a UNIX domain socket that will be used for + communication between the DataNode and local HDFS clients. + If the string "_PORT" is present in this path, it will be replaced by the + TCP port of the DataNode. + + + + + dfs.client.read.shortcircuit.skip.checksum + false + + If this configuration parameter is set, + short-circuit local reads will skip checksums. + This is normally not recommended, + but it may be useful for special setups. + You might consider using this + if you are doing your own checksumming outside of HDFS. + + + + + dfs.client.read.shortcircuit.streams.cache.size + 256 + + The DFSClient maintains a cache of recently opened file descriptors. + This parameter controls the size of that cache. + Setting this higher will use more file descriptors, + but potentially provide better performance on workloads + involving lots of seeks. + + + + + dfs.client.read.shortcircuit.streams.cache.expiry.ms + 300000 + + This controls the minimum amount of time + file descriptors need to sit in the client cache context + before they can be closed for being inactive for too long. + + + + + dfs.datanode.shared.file.descriptor.paths + /dev/shm,/tmp + + Comma separated paths to the directory on which + shared memory segments are created. + The client and the DataNode exchange information via + this shared memory segment. + It tries paths in order until creation of shared memory segment succeeds. + + + + + dfs.client.use.legacy.blockreader.local + false + + Legacy short-circuit reader implementation based on HDFS-2246 is used + if this configuration parameter is true. + This is for the platforms other than Linux + where the new implementation based on HDFS-347 is not available. + + + + + dfs.block.local-path-access.user + + + Comma separated list of the users allowd to open block files + on legacy short-circuit local read. + + + + + dfs.client.domain.socket.data.traffic + false + + This control whether we will try to pass normal data traffic + over UNIX domain socket rather than over TCP socket + on node-local data transfer. + This is currently experimental and turned off by default. + + + + + dfs.namenode.reject-unresolved-dn-topology-mapping + false + + If the value is set to true, then namenode will reject datanode + registration if the topology mapping for a datanode is not resolved and + NULL is returned (script defined by net.topology.script.file.name fails + to execute). Otherwise, datanode will be registered and the default rack + will be assigned as the topology path. Topology paths are important for + data resiliency, since they define fault domains. Thus it may be unwanted + behavior to allow datanode registration with the default rack if the + resolving topology failed. + + + + + dfs.client.slow.io.warning.threshold.ms + 30000 + The threshold in milliseconds at which we will log a slow + io warning in a dfsclient. By default, this parameter is set to 30000 + milliseconds (30 seconds). + + + + + dfs.datanode.slow.io.warning.threshold.ms + 300 + The threshold in milliseconds at which we will log a slow + io warning in a datanode. By default, this parameter is set to 300 + milliseconds. + + + + + dfs.namenode.xattrs.enabled + true + + Whether support for extended attributes is enabled on the NameNode. + + + + + dfs.namenode.fs-limits.max-xattrs-per-inode + 32 + + Maximum number of extended attributes per inode. + + + + + dfs.namenode.fs-limits.max-xattr-size + 16384 + + The maximum combined size of the name and value of an extended attribute in bytes. + + + + + dfs.namenode.startup.delay.block.deletion.sec + 0 + The delay in seconds at which we will pause the blocks deletion + after Namenode startup. By default it's disabled. + In the case a directory has large number of directories and files are + deleted, suggested delay is one hour to give the administrator enough time + to notice large number of pending deletion blocks and take corrective + action. + + + + + dfs.namenode.list.encryption.zones.num.responses + 100 + When listing encryption zones, the maximum number of zones + that will be returned in a batch. Fetching the list incrementally in + batches improves namenode performance. + + + + + dfs.namenode.inotify.max.events.per.rpc + 1000 + Maximum number of events that will be sent to an inotify client + in a single RPC response. The default value attempts to amortize away + the overhead for this RPC while avoiding huge memory requirements for the + client and NameNode (1000 events should consume no more than 1 MB.) + + + + + dfs.user.home.dir.prefix + /user + The directory to prepend to user name to get the user's + home direcotry. + + + + + dfs.datanode.cache.revocation.timeout.ms + 900000 + When the DFSClient reads from a block file which the DataNode is + caching, the DFSClient can skip verifying checksums. The DataNode will + keep the block file in cache until the client is done. If the client takes + an unusually long time, though, the DataNode may need to evict the block + file from the cache anyway. This value controls how long the DataNode will + wait for the client to release a replica that it is reading without + checksums. + + + + + dfs.datanode.cache.revocation.polling.ms + 500 + How often the DataNode should poll to see if the clients have + stopped using a replica that the DataNode wants to uncache. + + + + + dfs.datanode.block.id.layout.upgrade.threads + 12 + The number of threads to use when creating hard links from + current to previous blocks during upgrade of a DataNode to block ID-based + block layout (see HDFS-6482 for details on the layout). + + + + dfs.encryption.key.provider.uri + + The KeyProvider to use when interacting with encryption keys used + when reading and writing to an encryption zone. + + + + + dfs.storage.policy.enabled + true + + Allow users to change the storage policy on files and directories. + + + + + dfs.namenode.legacy-oiv-image.dir + + Determines where to save the namespace in the old fsimage format + during checkpointing by standby NameNode or SecondaryNameNode. Users can + dump the contents of the old format fsimage by oiv_legacy command. If + the value is not specified, old format fsimage will not be saved in + checkpoint. + + + + diff --git a/sahara/plugins/vanilla/v2_6_0/resources/mapred-default.xml b/sahara/plugins/vanilla/v2_6_0/resources/mapred-default.xml new file mode 100644 index 0000000000..7075d7de0a --- /dev/null +++ b/sahara/plugins/vanilla/v2_6_0/resources/mapred-default.xml @@ -0,0 +1,1955 @@ + + + + + + + + + + + + mapreduce.jobtracker.jobhistory.location + + If job tracker is static the history files are stored + in this single well known place. If No value is set here, by default, + it is in the local file system at ${hadoop.log.dir}/history. + + + + + mapreduce.jobtracker.jobhistory.task.numberprogresssplits + 12 + Every task attempt progresses from 0.0 to 1.0 [unless + it fails or is killed]. We record, for each task attempt, certain + statistics over each twelfth of the progress range. You can change + the number of intervals we divide the entire range of progress into + by setting this property. Higher values give more precision to the + recorded data, but costs more memory in the job tracker at runtime. + Each increment in this attribute costs 16 bytes per running task. + + + + + mapreduce.job.userhistorylocation + + User can specify a location to store the history files of + a particular job. If nothing is specified, the logs are stored in + output directory. The files are stored in "_logs/history/" in the directory. + User can stop logging by giving the value "none". + + + + + mapreduce.jobtracker.jobhistory.completed.location + + The completed job history files are stored at this single well + known location. If nothing is specified, the files are stored at + ${mapreduce.jobtracker.jobhistory.location}/done. + + + + + mapreduce.job.committer.setup.cleanup.needed + true + true, if job needs job-setup and job-cleanup. + false, otherwise + + + + + + mapreduce.task.io.sort.factor + 10 + The number of streams to merge at once while sorting + files. This determines the number of open file handles. + + + + mapreduce.task.io.sort.mb + 100 + The total amount of buffer memory to use while sorting + files, in megabytes. By default, gives each merge stream 1MB, which + should minimize seeks. + + + + mapreduce.map.sort.spill.percent + 0.80 + The soft limit in the serialization buffer. Once reached, a + thread will begin to spill the contents to disk in the background. Note that + collection will not block if this threshold is exceeded while a spill is + already in progress, so spills may be larger than this threshold when it is + set to less than .5 + + + + mapreduce.jobtracker.address + local + The host and port that the MapReduce job tracker runs + at. If "local", then jobs are run in-process as a single map + and reduce task. + + + + + mapreduce.local.clientfactory.class.name + org.apache.hadoop.mapred.LocalClientFactory + This the client factory that is responsible for + creating local job runner client + + + + mapreduce.jobtracker.http.address + 0.0.0.0:50030 + + The job tracker http server address and port the server will listen on. + If the port is 0 then the server will start on a free port. + + + + + mapreduce.jobtracker.handler.count + 10 + + The number of server threads for the JobTracker. This should be roughly + 4% of the number of tasktracker nodes. + + + + + mapreduce.tasktracker.report.address + 127.0.0.1:0 + The interface and port that task tracker server listens on. + Since it is only connected to by the tasks, it uses the local interface. + EXPERT ONLY. Should only be changed if your host does not have the loopback + interface. + + + + mapreduce.cluster.local.dir + ${hadoop.tmp.dir}/mapred/local + The local directory where MapReduce stores intermediate + data files. May be a comma-separated list of + directories on different devices in order to spread disk i/o. + Directories that do not exist are ignored. + + + + + mapreduce.jobtracker.system.dir + ${hadoop.tmp.dir}/mapred/system + The directory where MapReduce stores control files. + + + + + mapreduce.jobtracker.staging.root.dir + ${hadoop.tmp.dir}/mapred/staging + The root of the staging area for users' job files + In practice, this should be the directory where users' home + directories are located (usually /user) + + + + + mapreduce.cluster.temp.dir + ${hadoop.tmp.dir}/mapred/temp + A shared directory for temporary files. + + + + + mapreduce.tasktracker.local.dir.minspacestart + 0 + If the space in mapreduce.cluster.local.dir drops under this, + do not ask for more tasks. + Value in bytes. + + + + + mapreduce.tasktracker.local.dir.minspacekill + 0 + If the space in mapreduce.cluster.local.dir drops under this, + do not ask more tasks until all the current ones have finished and + cleaned up. Also, to save the rest of the tasks we have running, + kill one of them, to clean up some space. Start with the reduce tasks, + then go with the ones that have finished the least. + Value in bytes. + + + + + mapreduce.jobtracker.expire.trackers.interval + 600000 + Expert: The time-interval, in miliseconds, after which + a tasktracker is declared 'lost' if it doesn't send heartbeats. + + + + + mapreduce.tasktracker.instrumentation + org.apache.hadoop.mapred.TaskTrackerMetricsInst + Expert: The instrumentation class to associate with each TaskTracker. + + + + + mapreduce.tasktracker.resourcecalculatorplugin + + + Name of the class whose instance will be used to query resource information + on the tasktracker. + + The class must be an instance of + org.apache.hadoop.util.ResourceCalculatorPlugin. If the value is null, the + tasktracker attempts to use a class appropriate to the platform. + Currently, the only platform supported is Linux. + + + + + mapreduce.tasktracker.taskmemorymanager.monitoringinterval + 5000 + The interval, in milliseconds, for which the tasktracker waits + between two cycles of monitoring its tasks' memory usage. Used only if + tasks' memory management is enabled via mapred.tasktracker.tasks.maxmemory. + + + + + mapreduce.tasktracker.tasks.sleeptimebeforesigkill + 5000 + The time, in milliseconds, the tasktracker waits for sending a + SIGKILL to a task, after it has been sent a SIGTERM. This is currently + not used on WINDOWS where tasks are just sent a SIGTERM. + + + + + mapreduce.job.maps + 2 + The default number of map tasks per job. + Ignored when mapreduce.jobtracker.address is "local". + + + + + mapreduce.job.reduces + 1 + The default number of reduce tasks per job. Typically set to 99% + of the cluster's reduce capacity, so that if a node fails the reduces can + still be executed in a single wave. + Ignored when mapreduce.jobtracker.address is "local". + + + + + mapreduce.jobtracker.restart.recover + false + "true" to enable (job) recovery upon restart, + "false" to start afresh + + + + + mapreduce.jobtracker.jobhistory.block.size + 3145728 + The block size of the job history file. Since the job recovery + uses job history, its important to dump job history to disk as + soon as possible. Note that this is an expert level parameter. + The default value is set to 3 MB. + + + + + mapreduce.jobtracker.taskscheduler + org.apache.hadoop.mapred.JobQueueTaskScheduler + The class responsible for scheduling the tasks. + + + + + mapreduce.job.reducer.preempt.delay.sec + 0 + The threshold in terms of seconds after which an unsatisfied mapper + request triggers reducer preemption to free space. Default 0 implies that the + reduces should be preempted immediately after allocation if there is currently no + room for newly allocated mappers. + + + + + mapreduce.job.max.split.locations + 10 + The max number of block locations to store for each split for + locality calculation. + + + + + mapreduce.job.split.metainfo.maxsize + 10000000 + The maximum permissible size of the split metainfo file. + The JobTracker won't attempt to read split metainfo files bigger than + the configured value. + No limits if set to -1. + + + + + mapreduce.jobtracker.taskscheduler.maxrunningtasks.perjob + + The maximum number of running tasks for a job before + it gets preempted. No limits if undefined. + + + + + mapreduce.map.maxattempts + 4 + Expert: The maximum number of attempts per map task. + In other words, framework will try to execute a map task these many number + of times before giving up on it. + + + + + mapreduce.reduce.maxattempts + 4 + Expert: The maximum number of attempts per reduce task. + In other words, framework will try to execute a reduce task these many number + of times before giving up on it. + + + + + mapreduce.reduce.shuffle.fetch.retry.enabled + ${yarn.nodemanager.recovery.enabled} + Set to enable fetch retry during host restart. + + + + mapreduce.reduce.shuffle.fetch.retry.interval-ms + 1000 + Time of interval that fetcher retry to fetch again when some + non-fatal failure happens because of some events like NM restart. + + + + + mapreduce.reduce.shuffle.fetch.retry.timeout-ms + 30000 + Timeout value for fetcher to retry to fetch again when some + non-fatal failure happens because of some events like NM restart. + + + + mapreduce.reduce.shuffle.retry-delay.max.ms + 60000 + The maximum number of ms the reducer will delay before retrying + to download map data. + + + + + mapreduce.reduce.shuffle.parallelcopies + 5 + The default number of parallel transfers run by reduce + during the copy(shuffle) phase. + + + + + mapreduce.reduce.shuffle.connect.timeout + 180000 + Expert: The maximum amount of time (in milli seconds) reduce + task spends in trying to connect to a tasktracker for getting map output. + + + + + mapreduce.reduce.shuffle.read.timeout + 180000 + Expert: The maximum amount of time (in milli seconds) reduce + task waits for map output data to be available for reading after obtaining + connection. + + + + + mapreduce.shuffle.connection-keep-alive.enable + false + set to true to support keep-alive connections. + + + + mapreduce.shuffle.connection-keep-alive.timeout + 5 + The number of seconds a shuffle client attempts to retain + http connection. Refer "Keep-Alive: timeout=" header in + Http specification + + + + + mapreduce.task.timeout + 600000 + The number of milliseconds before a task will be + terminated if it neither reads an input, writes an output, nor + updates its status string. A value of 0 disables the timeout. + + + + + mapreduce.tasktracker.map.tasks.maximum + 2 + The maximum number of map tasks that will be run + simultaneously by a task tracker. + + + + + mapreduce.tasktracker.reduce.tasks.maximum + 2 + The maximum number of reduce tasks that will be run + simultaneously by a task tracker. + + + + + mapreduce.map.memory.mb + 1024 + The amount of memory to request from the scheduler for each + map task. + + + + + mapreduce.map.cpu.vcores + 1 + The number of virtual cores to request from the scheduler for + each map task. + + + + + mapreduce.reduce.memory.mb + 1024 + The amount of memory to request from the scheduler for each + reduce task. + + + + + mapreduce.reduce.cpu.vcores + 1 + The number of virtual cores to request from the scheduler for + each reduce task. + + + + + mapreduce.jobtracker.retiredjobs.cache.size + 1000 + The number of retired job status to keep in the cache. + + + + + mapreduce.tasktracker.outofband.heartbeat + false + Expert: Set this to true to let the tasktracker send an + out-of-band heartbeat on task-completion for better latency. + + + + + mapreduce.jobtracker.jobhistory.lru.cache.size + 5 + The number of job history files loaded in memory. The jobs are + loaded when they are first accessed. The cache is cleared based on LRU. + + + + + mapreduce.jobtracker.instrumentation + org.apache.hadoop.mapred.JobTrackerMetricsInst + Expert: The instrumentation class to associate with each JobTracker. + + + + + mapred.child.java.opts + -Xmx200m + Java opts for the task processes. + The following symbol, if present, will be interpolated: @taskid@ is replaced + by current TaskID. Any other occurrences of '@' will go unchanged. + For example, to enable verbose gc logging to a file named for the taskid in + /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: + -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc + + Usage of -Djava.library.path can cause programs to no longer function if + hadoop native libraries are used. These values should instead be set as part + of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and + mapreduce.reduce.env config settings. + + + + + + + + + mapred.child.env + + User added environment variables for the task processes. + Example : + 1) A=foo This will set the env variable A to foo + 2) B=$B:c This is inherit nodemanager's B env variable on Unix. + 3) B=%B%;c This is inherit nodemanager's B env variable on Windows. + + + + + + + + + mapreduce.admin.user.env + + + Expert: Additional execution environment entries for + map and reduce task processes. This is not an additive property. + You must preserve the original value if you want your map and + reduce tasks to have access to native libraries (compression, etc). + When this value is empty, the command to set execution + envrionment will be OS dependent: + For linux, use LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native. + For windows, use PATH = %PATH%;%HADOOP_COMMON_HOME%\\bin. + + + + + mapreduce.task.tmp.dir + ./tmp + To set the value of tmp directory for map and reduce tasks. + If the value is an absolute path, it is directly assigned. Otherwise, it is + prepended with task's working directory. The java tasks are executed with + option -Djava.io.tmpdir='the absolute path of the tmp dir'. Pipes and + streaming are set with environment variable, + TMPDIR='the absolute path of the tmp dir' + + + + + mapreduce.map.log.level + INFO + The logging level for the map task. The allowed levels are: + OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL. + + + + + mapreduce.reduce.log.level + INFO + The logging level for the reduce task. The allowed levels are: + OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL. + + + + + mapreduce.map.cpu.vcores + 1 + + The number of virtual cores required for each map task. + + + + + mapreduce.reduce.cpu.vcores + 1 + + The number of virtual cores required for each reduce task. + + + + + mapreduce.reduce.merge.inmem.threshold + 1000 + The threshold, in terms of the number of files + for the in-memory merge process. When we accumulate threshold number of files + we initiate the in-memory merge and spill to disk. A value of 0 or less than + 0 indicates we want to DON'T have any threshold and instead depend only on + the ramfs's memory consumption to trigger the merge. + + + + + mapreduce.reduce.shuffle.merge.percent + 0.66 + The usage threshold at which an in-memory merge will be + initiated, expressed as a percentage of the total memory allocated to + storing in-memory map outputs, as defined by + mapreduce.reduce.shuffle.input.buffer.percent. + + + + + mapreduce.reduce.shuffle.input.buffer.percent + 0.70 + The percentage of memory to be allocated from the maximum heap + size to storing map outputs during the shuffle. + + + + + mapreduce.reduce.input.buffer.percent + 0.0 + The percentage of memory- relative to the maximum heap size- to + retain map outputs during the reduce. When the shuffle is concluded, any + remaining map outputs in memory must consume less than this threshold before + the reduce can begin. + + + + + mapreduce.reduce.shuffle.memory.limit.percent + 0.25 + Expert: Maximum percentage of the in-memory limit that a + single shuffle can consume + + + + mapreduce.shuffle.ssl.enabled + false + + Whether to use SSL for for the Shuffle HTTP endpoints. + + + + + mapreduce.shuffle.ssl.file.buffer.size + 65536 + Buffer size for reading spills from file when using SSL. + + + + + mapreduce.shuffle.max.connections + 0 + Max allowed connections for the shuffle. Set to 0 (zero) + to indicate no limit on the number of connections. + + + + + mapreduce.shuffle.max.threads + 0 + Max allowed threads for serving shuffle connections. Set to zero + to indicate the default of 2 times the number of available + processors (as reported by Runtime.availableProcessors()). Netty is used to + serve requests, so a thread is not needed for each connection. + + + + + mapreduce.shuffle.transferTo.allowed + + This option can enable/disable using nio transferTo method in + the shuffle phase. NIO transferTo does not perform well on windows in the + shuffle phase. Thus, with this configuration property it is possible to + disable it, in which case custom transfer method will be used. Recommended + value is false when running Hadoop on Windows. For Linux, it is recommended + to set it to true. If nothing is set then the default value is false for + Windows, and true for Linux. + + + + + mapreduce.shuffle.transfer.buffer.size + 131072 + This property is used only if + mapreduce.shuffle.transferTo.allowed is set to false. In that case, + this property defines the size of the buffer used in the buffer copy code + for the shuffle phase. The size of this buffer determines the size of the IO + requests. + + + + + mapreduce.reduce.markreset.buffer.percent + 0.0 + The percentage of memory -relative to the maximum heap size- to + be used for caching values when using the mark-reset functionality. + + + + + mapreduce.map.speculative + true + If true, then multiple instances of some map tasks + may be executed in parallel. + + + + mapreduce.reduce.speculative + true + If true, then multiple instances of some reduce tasks + may be executed in parallel. + + + mapreduce.job.speculative.speculativecap + 0.1 + The max percent (0-1) of running tasks that + can be speculatively re-executed at any time. + + + + mapreduce.job.map.output.collector.class + org.apache.hadoop.mapred.MapTask$MapOutputBuffer + + The MapOutputCollector implementation(s) to use. This may be a comma-separated + list of class names, in which case the map task will try to initialize each + of the collectors in turn. The first to successfully initialize will be used. + + + + + mapreduce.job.speculative.slowtaskthreshold + 1.0 + The number of standard deviations by which a task's + ave progress-rates must be lower than the average of all running tasks' + for the task to be considered too slow. + + + + + mapreduce.job.speculative.slownodethreshold + 1.0 + The number of standard deviations by which a Task + Tracker's ave map and reduce progress-rates (finishTime-dispatchTime) + must be lower than the average of all successful map/reduce task's for + the TT to be considered too slow to give a speculative task to. + + + + + mapreduce.job.jvm.numtasks + 1 + How many tasks to run per jvm. If set to -1, there is + no limit. + + + + + mapreduce.job.ubertask.enable + false + Whether to enable the small-jobs "ubertask" optimization, + which runs "sufficiently small" jobs sequentially within a single JVM. + "Small" is defined by the following maxmaps, maxreduces, and maxbytes + settings. Note that configurations for application masters also affect + the "Small" definition - yarn.app.mapreduce.am.resource.mb must be + larger than both mapreduce.map.memory.mb and mapreduce.reduce.memory.mb, + and yarn.app.mapreduce.am.resource.cpu-vcores must be larger than + both mapreduce.map.cpu.vcores and mapreduce.reduce.cpu.vcores to enable + ubertask. Users may override this value. + + + + + mapreduce.job.ubertask.maxmaps + 9 + Threshold for number of maps, beyond which job is considered + too big for the ubertasking optimization. Users may override this value, + but only downward. + + + + + mapreduce.job.ubertask.maxreduces + 1 + Threshold for number of reduces, beyond which job is considered + too big for the ubertasking optimization. CURRENTLY THE CODE CANNOT SUPPORT + MORE THAN ONE REDUCE and will ignore larger values. (Zero is a valid max, + however.) Users may override this value, but only downward. + + + + + mapreduce.job.ubertask.maxbytes + + Threshold for number of input bytes, beyond which job is + considered too big for the ubertasking optimization. If no value is + specified, dfs.block.size is used as a default. Be sure to specify a + default value in mapred-site.xml if the underlying filesystem is not HDFS. + Users may override this value, but only downward. + + + + + mapreduce.job.emit-timeline-data + false + Specifies if the Application Master should emit timeline data + to the timeline server. Individual jobs can override this value. + + + + + mapreduce.input.fileinputformat.split.minsize + 0 + The minimum size chunk that map input should be split + into. Note that some file formats may have minimum split sizes that + take priority over this setting. + + + + mapreduce.input.fileinputformat.list-status.num-threads + 1 + The number of threads to use to list and fetch block locations + for the specified input paths. Note: multiple threads should not be used + if a custom non thread-safe path filter is used. + + + + + mapreduce.jobtracker.maxtasks.perjob + -1 + The maximum number of tasks for a single job. + A value of -1 indicates that there is no maximum. + + + + mapreduce.input.lineinputformat.linespermap + 1 + When using NLineInputFormat, the number of lines of input data + to include in each split. + + + + mapreduce.client.submit.file.replication + 10 + The replication level for submitted job files. This + should be around the square root of the number of nodes. + + + + + + mapreduce.tasktracker.dns.interface + default + The name of the Network Interface from which a task + tracker should report its IP address. + + + + + mapreduce.tasktracker.dns.nameserver + default + The host name or IP address of the name server (DNS) + which a TaskTracker should use to determine the host name used by + the JobTracker for communication and display purposes. + + + + + mapreduce.tasktracker.http.threads + 40 + The number of worker threads that for the http server. This is + used for map output fetching + + + + + mapreduce.tasktracker.http.address + 0.0.0.0:50060 + + The task tracker http server address and port. + If the port is 0 then the server will start on a free port. + + + + + mapreduce.task.files.preserve.failedtasks + false + Should the files for failed tasks be kept. This should only be + used on jobs that are failing, because the storage is never + reclaimed. It also prevents the map outputs from being erased + from the reduce directory as they are consumed. + + + + + + + mapreduce.output.fileoutputformat.compress + false + Should the job outputs be compressed? + + + + + mapreduce.output.fileoutputformat.compress.type + RECORD + If the job outputs are to compressed as SequenceFiles, how should + they be compressed? Should be one of NONE, RECORD or BLOCK. + + + + + mapreduce.output.fileoutputformat.compress.codec + org.apache.hadoop.io.compress.DefaultCodec + If the job outputs are compressed, how should they be compressed? + + + + + mapreduce.map.output.compress + false + Should the outputs of the maps be compressed before being + sent across the network. Uses SequenceFile compression. + + + + + mapreduce.map.output.compress.codec + org.apache.hadoop.io.compress.DefaultCodec + If the map outputs are compressed, how should they be + compressed? + + + + + map.sort.class + org.apache.hadoop.util.QuickSort + The default sort class for sorting keys. + + + + + mapreduce.task.userlog.limit.kb + 0 + The maximum size of user-logs of each task in KB. 0 disables the cap. + + + + + yarn.app.mapreduce.am.container.log.limit.kb + 0 + The maximum size of the MRAppMaster attempt container logs in KB. + 0 disables the cap. + + + + + yarn.app.mapreduce.task.container.log.backups + 0 + Number of backup files for task logs when using + ContainerRollingLogAppender (CRLA). See + org.apache.log4j.RollingFileAppender.maxBackupIndex. By default, + ContainerLogAppender (CLA) is used, and container logs are not rolled. CRLA + is enabled for tasks when both mapreduce.task.userlog.limit.kb and + yarn.app.mapreduce.task.container.log.backups are greater than zero. + + + + + yarn.app.mapreduce.am.container.log.backups + 0 + Number of backup files for the ApplicationMaster logs when using + ContainerRollingLogAppender (CRLA). See + org.apache.log4j.RollingFileAppender.maxBackupIndex. By default, + ContainerLogAppender (CLA) is used, and container logs are not rolled. CRLA + is enabled for the ApplicationMaster when both + mapreduce.task.userlog.limit.kb and + yarn.app.mapreduce.am.container.log.backups are greater than zero. + + + + + mapreduce.job.userlog.retain.hours + 24 + The maximum time, in hours, for which the user-logs are to be + retained after the job completion. + + + + + mapreduce.jobtracker.hosts.filename + + Names a file that contains the list of nodes that may + connect to the jobtracker. If the value is empty, all hosts are + permitted. + + + + mapreduce.jobtracker.hosts.exclude.filename + + Names a file that contains the list of hosts that + should be excluded by the jobtracker. If the value is empty, no + hosts are excluded. + + + + mapreduce.jobtracker.heartbeats.in.second + 100 + Expert: Approximate number of heart-beats that could arrive + at JobTracker in a second. Assuming each RPC can be processed + in 10msec, the default value is made 100 RPCs in a second. + + + + + mapreduce.jobtracker.tasktracker.maxblacklists + 4 + The number of blacklists for a taskTracker by various jobs + after which the task tracker could be blacklisted across + all jobs. The tracker will be given a tasks later + (after a day). The tracker will become a healthy + tracker after a restart. + + + + + mapreduce.job.maxtaskfailures.per.tracker + 3 + The number of task-failures on a tasktracker of a given job + after which new tasks of that job aren't assigned to it. It + MUST be less than mapreduce.map.maxattempts and + mapreduce.reduce.maxattempts otherwise the failed task will + never be tried on a different node. + + + + + mapreduce.client.output.filter + FAILED + The filter for controlling the output of the task's userlogs sent + to the console of the JobClient. + The permissible options are: NONE, KILLED, FAILED, SUCCEEDED and + ALL. + + + + + mapreduce.client.completion.pollinterval + 5000 + The interval (in milliseconds) between which the JobClient + polls the JobTracker for updates about job status. You may want to set this + to a lower value to make tests run faster on a single node system. Adjusting + this value in production may lead to unwanted client-server traffic. + + + + + mapreduce.client.progressmonitor.pollinterval + 1000 + The interval (in milliseconds) between which the JobClient + reports status to the console and checks for job completion. You may want to set this + to a lower value to make tests run faster on a single node system. Adjusting + this value in production may lead to unwanted client-server traffic. + + + + + mapreduce.jobtracker.persist.jobstatus.active + true + Indicates if persistency of job status information is + active or not. + + + + + mapreduce.jobtracker.persist.jobstatus.hours + 1 + The number of hours job status information is persisted in DFS. + The job status information will be available after it drops of the memory + queue and between jobtracker restarts. With a zero value the job status + information is not persisted at all in DFS. + + + + + mapreduce.jobtracker.persist.jobstatus.dir + /jobtracker/jobsInfo + The directory where the job status information is persisted + in a file system to be available after it drops of the memory queue and + between jobtracker restarts. + + + + + mapreduce.task.profile + false + To set whether the system should collect profiler + information for some of the tasks in this job? The information is stored + in the user log directory. The value is "true" if task profiling + is enabled. + + + + mapreduce.task.profile.maps + 0-2 + To set the ranges of map tasks to profile. + mapreduce.task.profile has to be set to true for the value to be accounted. + + + + + mapreduce.task.profile.reduces + 0-2 + To set the ranges of reduce tasks to profile. + mapreduce.task.profile has to be set to true for the value to be accounted. + + + + + mapreduce.task.profile.params + -agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s + JVM profiler parameters used to profile map and reduce task + attempts. This string may contain a single format specifier %s that will + be replaced by the path to profile.out in the task attempt log directory. + To specify different profiling options for map tasks and reduce tasks, + more specific parameters mapreduce.task.profile.map.params and + mapreduce.task.profile.reduce.params should be used. + + + + mapreduce.task.profile.map.params + ${mapreduce.task.profile.params} + Map-task-specific JVM profiler parameters. See + mapreduce.task.profile.params + + + + mapreduce.task.profile.reduce.params + ${mapreduce.task.profile.params} + Reduce-task-specific JVM profiler parameters. See + mapreduce.task.profile.params + + + + mapreduce.task.skip.start.attempts + 2 + The number of Task attempts AFTER which skip mode + will be kicked off. When skip mode is kicked off, the + tasks reports the range of records which it will process + next, to the TaskTracker. So that on failures, TT knows which + ones are possibly the bad records. On further executions, + those are skipped. + + + + + mapreduce.map.skip.proc.count.autoincr + true + The flag which if set to true, + SkipBadRecords.COUNTER_MAP_PROCESSED_RECORDS is incremented + by MapRunner after invoking the map function. This value must be set to + false for applications which process the records asynchronously + or buffer the input records. For example streaming. + In such cases applications should increment this counter on their own. + + + + + mapreduce.reduce.skip.proc.count.autoincr + true + The flag which if set to true, + SkipBadRecords.COUNTER_REDUCE_PROCESSED_GROUPS is incremented + by framework after invoking the reduce function. This value must be set to + false for applications which process the records asynchronously + or buffer the input records. For example streaming. + In such cases applications should increment this counter on their own. + + + + + mapreduce.job.skip.outdir + + If no value is specified here, the skipped records are + written to the output directory at _logs/skip. + User can stop writing skipped records by giving the value "none". + + + + + mapreduce.map.skip.maxrecords + 0 + The number of acceptable skip records surrounding the bad + record PER bad record in mapper. The number includes the bad record as well. + To turn the feature of detection/skipping of bad records off, set the + value to 0. + The framework tries to narrow down the skipped range by retrying + until this threshold is met OR all attempts get exhausted for this task. + Set the value to Long.MAX_VALUE to indicate that framework need not try to + narrow down. Whatever records(depends on application) get skipped are + acceptable. + + + + + mapreduce.reduce.skip.maxgroups + 0 + The number of acceptable skip groups surrounding the bad + group PER bad group in reducer. The number includes the bad group as well. + To turn the feature of detection/skipping of bad groups off, set the + value to 0. + The framework tries to narrow down the skipped range by retrying + until this threshold is met OR all attempts get exhausted for this task. + Set the value to Long.MAX_VALUE to indicate that framework need not try to + narrow down. Whatever groups(depends on application) get skipped are + acceptable. + + + + + mapreduce.ifile.readahead + true + Configuration key to enable/disable IFile readahead. + + + + + mapreduce.ifile.readahead.bytes + 4194304 + Configuration key to set the IFile readahead length in bytes. + + + + + + mapreduce.jobtracker.taskcache.levels + 2 + This is the max level of the task cache. For example, if + the level is 2, the tasks cached are at the host level and at the rack + level. + + + + + mapreduce.job.queuename + default + Queue to which a job is submitted. This must match one of the + queues defined in mapred-queues.xml for the system. Also, the ACL setup + for the queue must allow the current user to submit a job to the queue. + Before specifying a queue, ensure that the system is configured with + the queue, and access is allowed for submitting jobs to the queue. + + + + + mapreduce.job.tags + + Tags for the job that will be passed to YARN at submission + time. Queries to YARN for applications can filter on these tags. + + + + + mapreduce.cluster.acls.enabled + false + Specifies whether ACLs should be checked + for authorization of users for doing various queue and job level operations. + ACLs are disabled by default. If enabled, access control checks are made by + JobTracker and TaskTracker when requests are made by users for queue + operations like submit job to a queue and kill a job in the queue and job + operations like viewing the job-details (See mapreduce.job.acl-view-job) + or for modifying the job (See mapreduce.job.acl-modify-job) using + Map/Reduce APIs, RPCs or via the console and web user interfaces. + For enabling this flag(mapreduce.cluster.acls.enabled), this is to be set + to true in mapred-site.xml on JobTracker node and on all TaskTracker nodes. + + + + + mapreduce.job.acl-modify-job + + Job specific access-control list for 'modifying' the job. It + is only used if authorization is enabled in Map/Reduce by setting the + configuration property mapreduce.cluster.acls.enabled to true. + This specifies the list of users and/or groups who can do modification + operations on the job. For specifying a list of users and groups the + format to use is "user1,user2 group1,group". If set to '*', it allows all + users/groups to modify this job. If set to ' '(i.e. space), it allows + none. This configuration is used to guard all the modifications with respect + to this job and takes care of all the following operations: + o killing this job + o killing a task of this job, failing a task of this job + o setting the priority of this job + Each of these operations are also protected by the per-queue level ACL + "acl-administer-jobs" configured via mapred-queues.xml. So a caller should + have the authorization to satisfy either the queue-level ACL or the + job-level ACL. + + Irrespective of this ACL configuration, (a) job-owner, (b) the user who + started the cluster, (c) members of an admin configured supergroup + configured via mapreduce.cluster.permissions.supergroup and (d) queue + administrators of the queue to which this job was submitted to configured + via acl-administer-jobs for the specific queue in mapred-queues.xml can + do all the modification operations on a job. + + By default, nobody else besides job-owner, the user who started the cluster, + members of supergroup and queue administrators can perform modification + operations on a job. + + + + + mapreduce.job.acl-view-job + + Job specific access-control list for 'viewing' the job. It is + only used if authorization is enabled in Map/Reduce by setting the + configuration property mapreduce.cluster.acls.enabled to true. + This specifies the list of users and/or groups who can view private details + about the job. For specifying a list of users and groups the + format to use is "user1,user2 group1,group". If set to '*', it allows all + users/groups to modify this job. If set to ' '(i.e. space), it allows + none. This configuration is used to guard some of the job-views and at + present only protects APIs that can return possibly sensitive information + of the job-owner like + o job-level counters + o task-level counters + o tasks' diagnostic information + o task-logs displayed on the TaskTracker web-UI and + o job.xml showed by the JobTracker's web-UI + Every other piece of information of jobs is still accessible by any other + user, for e.g., JobStatus, JobProfile, list of jobs in the queue, etc. + + Irrespective of this ACL configuration, (a) job-owner, (b) the user who + started the cluster, (c) members of an admin configured supergroup + configured via mapreduce.cluster.permissions.supergroup and (d) queue + administrators of the queue to which this job was submitted to configured + via acl-administer-jobs for the specific queue in mapred-queues.xml can + do all the view operations on a job. + + By default, nobody else besides job-owner, the user who started the + cluster, memebers of supergroup and queue administrators can perform + view operations on a job. + + + + + mapreduce.tasktracker.indexcache.mb + 10 + The maximum memory that a task tracker allows for the + index cache that is used when serving map outputs to reducers. + + + + + mapreduce.job.token.tracking.ids.enabled + false + Whether to write tracking ids of tokens to + job-conf. When true, the configuration property + "mapreduce.job.token.tracking.ids" is set to the token-tracking-ids of + the job + + + + mapreduce.job.token.tracking.ids + + When mapreduce.job.token.tracking.ids.enabled is + set to true, this is set by the framework to the + token-tracking-ids used by the job. + + + + mapreduce.task.merge.progress.records + 10000 + The number of records to process during merge before + sending a progress notification to the TaskTracker. + + + + + mapreduce.task.combine.progress.records + 10000 + The number of records to process during combine output collection + before sending a progress notification. + + + + + mapreduce.job.reduce.slowstart.completedmaps + 0.05 + Fraction of the number of maps in the job which should be + complete before reduces are scheduled for the job. + + + + +mapreduce.job.complete.cancel.delegation.tokens + true + if false - do not unregister/cancel delegation tokens from + renewal, because same tokens may be used by spawned jobs + + + + + mapreduce.tasktracker.taskcontroller + org.apache.hadoop.mapred.DefaultTaskController + TaskController which is used to launch and manage task execution + + + + + mapreduce.tasktracker.group + + Expert: Group to which TaskTracker belongs. If + LinuxTaskController is configured via mapreduce.tasktracker.taskcontroller, + the group owner of the task-controller binary should be same as this group. + + + + + mapreduce.shuffle.port + 13562 + Default port that the ShuffleHandler will run on. ShuffleHandler + is a service run at the NodeManager to facilitate transfers of intermediate + Map outputs to requesting Reducers. + + + + + mapreduce.job.reduce.shuffle.consumer.plugin.class + org.apache.hadoop.mapreduce.task.reduce.Shuffle + + Name of the class whose instance will be used + to send shuffle requests by reducetasks of this job. + The class must be an instance of org.apache.hadoop.mapred.ShuffleConsumerPlugin. + + + + + + + mapreduce.tasktracker.healthchecker.script.path + + Absolute path to the script which is + periodicallyrun by the node health monitoring service to determine if + the node is healthy or not. If the value of this key is empty or the + file does not exist in the location configured here, the node health + monitoring service is not started. + + + + mapreduce.tasktracker.healthchecker.interval + 60000 + Frequency of the node health script to be run, + in milliseconds + + + + mapreduce.tasktracker.healthchecker.script.timeout + 600000 + Time after node health script should be killed if + unresponsive and considered that the script has failed. + + + + mapreduce.tasktracker.healthchecker.script.args + + List of arguments which are to be passed to + node health script when it is being launched comma seperated. + + + + + + + + + mapreduce.job.counters.limit + 120 + Limit on the number of user counters allowed per job. + + + + + mapreduce.framework.name + local + The runtime framework for executing MapReduce jobs. + Can be one of local, classic or yarn. + + + + + yarn.app.mapreduce.am.staging-dir + /tmp/hadoop-yarn/staging + The staging dir used while submitting jobs. + + + + + mapreduce.am.max-attempts + 2 + The maximum number of application attempts. It is a + application-specific setting. It should not be larger than the global number + set by resourcemanager. Otherwise, it will be override. The default number is + set to 2, to allow at least one retry for AM. + + + + + mapreduce.job.end-notification.url + + Indicates url which will be called on completion of job to inform + end status of job. + User can give at most 2 variables with URI : $jobId and $jobStatus. + If they are present in URI, then they will be replaced by their + respective values. + + + + + mapreduce.job.end-notification.retry.attempts + 0 + The number of times the submitter of the job wants to retry job + end notification if it fails. This is capped by + mapreduce.job.end-notification.max.attempts + + + + mapreduce.job.end-notification.retry.interval + 1000 + The number of milliseconds the submitter of the job wants to + wait before job end notification is retried if it fails. This is capped by + mapreduce.job.end-notification.max.retry.interval + + + + mapreduce.job.end-notification.max.attempts + 5 + true + The maximum number of times a URL will be read for providing job + end notification. Cluster administrators can set this to limit how long + after end of a job, the Application Master waits before exiting. Must be + marked as final to prevent users from overriding this. + + + + + mapreduce.job.end-notification.max.retry.interval + 5000 + true + The maximum amount of time (in milliseconds) to wait before + retrying job end notification. Cluster administrators can set this to + limit how long the Application Master waits before exiting. Must be marked + as final to prevent users from overriding this. + + + + yarn.app.mapreduce.am.env + + User added environment variables for the MR App Master + processes. Example : + 1) A=foo This will set the env variable A to foo + 2) B=$B:c This is inherit tasktracker's B env variable. + + + + + yarn.app.mapreduce.am.admin.user.env + + Environment variables for the MR App Master + processes for admin purposes. These values are set first and can be + overridden by the user env (yarn.app.mapreduce.am.env) Example : + 1) A=foo This will set the env variable A to foo + 2) B=$B:c This is inherit app master's B env variable. + + + + + yarn.app.mapreduce.am.command-opts + -Xmx1024m + Java opts for the MR App Master processes. + The following symbol, if present, will be interpolated: @taskid@ is replaced + by current TaskID. Any other occurrences of '@' will go unchanged. + For example, to enable verbose gc logging to a file named for the taskid in + /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: + -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc + + Usage of -Djava.library.path can cause programs to no longer function if + hadoop native libraries are used. These values should instead be set as part + of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and + mapreduce.reduce.env config settings. + + + + + yarn.app.mapreduce.am.admin-command-opts + + Java opts for the MR App Master processes for admin purposes. + It will appears before the opts set by yarn.app.mapreduce.am.command-opts and + thus its options can be overridden user. + + Usage of -Djava.library.path can cause programs to no longer function if + hadoop native libraries are used. These values should instead be set as part + of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and + mapreduce.reduce.env config settings. + + + + + yarn.app.mapreduce.am.job.task.listener.thread-count + 30 + The number of threads used to handle RPC calls in the + MR AppMaster from remote tasks + + + + yarn.app.mapreduce.am.job.client.port-range + + Range of ports that the MapReduce AM can use when binding. + Leave blank if you want all possible ports. + For example 50000-50050,50100-50200 + + + + yarn.app.mapreduce.am.job.committer.cancel-timeout + 60000 + The amount of time in milliseconds to wait for the output + committer to cancel an operation if the job is killed + + + + yarn.app.mapreduce.am.job.committer.commit-window + 10000 + Defines a time window in milliseconds for output commit + operations. If contact with the RM has occurred within this window then + commits are allowed, otherwise the AM will not allow output commits until + contact with the RM has been re-established. + + + + yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms + 1000 + The interval in ms at which the MR AppMaster should send + heartbeats to the ResourceManager + + + + yarn.app.mapreduce.client-am.ipc.max-retries + 3 + The number of client retries to the AM - before reconnecting + to the RM to fetch Application Status. + + + + yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts + 3 + The number of client retries on socket timeouts to the AM - before + reconnecting to the RM to fetch Application Status. + + + + yarn.app.mapreduce.client.max-retries + 3 + The number of client retries to the RM/HS before + throwing exception. This is a layer above the ipc. + + + + yarn.app.mapreduce.am.resource.mb + 1536 + The amount of memory the MR AppMaster needs. + + + + yarn.app.mapreduce.am.resource.cpu-vcores + 1 + + The number of virtual CPU cores the MR AppMaster needs. + + + + + CLASSPATH for MR applications. A comma-separated list + of CLASSPATH entries. If mapreduce.application.framework is set then this + must specify the appropriate classpath for that archive, and the name of + the archive must be present in the classpath. + If mapreduce.app-submission.cross-platform is false, platform-specific + environment vairable expansion syntax would be used to construct the default + CLASSPATH entries. + For Linux: + $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*, + $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*. + For Windows: + %HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/*, + %HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/lib/*. + + If mapreduce.app-submission.cross-platform is true, platform-agnostic default + CLASSPATH for MR applications would be used: + {{HADOOP_MAPRED_HOME}}/share/hadoop/mapreduce/*, + {{HADOOP_MAPRED_HOME}}/share/hadoop/mapreduce/lib/* + Parameter expansion marker will be replaced by NodeManager on container + launch based on the underlying OS accordingly. + + mapreduce.application.classpath + + + + + If enabled, user can submit an application cross-platform + i.e. submit an application from a Windows client to a Linux/Unix server or + vice versa. + + mapreduce.app-submission.cross-platform + false + + + + Path to the MapReduce framework archive. If set, the framework + archive will automatically be distributed along with the job, and this + path would normally reside in a public location in an HDFS filesystem. As + with distributed cache files, this can be a URL with a fragment specifying + the alias to use for the archive name. For example, + hdfs:/mapred/framework/hadoop-mapreduce-2.1.1.tar.gz#mrframework would + alias the localized archive as "mrframework". + + Note that mapreduce.application.classpath must include the appropriate + classpath for the specified framework. The base name of the archive, or + alias of the archive if an alias is used, must appear in the specified + classpath. + + mapreduce.application.framework.path + + + + + mapreduce.job.classloader + false + Whether to use a separate (isolated) classloader for + user classes in the task JVM. + + + + mapreduce.job.classloader.system.classes + + Used to override the default definition of the system classes for + the job classloader. The system classes are a comma-separated list of + classes that should be loaded from the system classpath, not the + user-supplied JARs, when mapreduce.job.classloader is enabled. Names ending + in '.' (period) are treated as package names, and names starting with a '-' + are treated as negative matches. + + + + + + + mapreduce.jobhistory.address + 0.0.0.0:10020 + MapReduce JobHistory Server IPC host:port + + + + mapreduce.jobhistory.webapp.address + 0.0.0.0:19888 + MapReduce JobHistory Server Web UI host:port + + + + mapreduce.jobhistory.keytab + + Location of the kerberos keytab file for the MapReduce + JobHistory Server. + + /etc/security/keytab/jhs.service.keytab + + + + mapreduce.jobhistory.principal + + Kerberos principal name for the MapReduce JobHistory Server. + + jhs/_HOST@REALM.TLD + + + + mapreduce.jobhistory.intermediate-done-dir + ${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate + + + + + mapreduce.jobhistory.done-dir + ${yarn.app.mapreduce.am.staging-dir}/history/done + + + + + mapreduce.jobhistory.cleaner.enable + true + + + + + mapreduce.jobhistory.cleaner.interval-ms + 86400000 + How often the job history cleaner checks for files to delete, + in milliseconds. Defaults to 86400000 (one day). Files are only deleted if + they are older than mapreduce.jobhistory.max-age-ms. + + + + + mapreduce.jobhistory.max-age-ms + 604800000 + Job history files older than this many milliseconds will + be deleted when the history cleaner runs. Defaults to 604800000 (1 week). + + + + + mapreduce.jobhistory.client.thread-count + 10 + The number of threads to handle client API requests + + + + mapreduce.jobhistory.datestring.cache.size + 200000 + Size of the date string cache. Effects the number of directories + which will be scanned to find a job. + + + + mapreduce.jobhistory.joblist.cache.size + 20000 + Size of the job list cache + + + + mapreduce.jobhistory.loadedjobs.cache.size + 5 + Size of the loaded job cache + + + + mapreduce.jobhistory.move.interval-ms + 180000 + Scan for history files to more from intermediate done dir to done + dir at this frequency. + + + + + mapreduce.jobhistory.move.thread-count + 3 + The number of threads used to move files. + + + + mapreduce.jobhistory.store.class + + The HistoryStorage class to use to cache history data. + + + + mapreduce.jobhistory.minicluster.fixed.ports + false + Whether to use fixed ports with the minicluster + + + + mapreduce.jobhistory.admin.address + 0.0.0.0:10033 + The address of the History server admin interface. + + + + mapreduce.jobhistory.admin.acl + * + ACL of who can be admin of the History server. + + + + mapreduce.jobhistory.recovery.enable + false + Enable the history server to store server state and recover + server state upon startup. If enabled then + mapreduce.jobhistory.recovery.store.class must be specified. + + + + mapreduce.jobhistory.recovery.store.class + org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService + The HistoryServerStateStoreService class to store history server + state for recovery. + + + + mapreduce.jobhistory.recovery.store.fs.uri + ${hadoop.tmp.dir}/mapred/history/recoverystore + + The URI where history server state will be stored if + HistoryServerFileSystemStateStoreService is configured as the recovery + storage class. + + + + mapreduce.jobhistory.http.policy + HTTP_ONLY + + This configures the HTTP endpoint for JobHistoryServer web UI. + The following values are supported: + - HTTP_ONLY : Service is provided only on http + - HTTPS_ONLY : Service is provided only on https + + + diff --git a/sahara/plugins/vanilla/v2_6_0/resources/oozie-default.xml b/sahara/plugins/vanilla/v2_6_0/resources/oozie-default.xml new file mode 100644 index 0000000000..455ef9d924 --- /dev/null +++ b/sahara/plugins/vanilla/v2_6_0/resources/oozie-default.xml @@ -0,0 +1,1929 @@ + + + + + + + + + + + + oozie.action.ship.launcher.jar + true + + If true, Oozie will create and ship a "launcher jar" that contains classes necessary for the launcher job. If false, + Oozie will not do this, and it is assumed that the necessary classes are in their respective sharelib jars or the + "oozie" sharelib instead. When false, the sharelib is required for ALL actions; when true, the sharelib is only + required for actions that need additional jars (e.g. Pig). The main advantage of setting this to false is that + launching jobs should be slightly faster. + + + + + oozie.action.mapreduce.uber.jar.enable + false + + If true, enables the oozie.mapreduce.uber.jar mapreduce workflow configuration property, which is used to specify an + uber jar in HDFS. Submitting a workflow with an uber jar requires at least Hadoop 2.2.0 or 1.2.0. If false, workflows + which specify the oozie.mapreduce.uber.jar configuration property will fail. + + + + + oozie.processing.timezone + UTC + + Oozie server timezone. Valid values are UTC and GMT(+/-)####, for example 'GMT+0530' would be India + timezone. All dates parsed and genered dates by Oozie Coordinator/Bundle will be done in the specified + timezone. The default value of 'UTC' should not be changed under normal circumtances. If for any reason + is changed, note that GMT(+/-)#### timezones do not observe DST changes. + + + + + + + oozie.base.url + http://localhost:8080/oozie + + Base Oozie URL. + + + + + + + oozie.system.id + oozie-${user.name} + + The Oozie system ID. + + + + + oozie.systemmode + NORMAL + + System mode for Oozie at startup. + + + + + oozie.delete.runtime.dir.on.shutdown + true + + If the runtime directory should be kept after Oozie shutdowns down. + + + + + oozie.services + + org.apache.oozie.service.SchedulerService, + org.apache.oozie.service.InstrumentationService, + org.apache.oozie.service.CallableQueueService, + org.apache.oozie.service.UUIDService, + org.apache.oozie.service.ELService, + org.apache.oozie.service.AuthorizationService, + org.apache.oozie.service.UserGroupInformationService, + org.apache.oozie.service.HadoopAccessorService, + org.apache.oozie.service.URIHandlerService, + org.apache.oozie.service.MemoryLocksService, + org.apache.oozie.service.DagXLogInfoService, + org.apache.oozie.service.SchemaService, + org.apache.oozie.service.LiteWorkflowAppService, + org.apache.oozie.service.JPAService, + org.apache.oozie.service.StoreService, + org.apache.oozie.service.CoordinatorStoreService, + org.apache.oozie.service.SLAStoreService, + org.apache.oozie.service.DBLiteWorkflowStoreService, + org.apache.oozie.service.CallbackService, + org.apache.oozie.service.ActionService, + org.apache.oozie.service.ActionCheckerService, + org.apache.oozie.service.RecoveryService, + org.apache.oozie.service.PurgeService, + org.apache.oozie.service.CoordinatorEngineService, + org.apache.oozie.service.BundleEngineService, + org.apache.oozie.service.DagEngineService, + org.apache.oozie.service.CoordMaterializeTriggerService, + org.apache.oozie.service.StatusTransitService, + org.apache.oozie.service.PauseTransitService, + org.apache.oozie.service.GroupsService, + org.apache.oozie.service.ProxyUserService + + + All services to be created and managed by Oozie Services singleton. + Class names must be separated by commas. + + + + + oozie.services.ext + + + To add/replace services defined in 'oozie.services' with custom implementations. + Class names must be separated by commas. + + + + + + oozie.service.HCatAccessorService.jmsconnections + + default=java.naming.factory.initial#org.apache.activemq.jndi.ActiveMQInitialContextFactory;java.naming.provider.url#tcp://localhost:61616;connectionFactoryNames#ConnectionFactory + + + Specify the map of endpoints to JMS configuration properties. In general, endpoint + identifies the HCatalog server URL. "default" is used if no endpoint is mentioned + in the query. If some JMS property is not defined, the system will use the property + defined jndi.properties. jndi.properties files is retrieved from the application classpath. + Mapping rules can also be provided for mapping Hcatalog servers to corresponding JMS providers. + hcat://${1}.${2}.server.com:8020=java.naming.factory.initial#Dummy.Factory;java.naming.provider.url#tcp://broker.${2}:61616 + + + + + + + oozie.service.JMSTopicService.topic.name + + default=${username} + + + Topic options are ${username} or ${jobId} or a fixed string which can be specified as default or for a + particular job type. + For e.g To have a fixed string topic for workflows, coordinators and bundles, + specify in the following comma-separated format: {jobtype1}={some_string1}, {jobtype2}={some_string2} + where job type can be WORKFLOW, COORDINATOR or BUNDLE. + e.g. Following defines topic for workflow job, workflow action, coordinator job, coordinator action, + bundle job and bundle action + WORKFLOW=workflow, + COORDINATOR=coordinator, + BUNDLE=bundle + For jobs with no defined topic, default topic will be ${username} + + + + + + oozie.jms.producer.connection.properties + java.naming.factory.initial#org.apache.activemq.jndi.ActiveMQInitialContextFactory;java.naming.provider.url#tcp://localhost:61616;connectionFactoryNames#ConnectionFactory + + + + + oozie.service.JMSAccessorService.connectioncontext.impl + + org.apache.oozie.jms.DefaultConnectionContext + + + Specifies the Connection Context implementation + + + + + + + + oozie.service.ConfigurationService.ignore.system.properties + + oozie.service.AuthorizationService.security.enabled + + + Specifies "oozie.*" properties to cannot be overriden via Java system properties. + Property names must be separted by commas. + + + + + + + oozie.service.SchedulerService.threads + 10 + + The number of threads to be used by the SchedulerService to run deamon tasks. + If maxed out, scheduled daemon tasks will be queued up and delayed until threads become available. + + + + + + + oozie.service.AuthorizationService.authorization.enabled + false + + Specifies whether security (user name/admin role) is enabled or not. + If disabled any user can manage Oozie system and manage any job. + + + + + oozie.service.AuthorizationService.default.group.as.acl + false + + Enables old behavior where the User's default group is the job's ACL. + + + + + + + oozie.service.InstrumentationService.logging.interval + 60 + + Interval, in seconds, at which instrumentation should be logged by the InstrumentationService. + If set to 0 it will not log instrumentation data. + + + + + + oozie.service.PurgeService.older.than + 30 + + Completed workflow jobs older than this value, in days, will be purged by the PurgeService. + + + + + oozie.service.PurgeService.coord.older.than + 7 + + Completed coordinator jobs older than this value, in days, will be purged by the PurgeService. + + + + + oozie.service.PurgeService.bundle.older.than + 7 + + Completed bundle jobs older than this value, in days, will be purged by the PurgeService. + + + + + oozie.service.PurgeService.purge.limit + 100 + + Completed Actions purge - limit each purge to this value + + + + + oozie.service.PurgeService.purge.interval + 3600 + + Interval at which the purge service will run, in seconds. + + + + + + + oozie.service.RecoveryService.wf.actions.older.than + 120 + + Age of the actions which are eligible to be queued for recovery, in seconds. + + + + + oozie.service.RecoveryService.callable.batch.size + 10 + + This value determines the number of callable which will be batched together + to be executed by a single thread. + + + + + oozie.service.RecoveryService.push.dependency.interval + 200 + + This value determines the delay for push missing dependency command queueing + in Recovery Service + + + + + oozie.service.RecoveryService.interval + 60 + + Interval at which the RecoverService will run, in seconds. + + + + + oozie.service.RecoveryService.coord.older.than + 600 + + Age of the Coordinator jobs or actions which are eligible to be queued for recovery, in seconds. + + + + + oozie.service.RecoveryService.bundle.older.than + 600 + + Age of the Bundle jobs which are eligible to be queued for recovery, in seconds. + + + + + + + oozie.service.CallableQueueService.queue.size + 10000 + Max callable queue size + + + + oozie.service.CallableQueueService.threads + 10 + Number of threads used for executing callables + + + + oozie.service.CallableQueueService.callable.concurrency + 3 + + Maximum concurrency for a given callable type. + Each command is a callable type (submit, start, run, signal, job, jobs, suspend,resume, etc). + Each action type is a callable type (Map-Reduce, Pig, SSH, FS, sub-workflow, etc). + All commands that use action executors (action-start, action-end, action-kill and action-check) use + the action type as the callable type. + + + + + oozie.service.CallableQueueService.callable.next.eligible + true + + If true, when a callable in the queue has already reached max concurrency, + Oozie continuously find next one which has not yet reach max concurrency. + + + + + oozie.service.CallableQueueService.InterruptMapMaxSize + 500 + + Maximum Size of the Interrupt Map, the interrupt element will not be inserted in the map if exceeded the size. + + + + + oozie.service.CallableQueueService.InterruptTypes + kill,resume,suspend,bundle_kill,bundle_resume,bundle_suspend,coord_kill,coord_change,coord_resume,coord_suspend + + Getting the types of XCommands that are considered to be of Interrupt type + + + + + + + oozie.service.CoordMaterializeTriggerService.lookup.interval + + 300 + Coordinator Job Lookup trigger command is scheduled at + this "interval" (in seconds). + + + + oozie.service.CoordMaterializeTriggerService.materialization.window + + 3600 + Coordinator Job Lookup command materialized each job for + this next "window" duration + + + + oozie.service.CoordMaterializeTriggerService.callable.batch.size + 10 + + This value determines the number of callable which will be batched together + to be executed by a single thread. + + + + + oozie.service.CoordMaterializeTriggerService.materialization.system.limit + 50 + + This value determines the number of coordinator jobs to be materialized at a given time. + + + + + oozie.service.coord.normal.default.timeout + + 10080 + Default timeout for a coordinator action input check (in minutes) for normal job. + + + + + oozie.service.coord.default.max.timeout + + 86400 + Default maximum timeout for a coordinator action input check (in minutes). 86400= 60days + + + + + oozie.service.coord.input.check.requeue.interval + + 60000 + Command re-queue interval for coordinator data input check (in millisecond). + + + + + oozie.service.coord.push.check.requeue.interval + + 600000 + Command re-queue interval for push dependencies (in millisecond). + + + + + oozie.service.coord.default.concurrency + + 1 + Default concurrency for a coordinator job to determine how many maximum action should + be executed at the same time. -1 means infinite concurrency. + + + + oozie.service.coord.default.throttle + + 12 + Default throttle for a coordinator job to determine how many maximum action should + be in WAITING state at the same time. + + + + oozie.service.coord.materialization.throttling.factor + + 0.05 + Determine how many maximum actions should be in WAITING state for a single job at any time. The value is calculated by + this factor X the total queue size. + + + + + + oozie.service.ELService.groups + job-submit,workflow,wf-sla-submit,coord-job-submit-freq,coord-job-submit-nofuncs,coord-job-submit-data,coord-job-submit-instances,coord-sla-submit,coord-action-create,coord-action-create-inst,coord-sla-create,coord-action-start + List of groups for different ELServices + + + + oozie.service.ELService.constants.job-submit + + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + + + + + oozie.service.ELService.functions.job-submit + + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + + + + + + oozie.service.ELService.constants.workflow + + KB=org.apache.oozie.util.ELConstantsFunctions#KB, + MB=org.apache.oozie.util.ELConstantsFunctions#MB, + GB=org.apache.oozie.util.ELConstantsFunctions#GB, + TB=org.apache.oozie.util.ELConstantsFunctions#TB, + PB=org.apache.oozie.util.ELConstantsFunctions#PB, + RECORDS=org.apache.oozie.action.hadoop.HadoopELFunctions#RECORDS, + MAP_IN=org.apache.oozie.action.hadoop.HadoopELFunctions#MAP_IN, + MAP_OUT=org.apache.oozie.action.hadoop.HadoopELFunctions#MAP_OUT, + REDUCE_IN=org.apache.oozie.action.hadoop.HadoopELFunctions#REDUCE_IN, + REDUCE_OUT=org.apache.oozie.action.hadoop.HadoopELFunctions#REDUCE_OUT, + GROUPS=org.apache.oozie.action.hadoop.HadoopELFunctions#GROUPS + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + + + + + oozie.service.ELService.ext.constants.workflow + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + oozie.service.ELService.functions.workflow + + firstNotNull=org.apache.oozie.util.ELConstantsFunctions#firstNotNull, + concat=org.apache.oozie.util.ELConstantsFunctions#concat, + replaceAll=org.apache.oozie.util.ELConstantsFunctions#replaceAll, + appendAll=org.apache.oozie.util.ELConstantsFunctions#appendAll, + trim=org.apache.oozie.util.ELConstantsFunctions#trim, + timestamp=org.apache.oozie.util.ELConstantsFunctions#timestamp, + urlEncode=org.apache.oozie.util.ELConstantsFunctions#urlEncode, + toJsonStr=org.apache.oozie.util.ELConstantsFunctions#toJsonStr, + toPropertiesStr=org.apache.oozie.util.ELConstantsFunctions#toPropertiesStr, + toConfigurationStr=org.apache.oozie.util.ELConstantsFunctions#toConfigurationStr, + wf:id=org.apache.oozie.DagELFunctions#wf_id, + wf:name=org.apache.oozie.DagELFunctions#wf_name, + wf:appPath=org.apache.oozie.DagELFunctions#wf_appPath, + wf:conf=org.apache.oozie.DagELFunctions#wf_conf, + wf:user=org.apache.oozie.DagELFunctions#wf_user, + wf:group=org.apache.oozie.DagELFunctions#wf_group, + wf:callback=org.apache.oozie.DagELFunctions#wf_callback, + wf:transition=org.apache.oozie.DagELFunctions#wf_transition, + wf:lastErrorNode=org.apache.oozie.DagELFunctions#wf_lastErrorNode, + wf:errorCode=org.apache.oozie.DagELFunctions#wf_errorCode, + wf:errorMessage=org.apache.oozie.DagELFunctions#wf_errorMessage, + wf:run=org.apache.oozie.DagELFunctions#wf_run, + wf:actionData=org.apache.oozie.DagELFunctions#wf_actionData, + wf:actionExternalId=org.apache.oozie.DagELFunctions#wf_actionExternalId, + wf:actionTrackerUri=org.apache.oozie.DagELFunctions#wf_actionTrackerUri, + wf:actionExternalStatus=org.apache.oozie.DagELFunctions#wf_actionExternalStatus, + hadoop:counters=org.apache.oozie.action.hadoop.HadoopELFunctions#hadoop_counters, + fs:exists=org.apache.oozie.action.hadoop.FsELFunctions#fs_exists, + fs:isDir=org.apache.oozie.action.hadoop.FsELFunctions#fs_isDir, + fs:dirSize=org.apache.oozie.action.hadoop.FsELFunctions#fs_dirSize, + fs:fileSize=org.apache.oozie.action.hadoop.FsELFunctions#fs_fileSize, + fs:blockSize=org.apache.oozie.action.hadoop.FsELFunctions#fs_blockSize, + hcat:exists=org.apache.oozie.coord.HCatELFunctions#hcat_exists + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + + + + + oozie.service.WorkflowAppService.WorkflowDefinitionMaxLength + 100000 + + The maximum length of the workflow definition in bytes + An error will be reported if the length exceeds the given maximum + + + + + oozie.service.ELService.ext.functions.workflow + + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + + oozie.service.ELService.constants.wf-sla-submit + + MINUTES=org.apache.oozie.util.ELConstantsFunctions#SUBMIT_MINUTES, + HOURS=org.apache.oozie.util.ELConstantsFunctions#SUBMIT_HOURS, + DAYS=org.apache.oozie.util.ELConstantsFunctions#SUBMIT_DAYS + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + + + + + oozie.service.ELService.ext.constants.wf-sla-submit + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + oozie.service.ELService.functions.wf-sla-submit + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + + + + oozie.service.ELService.ext.functions.wf-sla-submit + + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + +l + + + + oozie.service.ELService.constants.coord-job-submit-freq + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + + + + + oozie.service.ELService.ext.constants.coord-job-submit-freq + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + oozie.service.ELService.functions.coord-job-submit-freq + + coord:days=org.apache.oozie.coord.CoordELFunctions#ph1_coord_days, + coord:months=org.apache.oozie.coord.CoordELFunctions#ph1_coord_months, + coord:hours=org.apache.oozie.coord.CoordELFunctions#ph1_coord_hours, + coord:minutes=org.apache.oozie.coord.CoordELFunctions#ph1_coord_minutes, + coord:endOfDays=org.apache.oozie.coord.CoordELFunctions#ph1_coord_endOfDays, + coord:endOfMonths=org.apache.oozie.coord.CoordELFunctions#ph1_coord_endOfMonths, + coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, + coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + + + + + oozie.service.ELService.ext.functions.coord-job-submit-freq + + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + + oozie.service.ELService.constants.coord-job-submit-nofuncs + + MINUTE=org.apache.oozie.coord.CoordELConstants#SUBMIT_MINUTE, + HOUR=org.apache.oozie.coord.CoordELConstants#SUBMIT_HOUR, + DAY=org.apache.oozie.coord.CoordELConstants#SUBMIT_DAY, + MONTH=org.apache.oozie.coord.CoordELConstants#SUBMIT_MONTH, + YEAR=org.apache.oozie.coord.CoordELConstants#SUBMIT_YEAR + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + + + + + oozie.service.ELService.ext.constants.coord-job-submit-nofuncs + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + oozie.service.ELService.functions.coord-job-submit-nofuncs + + coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, + coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + + + + + oozie.service.ELService.ext.functions.coord-job-submit-nofuncs + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + + oozie.service.ELService.constants.coord-job-submit-instances + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + + + + + oozie.service.ELService.ext.constants.coord-job-submit-instances + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + oozie.service.ELService.functions.coord-job-submit-instances + + coord:hoursInDay=org.apache.oozie.coord.CoordELFunctions#ph1_coord_hoursInDay_echo, + coord:daysInMonth=org.apache.oozie.coord.CoordELFunctions#ph1_coord_daysInMonth_echo, + coord:tzOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_tzOffset_echo, + coord:current=org.apache.oozie.coord.CoordELFunctions#ph1_coord_current_echo, + coord:currentRange=org.apache.oozie.coord.CoordELFunctions#ph1_coord_currentRange_echo, + coord:offset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_offset_echo, + coord:latest=org.apache.oozie.coord.CoordELFunctions#ph1_coord_latest_echo, + coord:latestRange=org.apache.oozie.coord.CoordELFunctions#ph1_coord_latestRange_echo, + coord:future=org.apache.oozie.coord.CoordELFunctions#ph1_coord_future_echo, + coord:futureRange=org.apache.oozie.coord.CoordELFunctions#ph1_coord_futureRange_echo, + coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo, + coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, + coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + + + + + oozie.service.ELService.ext.functions.coord-job-submit-instances + + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + + + oozie.service.ELService.constants.coord-job-submit-data + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + + + + + oozie.service.ELService.ext.constants.coord-job-submit-data + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + oozie.service.ELService.functions.coord-job-submit-data + + coord:dataIn=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dataIn_echo, + coord:dataOut=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dataOut_echo, + coord:nominalTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_nominalTime_echo_wrap, + coord:actualTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_actualTime_echo_wrap, + coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph1_coord_dateOffset_echo, + coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo, + coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph1_coord_actionId_echo, + coord:name=org.apache.oozie.coord.CoordELFunctions#ph1_coord_name_echo, + coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, + coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user, + coord:databaseIn=org.apache.oozie.coord.HCatELFunctions#ph1_coord_databaseIn_echo, + coord:databaseOut=org.apache.oozie.coord.HCatELFunctions#ph1_coord_databaseOut_echo, + coord:tableIn=org.apache.oozie.coord.HCatELFunctions#ph1_coord_tableIn_echo, + coord:tableOut=org.apache.oozie.coord.HCatELFunctions#ph1_coord_tableOut_echo, + coord:dataInPartitionFilter=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataInPartitionFilter_echo, + coord:dataInPartitionMin=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataInPartitionMin_echo, + coord:dataInPartitionMax=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataInPartitionMax_echo, + coord:dataOutPartitions=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataOutPartitions_echo, + coord:dataOutPartitionValue=org.apache.oozie.coord.HCatELFunctions#ph1_coord_dataOutPartitionValue_echo + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + + + + + oozie.service.ELService.ext.functions.coord-job-submit-data + + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + + oozie.service.ELService.constants.coord-sla-submit + + MINUTES=org.apache.oozie.coord.CoordELConstants#SUBMIT_MINUTES, + HOURS=org.apache.oozie.coord.CoordELConstants#SUBMIT_HOURS, + DAYS=org.apache.oozie.coord.CoordELConstants#SUBMIT_DAYS + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + + + + + oozie.service.ELService.ext.constants.coord-sla-submit + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + oozie.service.ELService.functions.coord-sla-submit + + coord:nominalTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_nominalTime_echo_fixed, + coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, + coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + + + + oozie.service.ELService.ext.functions.coord-sla-submit + + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + + oozie.service.ELService.constants.coord-action-create + + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + + + + + oozie.service.ELService.ext.constants.coord-action-create + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + oozie.service.ELService.functions.coord-action-create + + coord:hoursInDay=org.apache.oozie.coord.CoordELFunctions#ph2_coord_hoursInDay, + coord:daysInMonth=org.apache.oozie.coord.CoordELFunctions#ph2_coord_daysInMonth, + coord:tzOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_tzOffset, + coord:current=org.apache.oozie.coord.CoordELFunctions#ph2_coord_current, + coord:currentRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_currentRange, + coord:offset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_offset, + coord:latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo, + coord:latestRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latestRange_echo, + coord:future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo, + coord:futureRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_futureRange_echo, + coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph2_coord_actionId, + coord:name=org.apache.oozie.coord.CoordELFunctions#ph2_coord_name, + coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime, + coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, + coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + + + + + oozie.service.ELService.ext.functions.coord-action-create + + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + + + oozie.service.ELService.constants.coord-action-create-inst + + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + + + + + oozie.service.ELService.ext.constants.coord-action-create-inst + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + oozie.service.ELService.functions.coord-action-create-inst + + coord:hoursInDay=org.apache.oozie.coord.CoordELFunctions#ph2_coord_hoursInDay, + coord:daysInMonth=org.apache.oozie.coord.CoordELFunctions#ph2_coord_daysInMonth, + coord:tzOffset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_tzOffset, + coord:current=org.apache.oozie.coord.CoordELFunctions#ph2_coord_current_echo, + coord:currentRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_currentRange_echo, + coord:offset=org.apache.oozie.coord.CoordELFunctions#ph2_coord_offset_echo, + coord:latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo, + coord:latestRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latestRange_echo, + coord:future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo, + coord:futureRange=org.apache.oozie.coord.CoordELFunctions#ph2_coord_futureRange_echo, + coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime, + coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, + coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + + + + + oozie.service.ELService.ext.functions.coord-action-create-inst + + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + + oozie.service.ELService.constants.coord-sla-create + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + + + + + oozie.service.ELService.ext.constants.coord-sla-create + + MINUTES=org.apache.oozie.coord.CoordELConstants#SUBMIT_MINUTES, + HOURS=org.apache.oozie.coord.CoordELConstants#SUBMIT_HOURS, + DAYS=org.apache.oozie.coord.CoordELConstants#SUBMIT_DAYS + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + oozie.service.ELService.functions.coord-sla-create + + coord:nominalTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_nominalTime, + coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, + coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + + + + oozie.service.ELService.ext.functions.coord-sla-create + + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + + oozie.service.ELService.constants.coord-action-start + + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + + + + + oozie.service.ELService.ext.constants.coord-action-start + + + EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + oozie.service.ELService.functions.coord-action-start + + coord:hoursInDay=org.apache.oozie.coord.CoordELFunctions#ph3_coord_hoursInDay, + coord:daysInMonth=org.apache.oozie.coord.CoordELFunctions#ph3_coord_daysInMonth, + coord:tzOffset=org.apache.oozie.coord.CoordELFunctions#ph3_coord_tzOffset, + coord:latest=org.apache.oozie.coord.CoordELFunctions#ph3_coord_latest, + coord:latestRange=org.apache.oozie.coord.CoordELFunctions#ph3_coord_latestRange, + coord:future=org.apache.oozie.coord.CoordELFunctions#ph3_coord_future, + coord:futureRange=org.apache.oozie.coord.CoordELFunctions#ph3_coord_futureRange, + coord:dataIn=org.apache.oozie.coord.CoordELFunctions#ph3_coord_dataIn, + coord:dataOut=org.apache.oozie.coord.CoordELFunctions#ph3_coord_dataOut, + coord:nominalTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_nominalTime, + coord:actualTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_actualTime, + coord:dateOffset=org.apache.oozie.coord.CoordELFunctions#ph3_coord_dateOffset, + coord:formatTime=org.apache.oozie.coord.CoordELFunctions#ph3_coord_formatTime, + coord:actionId=org.apache.oozie.coord.CoordELFunctions#ph3_coord_actionId, + coord:name=org.apache.oozie.coord.CoordELFunctions#ph3_coord_name, + coord:conf=org.apache.oozie.coord.CoordELFunctions#coord_conf, + coord:user=org.apache.oozie.coord.CoordELFunctions#coord_user, + coord:databaseIn=org.apache.oozie.coord.HCatELFunctions#ph3_coord_databaseIn, + coord:databaseOut=org.apache.oozie.coord.HCatELFunctions#ph3_coord_databaseOut, + coord:tableIn=org.apache.oozie.coord.HCatELFunctions#ph3_coord_tableIn, + coord:tableOut=org.apache.oozie.coord.HCatELFunctions#ph3_coord_tableOut, + coord:dataInPartitionFilter=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataInPartitionFilter, + coord:dataInPartitionMin=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataInPartitionMin, + coord:dataInPartitionMax=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataInPartitionMax, + coord:dataOutPartitions=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataOutPartitions, + coord:dataOutPartitionValue=org.apache.oozie.coord.HCatELFunctions#ph3_coord_dataOutPartitionValue + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + + + + + oozie.service.ELService.ext.functions.coord-action-start + + + + EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. + This property is a convenience property to add extensions to the built in executors without having to + include all the built in ones. + + + + + oozie.service.ELService.latest-el.use-current-time + false + + Determine whether to use the current time to determine the latest dependency or the action creation time. + This is for backward compatibility with older oozie behaviour. + + + + + + + oozie.service.UUIDService.generator + counter + + random : generated UUIDs will be random strings. + counter: generated UUIDs generated will be a counter postfixed with the system startup time. + + + + + + + oozie.service.DBLiteWorkflowStoreService.status.metrics.collection.interval + 5 + Workflow Status metrics collection interval in minutes. + + + + oozie.service.DBLiteWorkflowStoreService.status.metrics.window + 3600 + + Workflow Status metrics collection window in seconds. Workflow status will be instrumented for the window. + + + + + + + oozie.db.schema.name + oozie + + Oozie DataBase Name + + + + + + + oozie.service.JPAService.create.db.schema + true + + Creates Oozie DB. + + If set to true, it creates the DB schema if it does not exist. If the DB schema exists is a NOP. + If set to false, it does not create the DB schema. If the DB schema does not exist it fails start up. + + + + + oozie.service.JPAService.validate.db.connection + false + + Validates DB connections from the DB connection pool. + If the 'oozie.service.JPAService.create.db.schema' property is set to true, this property is ignored. + + + + + oozie.service.JPAService.validate.db.connection.eviction.interval + 300000 + + Validates DB connections from the DB connection pool. + When validate db connection 'TestWhileIdle' is true, the number of milliseconds to sleep + between runs of the idle object evictor thread. + + + + + oozie.service.JPAService.validate.db.connection.eviction.num + 10 + + Validates DB connections from the DB connection pool. + When validate db connection 'TestWhileIdle' is true, the number of objects to examine during + each run of the idle object evictor thread. + + + + + + oozie.service.JPAService.connection.data.source + org.apache.commons.dbcp.BasicDataSource + + DataSource to be used for connection pooling. + + + + + oozie.service.JPAService.jdbc.driver + org.apache.derby.jdbc.EmbeddedDriver + + JDBC driver class. + + + + + oozie.service.JPAService.jdbc.url + jdbc:derby:${oozie.data.dir}/${oozie.db.schema.name}-db;create=true + + JDBC URL. + + + + + oozie.service.JPAService.jdbc.username + sa + + DB user name. + + + + + oozie.service.JPAService.jdbc.password + + + DB user password. + + IMPORTANT: if password is emtpy leave a 1 space string, the service trims the value, + if empty Configuration assumes it is NULL. + + IMPORTANT: if the StoreServicePasswordService is active, it will reset this value with the value given in + the console. + + + + + oozie.service.JPAService.pool.max.active.conn + 10 + + Max number of connections. + + + + + + + oozie.service.SchemaService.wf.ext.schemas + oozie-sla-0.1.xsd,oozie-sla-0.2.xsd + + Schemas for additional actions types. + + IMPORTANT: if there are no schemas leave a 1 space string, the service trims the value, + if empty Configuration assumes it is NULL. + + + + + oozie.service.SchemaService.coord.ext.schemas + oozie-sla-0.1.xsd,oozie-sla-0.2.xsd + + Schemas for additional actions types. + + IMPORTANT: if there are no schemas leave a 1 space string, the service trims the value, + if empty Configuration assumes it is NULL. + + + + + oozie.service.SchemaService.sla.ext.schemas + + + Schemas for semantic validation for GMS SLA. + + IMPORTANT: if there are no schemas leave a 1 space string, the service trims the value, + if empty Configuration assumes it is NULL. + + + + + + oozie.service.CallbackService.base.url + ${oozie.base.url}/callback + + Base callback URL used by ActionExecutors. + + + + + + + oozie.servlet.CallbackServlet.max.data.len + 2048 + + Max size in characters for the action completion data output. + + + + + + + oozie.external.stats.max.size + -1 + + Max size in bytes for action stats. -1 means infinite value. + + + + + + + oozie.JobCommand.job.console.url + ${oozie.base.url}?job= + + Base console URL for a workflow job. + + + + + + + + oozie.service.ActionService.executor.classes + + org.apache.oozie.action.decision.DecisionActionExecutor, + org.apache.oozie.action.hadoop.JavaActionExecutor, + org.apache.oozie.action.hadoop.FsActionExecutor, + org.apache.oozie.action.hadoop.MapReduceActionExecutor, + org.apache.oozie.action.hadoop.PigActionExecutor, + org.apache.oozie.action.ssh.SshActionExecutor, + org.apache.oozie.action.oozie.SubWorkflowActionExecutor + + + List of ActionExecutors classes (separated by commas). + Only action types with associated executors can be used in workflows. + + + + + oozie.service.ActionService.executor.ext.classes + + + List of ActionExecutors extension classes (separated by commas). Only action types with associated + executors can be used in workflows. This property is a convenience property to add extensions to the built + in executors without having to include all the built in ones. + + + + + + + oozie.service.ActionCheckerService.action.check.interval + 60 + + The frequency at which the ActionCheckService will run. + + + + + oozie.service.ActionCheckerService.action.check.delay + 600 + + The time, in seconds, between an ActionCheck for the same action. + + + + + oozie.service.ActionCheckerService.callable.batch.size + 10 + + This value determines the number of actions which will be batched together + to be executed by a single thread. + + + + + + oozie.service.StatusTransitService.statusTransit.interval + 60 + + The frequency in seconds at which the StatusTransitService will run. + + + + + oozie.service.StatusTransitService.backward.support.for.coord.status + false + + true, if coordinator job submits using 'uri:oozie:coordinator:0.1' and wants to keep Oozie 2.x status transit. + if set true, + 1. SUCCEEDED state in coordinator job means materialization done. + 2. No DONEWITHERROR state in coordinator job + 3. No PAUSED or PREPPAUSED state in coordinator job + 4. PREPSUSPENDED becomes SUSPENDED in coordinator job + + + + + oozie.service.StatusTransitService.backward.support.for.states.without.error + true + + true, if you want to keep Oozie 3.2 status transit. + Change it to false for Oozie 4.x releases. + if set true, + No states like RUNNINGWITHERROR, SUSPENDEDWITHERROR and PAUSEDWITHERROR + for coordinator and bundle + + + + + + oozie.service.PauseTransitService.PauseTransit.interval + 60 + + The frequency in seconds at which the PauseTransitService will run. + + + + + + + + oozie.action.launcher.mapreduce.job.ubertask.enable + false + + Enables Uber Mode for the launcher job in YARN/Hadoop 2 (no effect in Hadoop 1). + Setting oozie.launcher.mapreduce.job.ubertask.enable in a an action's configuration section overrides this for that + action. + + + + + + + + oozie.action.retries.max + 3 + + The number of retries for executing an action in case of failure + + + + + oozie.action.hadoop.delete.hdfs.tmp.dir + false + + If set to true, it will delete temporary directory at the end of execution of map reduce action. + + + + + + + oozie.action.pig.delete.hdfs.tmp.dir + false + + If set to true, it will delete temporary directory at the end of execution of pig action. + + + + + + + oozie.action.ssh.delete.remote.tmp.dir + false + + If set to true, it will delete temporary directory at the end of execution of ssh action. + + + + + oozie.action.ssh.http.command + curl + + Command to use for callback to oozie, normally is 'curl' or 'wget'. + The command must available in PATH environment variable of the USER@HOST box shell. + + + + + oozie.action.ssh.http.command.post.options + --data-binary @#stdout --request POST --header "content-type:text/plain" + + The callback command POST options. + Used when the ouptut of the ssh action is captured. + + + + + oozie.action.ssh.allow.user.at.host + true + + Specifies whether the user specified by the ssh action is allowed or is to be replaced + by the Job user + + + + + + + oozie.service.HadoopAccessorService.kerberos.enabled + false + + Indicates if Oozie is configured to use Kerberos. + + + + + local.realm + LOCALHOST + + Kerberos Realm used by Oozie and Hadoop. Using 'local.realm' to be aligned with Hadoop configuration + + + + + oozie.service.HadoopAccessorService.keytab.file + ${user.home}/oozie.keytab + + Location of the Oozie user keytab file. + + + + + oozie.service.HadoopAccessorService.kerberos.principal + ${user.name}/localhost@${local.realm} + + Kerberos principal for Oozie service. + + + + + oozie.service.HadoopAccessorService.jobTracker.whitelist + + + Whitelisted job tracker for Oozie service. + + + + + oozie.service.HadoopAccessorService.nameNode.whitelist + + + Whitelisted job tracker for Oozie service. + + + + + oozie.service.HadoopAccessorService.hadoop.configurations + *=hadoop-conf + + Comma separated AUTHORITY=HADOOP_CONF_DIR, where AUTHORITY is the HOST:PORT of + the Hadoop service (JobTracker, YARN, HDFS). The wildcard '*' configuration is + used when there is no exact match for an authority. The HADOOP_CONF_DIR contains + the relevant Hadoop *-site.xml files. If the path is relative is looked within + the Oozie configuration directory; though the path can be absolute (i.e. to point + to Hadoop client conf/ directories in the local filesystem. + + + + + + oozie.service.HadoopAccessorService.action.configurations + *=action-conf + + Comma separated AUTHORITY=ACTION_CONF_DIR, where AUTHORITY is the HOST:PORT of + the Hadoop MapReduce service (JobTracker, YARN). The wildcard '*' configuration is + used when there is no exact match for an authority. The ACTION_CONF_DIR may contain + ACTION.xml files where ACTION is the action type ('java', 'map-reduce', 'pig', + 'hive', 'sqoop', etc.). If the ACTION.xml file exists, its properties will be used + as defaults properties for the action. If the path is relative is looked within + the Oozie configuration directory; though the path can be absolute (i.e. to point + to Hadoop client conf/ directories in the local filesystem. + + + + + + oozie.credentials.credentialclasses + + + A list of credential class mapping for CredentialsProvider + + + + + oozie.actions.main.classnames + distcp=org.apache.hadoop.tools.DistCp + + A list of class name mapping for Action classes + + + + + + oozie.service.WorkflowAppService.system.libpath + /user/${user.name}/share/lib + + System library path to use for workflow applications. + This path is added to workflow application if their job properties sets + the property 'oozie.use.system.libpath' to true. + + + + + use.system.libpath.for.mapreduce.and.pig.jobs + false + + If set to true, submissions of MapReduce and Pig jobs will include + automatically the system library path, thus not requiring users to + specify where the Pig JAR files are. Instead, the ones from the system + library path are used. + + + + + oozie.command.default.lock.timeout + 5000 + + Default timeout (in milliseconds) for commands for acquiring an exclusive lock on an entity. + + + + + + + oozie.service.LiteWorkflowStoreService.user.retry.max + 3 + + Automatic retry max count for workflow action is 3 in default. + + + + + oozie.service.LiteWorkflowStoreService.user.retry.inteval + 10 + + Automatic retry interval for workflow action is in minutes and the default value is 10 minutes. + + + + + oozie.service.LiteWorkflowStoreService.user.retry.error.code + JA008,JA009,JA017,JA018,JA019,FS009,FS008 + + Automatic retry interval for workflow action is handled for these specified error code: + FS009, FS008 is file exists error when using chmod in fs action. + JA018 is output directory exists error in workflow map-reduce action. + JA019 is error while executing distcp action. + JA017 is job not exists error in action executor. + JA008 is FileNotFoundException in action executor. + JA009 is IOException in action executor. + + + + + oozie.service.LiteWorkflowStoreService.user.retry.error.code.ext + + + Automatic retry interval for workflow action is handled for these specified extra error code. + + + + + oozie.service.LiteWorkflowStoreService.node.def.version + _oozie_inst_v_1 + + NodeDef default version, _oozie_inst_v_0 or _oozie_inst_v_1 + + + + + + + oozie.authentication.type + simple + + Defines authentication used for Oozie HTTP endpoint. + Supported values are: simple | kerberos | #AUTHENTICATION_HANDLER_CLASSNAME# + + + + + oozie.authentication.token.validity + 36000 + + Indicates how long (in seconds) an authentication token is valid before it has + to be renewed. + + + + + oozie.authentication.signature.secret + oozie + + The signature secret for signing the authentication tokens. + If not set a random secret is generated at startup time. + In order to authentiation to work correctly across multiple hosts + the secret must be the same across al the hosts. + + + + + oozie.authentication.cookie.domain + + + The domain to use for the HTTP cookie that stores the authentication token. + In order to authentiation to work correctly across multiple hosts + the domain must be correctly set. + + + + + oozie.authentication.simple.anonymous.allowed + true + + Indicates if anonymous requests are allowed when using 'simple' authentication. + + + + + oozie.authentication.kerberos.principal + HTTP/localhost@${local.realm} + + Indicates the Kerberos principal to be used for HTTP endpoint. + The principal MUST start with 'HTTP/' as per Kerberos HTTP SPNEGO specification. + + + + + oozie.authentication.kerberos.keytab + ${oozie.service.HadoopAccessorService.keytab.file} + + Location of the keytab file with the credentials for the principal. + Referring to the same keytab file Oozie uses for its Kerberos credentials for Hadoop. + + + + + oozie.authentication.kerberos.name.rules + DEFAULT + + The kerberos names rules is to resolve kerberos principal names, refer to Hadoop's + KerberosName for more details. + + + + + + oozie.coord.actions.default.length + 1000 + + Default number of coordinator actions to be retrieved by the info command + + + + + + oozie.validate.ForkJoin + true + + If true, fork and join should be validated at wf submission time. + + + + + oozie.coord.action.get.all.attributes + false + + Setting to true is not recommended as coord job/action info will bring all columns of the action in memory. + Set it true only if backward compatibility for action/job info is required. + + + + + oozie.service.HadoopAccessorService.supported.filesystems + hdfs,hftp,webhdfs + + Enlist the different filesystems supported for federation. If wildcard "*" is specified, + then ALL file schemes will be allowed. + + + + + oozie.service.URIHandlerService.uri.handlers + org.apache.oozie.dependency.FSURIHandler + + Enlist the different uri handlers supported for data availability checks. + + + + + + oozie.notification.url.connection.timeout + 10000 + + Defines the timeout, in milliseconds, for Oozie HTTP notification callbacks. Oozie does + HTTP notifications for workflow jobs which set the 'oozie.wf.action.notification.url', + 'oozie.wf.worklfow.notification.url' and/or 'oozie.coord.action.notification.url' + properties in their job.properties. Refer to section '5 Oozie Notifications' in the + Workflow specification for details. + + + + + + + oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache + false + + Due to a bug in Hadoop 2.0.2-alpha, MAPREDUCE-4820, launcher jobs fail to set + the distributed cache for the action job because the local JARs are implicitly + included triggering a duplicate check. + This flag removes the distributed cache files for the action as they'll be + included from the local JARs of the JobClient (MRApps) submitting the action + job from the launcher. + + + + + oozie.service.EventHandlerService.filter.app.types + workflow_job, coordinator_action + + The app-types among workflow/coordinator/bundle job/action for which + for which events system is enabled. + + + + + oozie.service.EventHandlerService.event.queue + org.apache.oozie.event.MemoryEventQueue + + The implementation for EventQueue in use by the EventHandlerService. + + + + + oozie.service.EventHandlerService.event.listeners + org.apache.oozie.jms.JMSJobEventListener + + + + oozie.service.EventHandlerService.queue.size + 10000 + + Maximum number of events to be contained in the event queue. + + + + + oozie.service.EventHandlerService.worker.interval + 30 + + The default interval (seconds) at which the worker threads will be scheduled to run + and process events. + + + + + oozie.service.EventHandlerService.batch.size + 10 + + The batch size for batched draining per thread from the event queue. + + + + + oozie.service.EventHandlerService.worker.threads + 3 + + Number of worker threads to be scheduled to run and process events. + + + + + oozie.sla.service.SLAService.capacity + 5000 + + Maximum number of sla records to be contained in the memory structure. + + + + + oozie.sla.service.SLAService.alert.events + END_MISS + + Default types of SLA events for being alerted of. + + + + + oozie.sla.service.SLAService.calculator.impl + org.apache.oozie.sla.SLACalculatorMemory + + The implementation for SLACalculator in use by the SLAService. + + + + + oozie.sla.service.SLAService.job.event.latency + 90000 + + Time in milliseconds to account of latency of getting the job status event + to compare against and decide sla miss/met + + + + diff --git a/sahara/plugins/vanilla/v2_6_0/resources/yarn-default.xml b/sahara/plugins/vanilla/v2_6_0/resources/yarn-default.xml new file mode 100644 index 0000000000..9d37196bde --- /dev/null +++ b/sahara/plugins/vanilla/v2_6_0/resources/yarn-default.xml @@ -0,0 +1,1528 @@ + + + + + + + + + + + + + + Factory to create client IPC classes. + yarn.ipc.client.factory.class + + + + Factory to create server IPC classes. + yarn.ipc.server.factory.class + + + + Factory to create serializeable records. + yarn.ipc.record.factory.class + + + + RPC class implementation + yarn.ipc.rpc.class + org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC + + + + + The hostname of the RM. + yarn.resourcemanager.hostname + 0.0.0.0 + + + + The address of the applications manager interface in the RM. + yarn.resourcemanager.address + ${yarn.resourcemanager.hostname}:8032 + + + + + The actual address the server will bind to. If this optional address is + set, the RPC and webapp servers will bind to this address and the port specified in + yarn.resourcemanager.address and yarn.resourcemanager.webapp.address, respectively. This + is most useful for making RM listen to all interfaces by setting to 0.0.0.0. + + yarn.resourcemanager.bind-host + + + + + The number of threads used to handle applications manager requests. + yarn.resourcemanager.client.thread-count + 50 + + + + The expiry interval for application master reporting. + yarn.am.liveness-monitor.expiry-interval-ms + 600000 + + + + The Kerberos principal for the resource manager. + yarn.resourcemanager.principal + + + + The address of the scheduler interface. + yarn.resourcemanager.scheduler.address + ${yarn.resourcemanager.hostname}:8030 + + + + Number of threads to handle scheduler interface. + yarn.resourcemanager.scheduler.client.thread-count + 50 + + + + + This configures the HTTP endpoint for Yarn Daemons.The following + values are supported: + - HTTP_ONLY : Service is provided only on http + - HTTPS_ONLY : Service is provided only on https + + yarn.http.policy + HTTP_ONLY + + + + The http address of the RM web application. + yarn.resourcemanager.webapp.address + ${yarn.resourcemanager.hostname}:8088 + + + + The https adddress of the RM web application. + yarn.resourcemanager.webapp.https.address + ${yarn.resourcemanager.hostname}:8090 + + + + yarn.resourcemanager.resource-tracker.address + ${yarn.resourcemanager.hostname}:8031 + + + + Are acls enabled. + yarn.acl.enable + false + + + + ACL of who can be admin of the YARN cluster. + yarn.admin.acl + * + + + + The address of the RM admin interface. + yarn.resourcemanager.admin.address + ${yarn.resourcemanager.hostname}:8033 + + + + Number of threads used to handle RM admin interface. + yarn.resourcemanager.admin.client.thread-count + 1 + + + + Maximum time to wait to establish connection to + ResourceManager. + yarn.resourcemanager.connect.max-wait.ms + 900000 + + + + How often to try connecting to the + ResourceManager. + yarn.resourcemanager.connect.retry-interval.ms + 30000 + + + + The maximum number of application attempts. It's a global + setting for all application masters. Each application master can specify + its individual maximum number of application attempts via the API, but the + individual number cannot be more than the global upper bound. If it is, + the resourcemanager will override it. The default number is set to 2, to + allow at least one retry for AM. + yarn.resourcemanager.am.max-attempts + 2 + + + + How often to check that containers are still alive. + yarn.resourcemanager.container.liveness-monitor.interval-ms + 600000 + + + + The keytab for the resource manager. + yarn.resourcemanager.keytab + /etc/krb5.keytab + + + + Flag to enable override of the default kerberos authentication + filter with the RM authentication filter to allow authentication using + delegation tokens(fallback to kerberos if the tokens are missing). Only + applicable when the http authentication type is kerberos. + yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled + true + + + + How long to wait until a node manager is considered dead. + yarn.nm.liveness-monitor.expiry-interval-ms + 600000 + + + + Path to file with nodes to include. + yarn.resourcemanager.nodes.include-path + + + + + Path to file with nodes to exclude. + yarn.resourcemanager.nodes.exclude-path + + + + + Number of threads to handle resource tracker calls. + yarn.resourcemanager.resource-tracker.client.thread-count + 50 + + + + The class to use as the resource scheduler. + yarn.resourcemanager.scheduler.class + org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler + + + + The minimum allocation for every container request at the RM, + in MBs. Memory requests lower than this won't take effect, + and the specified value will get allocated at minimum. + yarn.scheduler.minimum-allocation-mb + 1024 + + + + The maximum allocation for every container request at the RM, + in MBs. Memory requests higher than this won't take effect, + and will get capped to this value. + yarn.scheduler.maximum-allocation-mb + 8192 + + + + The minimum allocation for every container request at the RM, + in terms of virtual CPU cores. Requests lower than this won't take effect, + and the specified value will get allocated the minimum. + yarn.scheduler.minimum-allocation-vcores + 1 + + + + The maximum allocation for every container request at the RM, + in terms of virtual CPU cores. Requests higher than this won't take effect, + and will get capped to this value. + yarn.scheduler.maximum-allocation-vcores + 32 + + + + Enable RM to recover state after starting. If true, then + yarn.resourcemanager.store.class must be specified. + yarn.resourcemanager.recovery.enabled + false + + + + Enable RM work preserving recovery. This configuration is private + to YARN for experimenting the feature. + + yarn.resourcemanager.work-preserving-recovery.enabled + false + + + + Set the amount of time RM waits before allocating new + containers on work-preserving-recovery. Such wait period gives RM a chance + to settle down resyncing with NMs in the cluster on recovery, before assigning + new containers to applications. + + yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms + 10000 + + + + The class to use as the persistent store. + + If org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore + is used, the store is implicitly fenced; meaning a single ResourceManager + is able to use the store at any point in time. More details on this + implicit fencing, along with setting up appropriate ACLs is discussed + under yarn.resourcemanager.zk-state-store.root-node.acl. + + yarn.resourcemanager.store.class + org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore + + + + The maximum number of completed applications RM state + store keeps, less than or equals to ${yarn.resourcemanager.max-completed-applications}. + By default, it equals to ${yarn.resourcemanager.max-completed-applications}. + This ensures that the applications kept in the state store are consistent with + the applications remembered in RM memory. + Any values larger than ${yarn.resourcemanager.max-completed-applications} will + be reset to ${yarn.resourcemanager.max-completed-applications}. + Note that this value impacts the RM recovery performance.Typically, + a smaller value indicates better performance on RM recovery. + + yarn.resourcemanager.state-store.max-completed-applications + ${yarn.resourcemanager.max-completed-applications} + + + + Host:Port of the ZooKeeper server to be used by the RM. This + must be supplied when using the ZooKeeper based implementation of the + RM state store and/or embedded automatic failover in a HA setting. + + yarn.resourcemanager.zk-address + + + + + Number of times RM tries to connect to ZooKeeper. + yarn.resourcemanager.zk-num-retries + 1000 + + + + Retry interval in milliseconds when connecting to ZooKeeper. + When HA is enabled, the value here is NOT used. It is generated + automatically from yarn.resourcemanager.zk-timeout-ms and + yarn.resourcemanager.zk-num-retries. + + yarn.resourcemanager.zk-retry-interval-ms + 1000 + + + + Full path of the ZooKeeper znode where RM state will be + stored. This must be supplied when using + org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore + as the value for yarn.resourcemanager.store.class + yarn.resourcemanager.zk-state-store.parent-path + /rmstore + + + + ZooKeeper session timeout in milliseconds. Session expiration + is managed by the ZooKeeper cluster itself, not by the client. This value is + used by the cluster to determine when the client's session expires. + Expirations happens when the cluster does not hear from the client within + the specified session timeout period (i.e. no heartbeat). + yarn.resourcemanager.zk-timeout-ms + 10000 + + + + ACL's to be used for ZooKeeper znodes. + yarn.resourcemanager.zk-acl + world:anyone:rwcda + + + + + ACLs to be used for the root znode when using ZKRMStateStore in a HA + scenario for fencing. + + ZKRMStateStore supports implicit fencing to allow a single + ResourceManager write-access to the store. For fencing, the + ResourceManagers in the cluster share read-write-admin privileges on the + root node, but the Active ResourceManager claims exclusive create-delete + permissions. + + By default, when this property is not set, we use the ACLs from + yarn.resourcemanager.zk-acl for shared admin access and + rm-address:random-number for username-based exclusive create-delete + access. + + This property allows users to set ACLs of their choice instead of using + the default mechanism. For fencing to work, the ACLs should be + carefully set differently on each ResourceManger such that all the + ResourceManagers have shared admin access and the Active ResourceManger + takes over (exclusively) the create-delete access. + + yarn.resourcemanager.zk-state-store.root-node.acl + + + + + Specify the auths to be used for the ACL's specified in both the + yarn.resourcemanager.zk-acl and + yarn.resourcemanager.zk-state-store.root-node.acl properties. This + takes a comma-separated list of authentication mechanisms, each of the + form 'scheme:auth' (the same syntax used for the 'addAuth' command in + the ZK CLI). + + yarn.resourcemanager.zk-auth + + + + URI pointing to the location of the FileSystem path where + RM state will be stored. This must be supplied when using + org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore + as the value for yarn.resourcemanager.store.class + yarn.resourcemanager.fs.state-store.uri + ${hadoop.tmp.dir}/yarn/system/rmstore + + + + + hdfs client retry policy specification. hdfs client retry + is always enabled. Specified in pairs of sleep-time and number-of-retries + and (t0, n0), (t1, n1), ..., the first n0 retries sleep t0 milliseconds on + average, the following n1 retries sleep t1 milliseconds on average, and so on. + + yarn.resourcemanager.fs.state-store.retry-policy-spec + 2000, 500 + + + + Enable RM high-availability. When enabled, + (1) The RM starts in the Standby mode by default, and transitions to + the Active mode when prompted to. + (2) The nodes in the RM ensemble are listed in + yarn.resourcemanager.ha.rm-ids + (3) The id of each RM either comes from yarn.resourcemanager.ha.id + if yarn.resourcemanager.ha.id is explicitly specified or can be + figured out by matching yarn.resourcemanager.address.{id} with local address + (4) The actual physical addresses come from the configs of the pattern + - {rpc-config}.{id} + yarn.resourcemanager.ha.enabled + false + + + + Enable automatic failover. + By default, it is enabled only when HA is enabled + yarn.resourcemanager.ha.automatic-failover.enabled + true + + + + Enable embedded automatic failover. + By default, it is enabled only when HA is enabled. + The embedded elector relies on the RM state store to handle fencing, + and is primarily intended to be used in conjunction with ZKRMStateStore. + + yarn.resourcemanager.ha.automatic-failover.embedded + true + + + + The base znode path to use for storing leader information, + when using ZooKeeper based leader election. + yarn.resourcemanager.ha.automatic-failover.zk-base-path + /yarn-leader-election + + + + Name of the cluster. In a HA setting, + this is used to ensure the RM participates in leader + election for this cluster and ensures it does not affect + other clusters + yarn.resourcemanager.cluster-id + + + + + The list of RM nodes in the cluster when HA is + enabled. See description of yarn.resourcemanager.ha + .enabled for full details on how this is used. + yarn.resourcemanager.ha.rm-ids + + + + + The id (string) of the current RM. When HA is enabled, this + is an optional config. The id of current RM can be set by explicitly + specifying yarn.resourcemanager.ha.id or figured out by matching + yarn.resourcemanager.address.{id} with local address + See description of yarn.resourcemanager.ha.enabled + for full details on how this is used. + yarn.resourcemanager.ha.id + + + + + When HA is enabled, the class to be used by Clients, AMs and + NMs to failover to the Active RM. It should extend + org.apache.hadoop.yarn.client.RMFailoverProxyProvider + yarn.client.failover-proxy-provider + org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider + + + + When HA is enabled, the max number of times + FailoverProxyProvider should attempt failover. When set, + this overrides the yarn.resourcemanager.connect.max-wait.ms. When + not set, this is inferred from + yarn.resourcemanager.connect.max-wait.ms. + yarn.client.failover-max-attempts + + + + + When HA is enabled, the sleep base (in milliseconds) to be + used for calculating the exponential delay between failovers. When set, + this overrides the yarn.resourcemanager.connect.* settings. When + not set, yarn.resourcemanager.connect.retry-interval.ms is used instead. + + yarn.client.failover-sleep-base-ms + + + + + When HA is enabled, the maximum sleep time (in milliseconds) + between failovers. When set, this overrides the + yarn.resourcemanager.connect.* settings. When not set, + yarn.resourcemanager.connect.retry-interval.ms is used instead. + yarn.client.failover-sleep-max-ms + + + + + When HA is enabled, the number of retries per + attempt to connect to a ResourceManager. In other words, + it is the ipc.client.connect.max.retries to be used during + failover attempts + yarn.client.failover-retries + 0 + + + + When HA is enabled, the number of retries per + attempt to connect to a ResourceManager on socket timeouts. In other + words, it is the ipc.client.connect.max.retries.on.timeouts to be used + during failover attempts + yarn.client.failover-retries-on-socket-timeouts + 0 + + + + The maximum number of completed applications RM keeps. + yarn.resourcemanager.max-completed-applications + 10000 + + + + Interval at which the delayed token removal thread runs + yarn.resourcemanager.delayed.delegation-token.removal-interval-ms + 30000 + + + + If true, ResourceManager will have proxy-user privileges. + Use case: In a secure cluster, YARN requires the user hdfs delegation-tokens to + do localization and log-aggregation on behalf of the user. If this is set to true, + ResourceManager is able to request new hdfs delegation tokens on behalf of + the user. This is needed by long-running-service, because the hdfs tokens + will eventually expire and YARN requires new valid tokens to do localization + and log-aggregation. Note that to enable this use case, the corresponding + HDFS NameNode has to configure ResourceManager as the proxy-user so that + ResourceManager can itself ask for new tokens on behalf of the user when + tokens are past their max-life-time. + yarn.resourcemanager.proxy-user-privileges.enabled + false + + + + Interval for the roll over for the master key used to generate + application tokens + + yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs + 86400 + + + + Interval for the roll over for the master key used to generate + container tokens. It is expected to be much greater than + yarn.nm.liveness-monitor.expiry-interval-ms and + yarn.rm.container-allocation.expiry-interval-ms. Otherwise the + behavior is undefined. + + yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs + 86400 + + + + The heart-beat interval in milliseconds for every NodeManager in the cluster. + yarn.resourcemanager.nodemanagers.heartbeat-interval-ms + 1000 + + + + The minimum allowed version of a connecting nodemanager. The valid values are + NONE (no version checking), EqualToRM (the nodemanager's version is equal to + or greater than the RM version), or a Version String. + yarn.resourcemanager.nodemanager.minimum.version + NONE + + + + Enable a set of periodic monitors (specified in + yarn.resourcemanager.scheduler.monitor.policies) that affect the + scheduler. + yarn.resourcemanager.scheduler.monitor.enable + false + + + + The list of SchedulingEditPolicy classes that interact with + the scheduler. A particular module may be incompatible with the + scheduler, other policies, or a configuration of either. + yarn.resourcemanager.scheduler.monitor.policies + org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy + + + + The class to use as the configuration provider. + If org.apache.hadoop.yarn.LocalConfigurationProvider is used, + the local configuration will be loaded. + If org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider is used, + the configuration which will be loaded should be uploaded to remote File system first. + + yarn.resourcemanager.configuration.provider-class + org.apache.hadoop.yarn.LocalConfigurationProvider + + + + + The setting that controls whether yarn system metrics is + published on the timeline server or not by RM. + yarn.resourcemanager.system-metrics-publisher.enabled + false + + + + Number of worker threads that send the yarn system metrics + data. + yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size + 10 + + + + + The hostname of the NM. + yarn.nodemanager.hostname + 0.0.0.0 + + + + The address of the container manager in the NM. + yarn.nodemanager.address + ${yarn.nodemanager.hostname}:0 + + + + + The actual address the server will bind to. If this optional address is + set, the RPC and webapp servers will bind to this address and the port specified in + yarn.nodemanager.address and yarn.nodemanager.webapp.address, respectively. This is + most useful for making NM listen to all interfaces by setting to 0.0.0.0. + + yarn.nodemanager.bind-host + + + + + Environment variables that should be forwarded from the NodeManager's environment to the container's. + yarn.nodemanager.admin-env + MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX + + + + Environment variables that containers may override rather than use NodeManager's default. + yarn.nodemanager.env-whitelist + JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,HADOOP_YARN_HOME + + + + who will execute(launch) the containers. + yarn.nodemanager.container-executor.class + org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor + + + + + Number of threads container manager uses. + yarn.nodemanager.container-manager.thread-count + 20 + + + + Number of threads used in cleanup. + yarn.nodemanager.delete.thread-count + 4 + + + + + Number of seconds after an application finishes before the nodemanager's + DeletionService will delete the application's localized file directory + and log directory. + + To diagnose Yarn application problems, set this property's value large + enough (for example, to 600 = 10 minutes) to permit examination of these + directories. After changing the property's value, you must restart the + nodemanager in order for it to have an effect. + + The roots of Yarn applications' work directories is configurable with + the yarn.nodemanager.local-dirs property (see below), and the roots + of the Yarn applications' log directories is configurable with the + yarn.nodemanager.log-dirs property (see also below). + + yarn.nodemanager.delete.debug-delay-sec + 0 + + + + Keytab for NM. + yarn.nodemanager.keytab + /etc/krb5.keytab + + + + List of directories to store localized files in. An + application's localized file directory will be found in: + ${yarn.nodemanager.local-dirs}/usercache/${user}/appcache/application_${appid}. + Individual containers' work directories, called container_${contid}, will + be subdirectories of this. + + yarn.nodemanager.local-dirs + ${hadoop.tmp.dir}/nm-local-dir + + + + It limits the maximum number of files which will be localized + in a single local directory. If the limit is reached then sub-directories + will be created and new files will be localized in them. If it is set to + a value less than or equal to 36 [which are sub-directories (0-9 and then + a-z)] then NodeManager will fail to start. For example; [for public + cache] if this is configured with a value of 40 ( 4 files + + 36 sub-directories) and the local-dir is "/tmp/local-dir1" then it will + allow 4 files to be created directly inside "/tmp/local-dir1/filecache". + For files that are localized further it will create a sub-directory "0" + inside "/tmp/local-dir1/filecache" and will localize files inside it + until it becomes full. If a file is removed from a sub-directory that + is marked full, then that sub-directory will be used back again to + localize files. + + yarn.nodemanager.local-cache.max-files-per-directory + 8192 + + + + Address where the localizer IPC is. + yarn.nodemanager.localizer.address + ${yarn.nodemanager.hostname}:8040 + + + + Interval in between cache cleanups. + yarn.nodemanager.localizer.cache.cleanup.interval-ms + 600000 + + + + Target size of localizer cache in MB, per nodemanager. It is + a target retention size that only includes resources with PUBLIC and + PRIVATE visibility and excludes resources with APPLICATION visibility + + yarn.nodemanager.localizer.cache.target-size-mb + 10240 + + + + Number of threads to handle localization requests. + yarn.nodemanager.localizer.client.thread-count + 5 + + + + Number of threads to use for localization fetching. + yarn.nodemanager.localizer.fetch.thread-count + 4 + + + + + Where to store container logs. An application's localized log directory + will be found in ${yarn.nodemanager.log-dirs}/application_${appid}. + Individual containers' log directories will be below this, in directories + named container_{$contid}. Each container directory will contain the files + stderr, stdin, and syslog generated by that container. + + yarn.nodemanager.log-dirs + ${yarn.log.dir}/userlogs + + + + Whether to enable log aggregation. Log aggregation collects + each container's logs and moves these logs onto a file-system, for e.g. + HDFS, after the application completes. Users can configure the + "yarn.nodemanager.remote-app-log-dir" and + "yarn.nodemanager.remote-app-log-dir-suffix" properties to determine + where these logs are moved to. Users can access the logs via the + Application Timeline Server. + + yarn.log-aggregation-enable + false + + + + How long to keep aggregation logs before deleting them. -1 disables. + Be careful set this too small and you will spam the name node. + yarn.log-aggregation.retain-seconds + -1 + + + + How long to wait between aggregated log retention checks. + If set to 0 or a negative value then the value is computed as one-tenth + of the aggregated log retention time. Be careful set this too small and + you will spam the name node. + yarn.log-aggregation.retain-check-interval-seconds + -1 + + + + Time in seconds to retain user logs. Only applicable if + log aggregation is disabled + + yarn.nodemanager.log.retain-seconds + 10800 + + + + Where to aggregate logs to. + yarn.nodemanager.remote-app-log-dir + /tmp/logs + + + The remote log dir will be created at + {yarn.nodemanager.remote-app-log-dir}/${user}/{thisParam} + + yarn.nodemanager.remote-app-log-dir-suffix + logs + + + + Amount of physical memory, in MB, that can be allocated + for containers. + yarn.nodemanager.resource.memory-mb + 8192 + + + + Whether physical memory limits will be enforced for + containers. + yarn.nodemanager.pmem-check-enabled + true + + + + Whether virtual memory limits will be enforced for + containers. + yarn.nodemanager.vmem-check-enabled + true + + + + Ratio between virtual memory to physical memory when + setting memory limits for containers. Container allocations are + expressed in terms of physical memory, and virtual memory usage + is allowed to exceed this allocation by this ratio. + + yarn.nodemanager.vmem-pmem-ratio + 2.1 + + + + Number of vcores that can be allocated + for containers. This is used by the RM scheduler when allocating + resources for containers. This is not used to limit the number of + physical cores used by YARN containers. + yarn.nodemanager.resource.cpu-vcores + 8 + + + + Percentage of CPU that can be allocated + for containers. This setting allows users to limit the amount of + CPU that YARN containers use. Currently functional only + on Linux using cgroups. The default is to use 100% of CPU. + + yarn.nodemanager.resource.percentage-physical-cpu-limit + 100 + + + + NM Webapp address. + yarn.nodemanager.webapp.address + ${yarn.nodemanager.hostname}:8042 + + + + How often to monitor containers. + yarn.nodemanager.container-monitor.interval-ms + 3000 + + + + Class that calculates containers current resource utilization. + yarn.nodemanager.container-monitor.resource-calculator.class + + + + Frequency of running node health script. + yarn.nodemanager.health-checker.interval-ms + 600000 + + + + Script time out period. + yarn.nodemanager.health-checker.script.timeout-ms + 1200000 + + + + The health check script to run. + yarn.nodemanager.health-checker.script.path + + + + + The arguments to pass to the health check script. + yarn.nodemanager.health-checker.script.opts + + + + + Frequency of running disk health checker code. + yarn.nodemanager.disk-health-checker.interval-ms + 120000 + + + + The minimum fraction of number of disks to be healthy for the + nodemanager to launch new containers. This correspond to both + yarn-nodemanager.local-dirs and yarn.nodemanager.log-dirs. i.e. If there + are less number of healthy local-dirs (or log-dirs) available, then + new containers will not be launched on this node. + yarn.nodemanager.disk-health-checker.min-healthy-disks + 0.25 + + + + The maximum percentage of disk space utilization allowed after + which a disk is marked as bad. Values can range from 0.0 to 100.0. + If the value is greater than or equal to 100, the nodemanager will check + for full disk. This applies to yarn-nodemanager.local-dirs and + yarn.nodemanager.log-dirs. + yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage + 90.0 + + + + The minimum space that must be available on a disk for + it to be used. This applies to yarn-nodemanager.local-dirs and + yarn.nodemanager.log-dirs. + yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb + 0 + + + + The path to the Linux container executor. + yarn.nodemanager.linux-container-executor.path + + + + The class which should help the LCE handle resources. + yarn.nodemanager.linux-container-executor.resources-handler.class + org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler + + + + + The cgroups hierarchy under which to place YARN proccesses (cannot contain commas). + If yarn.nodemanager.linux-container-executor.cgroups.mount is false (that is, if cgroups have + been pre-configured), then this cgroups hierarchy must already exist and be writable by the + NodeManager user, otherwise the NodeManager may fail. + Only used when the LCE resources handler is set to the CgroupsLCEResourcesHandler. + yarn.nodemanager.linux-container-executor.cgroups.hierarchy + /hadoop-yarn + + + + Whether the LCE should attempt to mount cgroups if not found. + Only used when the LCE resources handler is set to the CgroupsLCEResourcesHandler. + yarn.nodemanager.linux-container-executor.cgroups.mount + false + + + + Where the LCE should attempt to mount cgroups if not found. Common locations + include /sys/fs/cgroup and /cgroup; the default location can vary depending on the Linux + distribution in use. This path must exist before the NodeManager is launched. + Only used when the LCE resources handler is set to the CgroupsLCEResourcesHandler, and + yarn.nodemanager.linux-container-executor.cgroups.mount is true. + yarn.nodemanager.linux-container-executor.cgroups.mount-path + + + + This determines which of the two modes that LCE should use on a non-secure + cluster. If this value is set to true, then all containers will be launched as the user + specified in yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user. If + this value is set to false, then containers will run as the user who submitted the + application. + + yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users + true + + + + The UNIX user that containers will run as when Linux-container-executor + is used in nonsecure mode (a use case for this is using cgroups) if the + yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users is set + to true. + yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user + nobody + + + + The allowed pattern for UNIX user names enforced by + Linux-container-executor when used in nonsecure mode (use case for this + is using cgroups). The default value is taken from /usr/sbin/adduser + yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern + ^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$ + + + + This flag determines whether apps should run with strict resource limits + or be allowed to consume spare resources if they need them. For example, turning the + flag on will restrict apps to use only their share of CPU, even if the node has spare + CPU cycles. The default value is false i.e. use available resources. Please note that + turning this flag on may reduce job throughput on the cluster. + yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage + false + + + + T-file compression types used to compress aggregated logs. + yarn.nodemanager.log-aggregation.compression-type + none + + + + The kerberos principal for the node manager. + yarn.nodemanager.principal + + + + + the valid service name should only contain a-zA-Z0-9_ and can not start with numbers + yarn.nodemanager.aux-services + + + + + + No. of ms to wait between sending a SIGTERM and SIGKILL to a container + yarn.nodemanager.sleep-delay-before-sigkill.ms + 250 + + + + Max time to wait for a process to come up when trying to cleanup a container + yarn.nodemanager.process-kill-wait.ms + 2000 + + + + The minimum allowed version of a resourcemanager that a nodemanager will connect to. + The valid values are NONE (no version checking), EqualToNM (the resourcemanager's version is + equal to or greater than the NM version), or a Version String. + yarn.nodemanager.resourcemanager.minimum.version + NONE + + + + Max number of threads in NMClientAsync to process container + management events + yarn.client.nodemanager-client-async.thread-pool-max-size + 500 + + + + Max time to wait to establish a connection to NM + yarn.client.nodemanager-connect.max-wait-ms + 900000 + + + + Time interval between each attempt to connect to NM + yarn.client.nodemanager-connect.retry-interval-ms + 10000 + + + + + Maximum number of proxy connections to cache for node managers. If set + to a value greater than zero then the cache is enabled and the NMClient + and MRAppMaster will cache the specified number of node manager proxies. + There will be at max one proxy per node manager. Ex. configuring it to a + value of 5 will make sure that client will at max have 5 proxies cached + with 5 different node managers. These connections for these proxies will + be timed out if idle for more than the system wide idle timeout period. + Note that this could cause issues on large clusters as many connections + could linger simultaneously and lead to a large number of connection + threads. The token used for authentication will be used only at + connection creation time. If a new token is received then the earlier + connection should be closed in order to use the new token. This and + (yarn.client.nodemanager-client-async.thread-pool-max-size) are related + and should be in sync (no need for them to be equal). + If the value of this property is zero then the connection cache is + disabled and connections will use a zero idle timeout to prevent too + many connection threads on large clusters. + + yarn.client.max-cached-nodemanagers-proxies + 0 + + + + Enable the node manager to recover after starting + yarn.nodemanager.recovery.enabled + false + + + + The local filesystem directory in which the node manager will + store state when recovery is enabled. + yarn.nodemanager.recovery.dir + ${hadoop.tmp.dir}/yarn-nm-recovery + + + + + + yarn.nodemanager.docker-container-executor.exec-name + /usr/bin/docker + + Name or path to the Docker client. + + + + + + yarn.nodemanager.aux-services.mapreduce_shuffle.class + org.apache.hadoop.mapred.ShuffleHandler + + + + mapreduce.job.jar + + + + + mapreduce.job.hdfs-servers + ${fs.defaultFS} + + + + + + The kerberos principal for the proxy, if the proxy is not + running as part of the RM. + yarn.web-proxy.principal + + + + + Keytab for WebAppProxy, if the proxy is not running as part of + the RM. + yarn.web-proxy.keytab + + + + The address for the web proxy as HOST:PORT, if this is not + given then the proxy will run as part of the RM + yarn.web-proxy.address + + + + + + + + CLASSPATH for YARN applications. A comma-separated list + of CLASSPATH entries. When this value is empty, the following default + CLASSPATH for YARN applications would be used. + For Linux: + $HADOOP_CONF_DIR, + $HADOOP_COMMON_HOME/share/hadoop/common/*, + $HADOOP_COMMON_HOME/share/hadoop/common/lib/*, + $HADOOP_HDFS_HOME/share/hadoop/hdfs/*, + $HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*, + $HADOOP_YARN_HOME/share/hadoop/yarn/*, + $HADOOP_YARN_HOME/share/hadoop/yarn/lib/* + For Windows: + %HADOOP_CONF_DIR%, + %HADOOP_COMMON_HOME%/share/hadoop/common/*, + %HADOOP_COMMON_HOME%/share/hadoop/common/lib/*, + %HADOOP_HDFS_HOME%/share/hadoop/hdfs/*, + %HADOOP_HDFS_HOME%/share/hadoop/hdfs/lib/*, + %HADOOP_YARN_HOME%/share/hadoop/yarn/*, + %HADOOP_YARN_HOME%/share/hadoop/yarn/lib/* + + yarn.application.classpath + + + + + + + Indicate to clients whether timeline service is enabled or not. + If enabled, clients will put entities and events to the timeline server. + + yarn.timeline-service.enabled + false + + + + The hostname of the timeline service web application. + yarn.timeline-service.hostname + 0.0.0.0 + + + + This is default address for the timeline server to start the + RPC server. + yarn.timeline-service.address + ${yarn.timeline-service.hostname}:10200 + + + + The http address of the timeline service web application. + yarn.timeline-service.webapp.address + ${yarn.timeline-service.hostname}:8188 + + + + The https address of the timeline service web application. + yarn.timeline-service.webapp.https.address + ${yarn.timeline-service.hostname}:8190 + + + + + The actual address the server will bind to. If this optional address is + set, the RPC and webapp servers will bind to this address and the port specified in + yarn.timeline-service.address and yarn.timeline-service.webapp.address, respectively. + This is most useful for making the service listen to all interfaces by setting to + 0.0.0.0. + + yarn.timeline-service.bind-host + + + + + Store class name for timeline store. + yarn.timeline-service.store-class + org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore + + + + Enable age off of timeline store data. + yarn.timeline-service.ttl-enable + true + + + + Time to live for timeline store data in milliseconds. + yarn.timeline-service.ttl-ms + 604800000 + + + + Store file name for leveldb timeline store. + yarn.timeline-service.leveldb-timeline-store.path + ${hadoop.tmp.dir}/yarn/timeline + + + + Length of time to wait between deletion cycles of leveldb timeline store in milliseconds. + yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms + 300000 + + + + Size of read cache for uncompressed blocks for leveldb timeline store in bytes. + yarn.timeline-service.leveldb-timeline-store.read-cache-size + 104857600 + + + + Size of cache for recently read entity start times for leveldb timeline store in number of entities. + yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size + 10000 + + + + Size of cache for recently written entity start times for leveldb timeline store in number of entities. + yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size + 10000 + + + + Handler thread count to serve the client RPC requests. + yarn.timeline-service.handler-thread-count + 10 + + + + yarn.timeline-service.http-authentication.type + simple + + Defines authentication used for the timeline server HTTP endpoint. + Supported values are: simple | kerberos | #AUTHENTICATION_HANDLER_CLASSNAME# + + + + + yarn.timeline-service.http-authentication.simple.anonymous.allowed + true + + Indicates if anonymous requests are allowed by the timeline server when using + 'simple' authentication. + + + + + The Kerberos principal for the timeline server. + yarn.timeline-service.principal + + + + + The Kerberos keytab for the timeline server. + yarn.timeline-service.keytab + /etc/krb5.keytab + + + + + Default maximum number of retires for timeline servive client. + + yarn.timeline-service.client.max-retries + 30 + + + + + Default retry time interval for timeline servive client. + + yarn.timeline-service.client.retry-interval-ms + 1000 + + + + + The interval that the yarn client library uses to poll the + completion status of the asynchronous API of application client protocol. + + yarn.client.application-client-protocol.poll-interval-ms + 200 + + + + RSS usage of a process computed via + /proc/pid/stat is not very accurate as it includes shared pages of a + process. /proc/pid/smaps provides useful information like + Private_Dirty, Private_Clean, Shared_Dirty, Shared_Clean which can be used + for computing more accurate RSS. When this flag is enabled, RSS is computed + as Min(Shared_Dirty, Pss) + Private_Clean + Private_Dirty. It excludes + read-only shared mappings in RSS computation. + + yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled + false + + + + + + + Is the registry enabled: does the RM start it up, + create the user and system paths, and purge + service records when containers, application attempts + and applications complete + + hadoop.registry.rm.enabled + false + + + + + + hadoop.registry.zk.root + /registry + + + + + Zookeeper session timeout in milliseconds + + hadoop.registry.zk.session.timeout.ms + 60000 + + + + + Zookeeper session timeout in milliseconds + + hadoop.registry.zk.connection.timeout.ms + 15000 + + + + + Zookeeper connection retry count before failing + + hadoop.registry.zk.retry.times + 5 + + + + + + hadoop.registry.zk.retry.interval.ms + 1000 + + + + + Zookeeper retry limit in milliseconds, during + exponential backoff: {@value} + + This places a limit even + if the retry times and interval limit, combined + with the backoff policy, result in a long retry + period + + hadoop.registry.zk.retry.ceiling.ms + 60000 + + + + + List of hostname:port pairs defining the + zookeeper quorum binding for the registry + + hadoop.registry.zk.quorum + localhost:2181 + + + + + Key to set if the registry is secure. Turning it on + changes the permissions policy from "open access" + to restrictions on kerberos with the option of + a user adding one or more auth key pairs down their + own tree. + + hadoop.registry.secure + false + + + + + A comma separated list of Zookeeper ACL identifiers with + system access to the registry in a secure cluster. + + These are given full access to all entries. + + If there is an "@" at the end of a SASL entry it + instructs the registry client to append the default kerberos domain. + + hadoop.registry.system.acls + sasl:yarn@, sasl:mapred@, sasl:mapred@hdfs@ + + + + + The kerberos realm: used to set the realm of + system principals which do not declare their realm, + and any other accounts that need the value. + + If empty, the default realm of the running process + is used. + + If neither are known and the realm is needed, then the registry + service/client will fail. + + hadoop.registry.kerberos.realm + + + + + + Key to define the JAAS context. Used in secure + mode + + hadoop.registry.jaas.context + Client + + + + Defines how often NMs wake up to upload log files. + The default value is -1. By default, the logs will be uploaded when + the application is finished. By setting this configure, logs can be uploaded + periodically when the application is running. The minimum rolling-interval-seconds + can be set is 3600. + + yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds + -1 + + diff --git a/sahara/plugins/vanilla/v2_6_0/versionhandler.py b/sahara/plugins/vanilla/v2_6_0/versionhandler.py new file mode 100644 index 0000000000..9c27d7bff6 --- /dev/null +++ b/sahara/plugins/vanilla/v2_6_0/versionhandler.py @@ -0,0 +1,141 @@ +# Copyright (c) 2014 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo.config import cfg + +from sahara import conductor +from sahara import context +from sahara.openstack.common import log as logging +from sahara.plugins import utils +from sahara.plugins.vanilla import abstractversionhandler as avm +from sahara.plugins.vanilla.hadoop2 import config as c +from sahara.plugins.vanilla.hadoop2 import edp_engine +from sahara.plugins.vanilla.hadoop2 import run_scripts as run +from sahara.plugins.vanilla.hadoop2 import scaling as sc +from sahara.plugins.vanilla.hadoop2 import validation as vl +from sahara.plugins.vanilla import utils as vu +from sahara.plugins.vanilla.v2_4_1 import config_helper as c_helper + + +conductor = conductor.API +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +class VersionHandler(avm.AbstractVersionHandler): + def __init__(self): + self.pctx = { + 'env_confs': c_helper.get_env_configs(), + 'all_confs': c_helper.get_plugin_configs() + } + + def get_plugin_configs(self): + return self.pctx['all_confs'] + + def get_node_processes(self): + return { + "Hadoop": [], + "MapReduce": ["historyserver"], + "HDFS": ["namenode", "datanode", "secondarynamenode"], + "YARN": ["resourcemanager", "nodemanager"], + "JobFlow": ["oozie"] + } + + def validate(self, cluster): + vl.validate_cluster_creating(self.pctx, cluster) + + def update_infra(self, cluster): + pass + + def configure_cluster(self, cluster): + c.configure_cluster(self.pctx, cluster) + + def start_cluster(self, cluster): + nn = vu.get_namenode(cluster) + run.format_namenode(nn) + run.start_hadoop_process(nn, 'namenode') + + for snn in vu.get_secondarynamenodes(cluster): + run.start_hadoop_process(snn, 'secondarynamenode') + + rm = vu.get_resourcemanager(cluster) + if rm: + run.start_yarn_process(rm, 'resourcemanager') + + run.start_all_processes(utils.get_instances(cluster), + ['datanode', 'nodemanager']) + + run.await_datanodes(cluster) + + hs = vu.get_historyserver(cluster) + if hs: + run.start_historyserver(hs) + + oo = vu.get_oozie(cluster) + if oo: + run.start_oozie_process(self.pctx, oo) + + self._set_cluster_info(cluster) + + def decommission_nodes(self, cluster, instances): + sc.decommission_nodes(self.pctx, cluster, instances) + + def validate_scaling(self, cluster, existing, additional): + vl.validate_additional_ng_scaling(cluster, additional) + vl.validate_existing_ng_scaling(self.pctx, cluster, existing) + + def scale_cluster(self, cluster, instances): + sc.scale_cluster(self.pctx, cluster, instances) + + def _set_cluster_info(self, cluster): + nn = vu.get_namenode(cluster) + rm = vu.get_resourcemanager(cluster) + hs = vu.get_historyserver(cluster) + oo = vu.get_oozie(cluster) + + info = {} + + if rm: + info['YARN'] = { + 'Web UI': 'http://%s:%s' % (rm.management_ip, '8088'), + 'ResourceManager': 'http://%s:%s' % (rm.management_ip, '8032') + } + + if nn: + info['HDFS'] = { + 'Web UI': 'http://%s:%s' % (nn.management_ip, '50070'), + 'NameNode': 'hdfs://%s:%s' % (nn.hostname(), '9000') + } + + if oo: + info['JobFlow'] = { + 'Oozie': 'http://%s:%s' % (oo.management_ip, '11000') + } + + if hs: + info['MapReduce JobHistory Server'] = { + 'Web UI': 'http://%s:%s' % (hs.management_ip, '19888') + } + + ctx = context.ctx() + conductor.cluster_update(ctx, cluster, {'info': info}) + + def get_edp_engine(self, cluster, job_type): + if job_type in edp_engine.EdpOozieEngine.get_supported_job_types(): + return edp_engine.EdpOozieEngine(cluster) + return None + + def get_open_ports(self, node_group): + return c.get_open_ports(node_group)