From b3f8af6040c96432820206bca0ae51312c4b5c35 Mon Sep 17 00:00:00 2001 From: Artem Osadchyi Date: Wed, 26 Aug 2015 14:05:55 +0300 Subject: [PATCH] Added support of Hue 3.8.1 to MapR plugin Implements: blueprint mapr-hue-381 Change-Id: Ie8db13b91ee2fa37b3eeffa30ba613d1b6ce2ef7 --- sahara/plugins/mapr/services/hue/hue.py | 17 + .../services/hue/resources/hue_3.8.1.template | 1136 +++++++++++++++++ sahara/plugins/mapr/util/validation_utils.py | 26 + .../versions/v5_0_0_mrv1/version_handler.py | 1 + .../versions/v5_0_0_mrv2/version_handler.py | 1 + 5 files changed, 1181 insertions(+) create mode 100644 sahara/plugins/mapr/services/hue/resources/hue_3.8.1.template diff --git a/sahara/plugins/mapr/services/hue/hue.py b/sahara/plugins/mapr/services/hue/hue.py index 2aaaa87a..2c1742c4 100755 --- a/sahara/plugins/mapr/services/hue/hue.py +++ b/sahara/plugins/mapr/services/hue/hue.py @@ -46,6 +46,13 @@ HUE = np.NodeProcess( open_ports=[8002, 8888] ) +HUE_LIVY = np.NodeProcess( + name="hue-livy", + ui_name="Hue Livy", + package="mapr-hue-livy", + open_ports=[8998] +) + @six.add_metaclass(s.Single) class Hue(s.Service): @@ -214,3 +221,13 @@ class HueV370(Hue): def __init__(self): super(HueV370, self).__init__() self._version = '3.7.0' + + +@six.add_metaclass(s.Single) +class HueV381(Hue): + def __init__(self): + super(HueV381, self).__init__() + self._version = "3.8.1" + self._dependencies = [("mapr-hue-base", self.version)] + self._node_processes = [HUE, HUE_LIVY] + self._validation_rules.append(vu.at_most(1, HUE_LIVY)) diff --git a/sahara/plugins/mapr/services/hue/resources/hue_3.8.1.template b/sahara/plugins/mapr/services/hue/resources/hue_3.8.1.template new file mode 100644 index 00000000..845a633b --- /dev/null +++ b/sahara/plugins/mapr/services/hue/resources/hue_3.8.1.template @@ -0,0 +1,1136 @@ +# Hue configuration file +# =================================== +# +# For complete documentation about the contents of this file, run +# $ /build/env/bin/hue config_help +# +# All .ini files under the current directory are treated equally. Their +# contents are merged to form the Hue configuration, which can +# can be viewed on the Hue at +# http://:/dump_config + + +########################################################################### +# General configuration for core Desktop features (authentication, etc) +########################################################################### + +[desktop] + + # Set this to a random string, the longer the better. + # This is used for secure hashing in the session store. + secret_key={{ secret_key }} + + # Webserver listens on this address and port + http_host=0.0.0.0 + http_port=8888 + + # Time zone name + time_zone=America/Los_Angeles + + # Enable or disable Django debug mode. + django_debug_mode=false + + # Enable or disable backtrace for server error + http_500_debug_mode=false + + # Enable or disable memory profiling. + ## memory_profiler=false + + # Server email for internal error messages + ## django_server_email='hue@localhost.localdomain' + + # Email backend + ## django_email_backend=django.core.mail.backends.smtp.EmailBackend + + # Webserver runs as this user + server_user=mapr + server_group=mapr + + # This should be the Hue admin and proxy user + default_user=mapr + # This should be the hadoop cluster admin + default_hdfs_superuser=mapr + + default_jobtracker_host=maprfs:/// + # If set to false, runcpserver will not actually start the web server. + # Used if Apache is being used as a WSGI container. + ## enable_server=yes + + # This should be the Hue admin and proxy user + ## default_user=hue + + # This should be the hadoop cluster admin + ## default_hdfs_superuser=hdfs + + # If set to false, runcpserver will not actually start the web server. + # Used if Apache is being used as a WSGI container. + ## enable_server=yes + + # Number of threads used by the CherryPy web server + ## cherrypy_server_threads=40 + + # Filename of SSL Certificate + ## ssl_certificate= + + # Filename of SSL RSA Private Key + ## ssl_private_key= + + # SSL certificate password + ## ssl_password= + + # Execute this script to produce the SSL password. This will be used when `ssl_password` is not set. + ## ssl_password_script= + + # List of allowed and disallowed ciphers in cipher list format. + # See http://www.openssl.org/docs/apps/ciphers.html for more information on cipher list format. + ## ssl_cipher_list=DEFAULT:!aNULL:!eNULL:!LOW:!EXPORT:!SSLv2 + + # LDAP username and password of the hue user used for LDAP authentications. + # Set it to use LDAP Authentication with HiveServer2 and Impala. + ## ldap_username=hue + ## ldap_password= + + # Default encoding for site data + ## default_site_encoding=utf-8 + + # Help improve Hue with anonymous usage analytics. + # Use Google Analytics to see how many times an application or specific section of an application is used, nothing more. + ## collect_usage=true + + # Support for HTTPS termination at the load-balancer level with SECURE_PROXY_SSL_HEADER. + ## secure_proxy_ssl_header=false + + # Comma-separated list of Django middleware classes to use. + # See https://docs.djangoproject.com/en/1.4/ref/middleware/ for more details on middlewares in Django. + ## middleware=desktop.auth.backend.LdapSynchronizationBackend + + # Comma-separated list of regular expressions, which match the redirect URL. + # For example, to restrict to your local domain and FQDN, the following value can be used: + # ^\/.*$,^http:\/\/www.mydomain.com\/.*$ + ## redirect_whitelist=^\/.*$ + + # Comma separated list of apps to not load at server startup. + # e.g.: pig,zookeeper + app_blacklist=spark,search,zookeeper,security{{ ",impala" if not impala_host }}{{ ",hbase" if not hbase_host }}{{ ",sqoop" if not sqoop_host }} + + # The directory where to store the auditing logs. Auditing is disable if the value is empty. + # e.g. /var/log/hue/audit.log + ## audit_event_log_dir= + + # Size in KB/MB/GB for audit log to rollover. + ## audit_log_max_file_size=100MB + + # A json file containing a list of log redaction rules for cleaning sensitive data + # from log files. It is defined as: + # + # { + # "version": 1, + # "rules": [ + # { + # "description": "This is the first rule", + # "trigger": "triggerstring 1", + # "search": "regex 1", + # "replace": "replace 1" + # }, + # { + # "description": "This is the second rule", + # "trigger": "triggerstring 2", + # "search": "regex 2", + # "replace": "replace 2" + # } + # ] + # } + # + # Redaction works by searching a string for the [TRIGGER] string. If found, + # the [REGEX] is used to replace sensitive information with the + # [REDACTION_MASK]. If specified with `log_redaction_string`, the + # `log_redaction_string` rules will be executed after the + # `log_redaction_file` rules. + # + # For example, here is a file that would redact passwords and social security numbers: + + # { + # "version": 1, + # "rules": [ + # { + # "description": "Redact passwords", + # "trigger": "password", + # "search": "password=\".*\"", + # "replace": "password=\"???\"" + # }, + # { + # "description": "Redact social security numbers", + # "trigger": "", + # "search": "\d{3}-\d{2}-\d{4}", + # "replace": "XXX-XX-XXXX" + # } + # ] + # } + ## log_redaction_file= + + # Comma separated list of strings representing the host/domain names that the Hue server can serve. + # e.g.: localhost,domain1,* + ## allowed_hosts=* + + # Administrators + # ---------------- + [[django_admins]] + ## [[[admin1]]] + ## name=john + ## email=john@doe.com + + # UI customizations + # ------------------- + [[custom]] + + # Top banner HTML code + # e.g.

Test Lab A2 Hue Services

+ ## banner_top_html= + + # Configuration options for user authentication into the web application + # ------------------------------------------------------------------------ + [[auth]] + + # Authentication backend. Common settings are: + # - django.contrib.auth.backends.ModelBackend (entirely Django backend) + # - desktop.auth.backend.AllowAllBackend (allows everyone) + # - desktop.auth.backend.AllowFirstUserDjangoBackend + # (Default. Relies on Django and user manager, after the first login) + # - desktop.auth.backend.LdapBackend + # - desktop.auth.backend.PamBackend - WARNING: existing users in Hue may be unaccessible if they not exist in OS + # - desktop.auth.backend.SpnegoDjangoBackend + # - desktop.auth.backend.RemoteUserDjangoBackend + # - libsaml.backend.SAML2Backend + # - libopenid.backend.OpenIDBackend + # - liboauth.backend.OAuthBackend + # (Support Twitter, Facebook, Google+ and Linkedin + backend=desktop.auth.backend.PamBackend + + # The service to use when querying PAM. + pam_service=sudo sshd login + + # When using the desktop.auth.backend.RemoteUserDjangoBackend, this sets + # the normalized name of the header that contains the remote user. + # The HTTP header in the request is converted to a key by converting + # all characters to uppercase, replacing any hyphens with underscores + # and adding an HTTP_ prefix to the name. So, for example, if the header + # is called Remote-User that would be configured as HTTP_REMOTE_USER + # + # Defaults to HTTP_REMOTE_USER + ## remote_user_header=HTTP_REMOTE_USER + + # Ignore the case of usernames when searching for existing users. + # Only supported in remoteUserDjangoBackend. + ## ignore_username_case=true + + # Ignore the case of usernames when searching for existing users to authenticate with. + # Only supported in remoteUserDjangoBackend. + ## force_username_lowercase=true + + # Users will expire after they have not logged in for 'n' amount of seconds. + # A negative number means that users will never expire. + ## expires_after=-1 + + # Apply 'expires_after' to superusers. + ## expire_superusers=true + + # Configuration options for connecting to LDAP and Active Directory + # ------------------------------------------------------------------- + [[ldap]] + + # The search base for finding users and groups + ## base_dn="DC=mycompany,DC=com" + + # URL of the LDAP server + ## ldap_url=ldap://auth.mycompany.com + + # A PEM-format file containing certificates for the CA's that + # Hue will trust for authentication over TLS. + # The certificate for the CA that signed the + # LDAP server certificate must be included among these certificates. + # See more here http://www.openldap.org/doc/admin24/tls.html. + ## ldap_cert= + ## use_start_tls=true + + # Distinguished name of the user to bind as -- not necessary if the LDAP server + # supports anonymous searches + ## bind_dn="CN=ServiceAccount,DC=mycompany,DC=com" + + # Password of the bind user -- not necessary if the LDAP server supports + # anonymous searches + ## bind_password= + + # Execute this script to produce the bind user password. This will be used + # when `bind_password` is not set. + ## bind_password_script= + + # Pattern for searching for usernames -- Use for the parameter + # For use when using LdapBackend for Hue authentication + # If nt_domain is specified, this config is completely ignored. + # If nt_domain is not specified, this should take on the form "cn=,dc=example,dc=com", + # where is replaced by whatever is provided at the login page. Depending on your ldap schema, + # you can also specify additional/alternative comma-separated attributes like uid, ou, etc + ## ldap_username_pattern="uid=,ou=People,dc=mycompany,dc=com" + + # Create users in Hue when they try to login with their LDAP credentials + # For use when using LdapBackend for Hue authentication + ## create_users_on_login = true + + # Synchronize a users groups when they login + ## sync_groups_on_login=false + + # Ignore the case of usernames when searching for existing users in Hue. + ## ignore_username_case=false + + # Ignore the case of usernames when searching for existing users to authenticate with. + # Only supported in remoteUserDjangoBackend. + ## force_username_lowercase=false + + # Use search bind authentication. + # If set to true, hue will perform ldap search using bind credentials above (bind_dn, bind_password) + # Hue will then search using the 'base_dn' for an entry with attr defined in 'user_name_attr', with the value + # of short name provided on the login page. The search filter defined in 'user_filter' will also be used to limit + # the search. Hue will search the entire subtree starting from base_dn. + # If search_bind_authentication is set to false, Hue performs a direct bind to LDAP using the credentials provided + # (not bind_dn and bind_password specified in hue.ini). There are 2 modes here - 'nt_domain' is specified or not. + ## search_bind_authentication=true + + # Choose which kind of subgrouping to use: nested or suboordinate (deprecated). + ## subgroups=suboordinate + + # Define the number of levels to search for nested members. + ## nested_members_search_depth=10 + + # Whether or not to follow referrals + ## follow_referrals=false + + # Enable python-ldap debugging. + ## debug=false + + # Sets the debug level within the underlying LDAP C lib. + ## debug_level=255 + + # Possible values for trace_level are 0 for no logging, 1 for only logging the method calls with arguments, + # 2 for logging the method calls with arguments and the complete results and 9 for also logging the traceback of method calls. + ## trace_level=0 + + [[[users]]] + + # Base filter for searching for users + ## user_filter="objectclass=*" + + # The username attribute in the LDAP schema + ## user_name_attr=sAMAccountName + + [[[groups]]] + + # Base filter for searching for groups + ## group_filter="objectclass=*" + + # The group name attribute in the LDAP schema + ## group_name_attr=cn + + # The attribute of the group object which identifies the members of the group + ## group_member_attr=members + + [[[ldap_servers]]] + + ## [[[[mycompany]]]] + + # The search base for finding users and groups + ## base_dn="DC=mycompany,DC=com" + + # URL of the LDAP server + ## ldap_url=ldap://auth.mycompany.com + + # A PEM-format file containing certificates for the CA's that + # Hue will trust for authentication over TLS. + # The certificate for the CA that signed the + # LDAP server certificate must be included among these certificates. + # See more here http://www.openldap.org/doc/admin24/tls.html. + ## ldap_cert= + ## use_start_tls=true + + # Distinguished name of the user to bind as -- not necessary if the LDAP server + # supports anonymous searches + ## bind_dn="CN=ServiceAccount,DC=mycompany,DC=com" + + # Password of the bind user -- not necessary if the LDAP server supports + # anonymous searches + ## bind_password= + + # Execute this script to produce the bind user password. This will be used + # when `bind_password` is not set. + ## bind_password_script= + + # Pattern for searching for usernames -- Use for the parameter + # For use when using LdapBackend for Hue authentication + ## ldap_username_pattern="uid=,ou=People,dc=mycompany,dc=com" + + ## Use search bind authentication. + ## search_bind_authentication=true + + # Whether or not to follow referrals + ## follow_referrals=false + + # Enable python-ldap debugging. + ## debug=false + + # Sets the debug level within the underlying LDAP C lib. + ## debug_level=255 + + # Possible values for trace_level are 0 for no logging, 1 for only logging the method calls with arguments, + # 2 for logging the method calls with arguments and the complete results and 9 for also logging the traceback of method calls. + ## trace_level=0 + + ## [[[[[users]]]]] + + # Base filter for searching for users + ## user_filter="objectclass=Person" + + # The username attribute in the LDAP schema + ## user_name_attr=sAMAccountName + + ## [[[[[groups]]]]] + + # Base filter for searching for groups + ## group_filter="objectclass=groupOfNames" + + # The username attribute in the LDAP schema + ## group_name_attr=cn + + # Configuration options for specifying the Desktop Database. For more info, + # see http://docs.djangoproject.com/en/1.4/ref/settings/#database-engine + # ------------------------------------------------------------------------ + [[database]] + # Database engine is typically one of: + # postgresql_psycopg2, mysql, sqlite3 or oracle. + # + # Note that for sqlite3, 'name', below is a path to the filename. For other backends, it is the database name. + # Note for Oracle, options={'threaded':true} must be set in order to avoid crashes. + # Note for Oracle, you can use the Oracle Service Name by setting "port=0" and then "name=:/". + engine=mysql + host={{ db_host }} + port=3306 + user={{ hue_user }} + password={{ hue_password }} + name={{ hue_name }} + ## options={} + + # Configuration options for specifying the Desktop session. + # For more info, see https://docs.djangoproject.com/en/1.4/topics/http/sessions/ + # ------------------------------------------------------------------------ + [[session]] + # The cookie containing the users' session ID will expire after this amount of time in seconds. + # Default is 2 weeks. + ## ttl=1209600 + + # The cookie containing the users' session ID will be secure. + # Should only be enabled with HTTPS. + ## secure=false + + # The cookie containing the users' session ID will use the HTTP only flag. + ## http_only=false + + # Use session-length cookies. Logs out the user when she closes the browser window. + ## expire_at_browser_close=false + + + # Configuration options for connecting to an external SMTP server + # ------------------------------------------------------------------------ + [[smtp]] + + # The SMTP server information for email notification delivery + host=localhost + port=25 + user= + password= + + # Whether to use a TLS (secure) connection when talking to the SMTP server + tls=no + + # Default email address to use for various automated notification from Hue + ## default_from_email=hue@localhost + + + # Configuration options for Kerberos integration for secured Hadoop clusters + # ------------------------------------------------------------------------ + [[kerberos]] + + # Path to Hue's Kerberos keytab file + ## hue_keytab= + # Kerberos principal name for Hue + ## hue_principal=hue/hostname.foo.com + # Path to kinit + ## kinit_path=/path/to/kinit + + + # Configuration options for using OAuthBackend (core) login + # ------------------------------------------------------------------------ + [[oauth]] + # The Consumer key of the application + ## consumer_key=XXXXXXXXXXXXXXXXXXXXX + + # The Consumer secret of the application + ## consumer_secret=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + + # The Request token URL + ## request_token_url=https://api.twitter.com/oauth/request_token + + # The Access token URL + ## access_token_url=https://api.twitter.com/oauth/access_token + + # The Authorize URL + ## authenticate_url=https://api.twitter.com/oauth/authorize + + +########################################################################### +# Settings to configure SAML +########################################################################### + +[libsaml] + # Xmlsec1 binary path. This program should be executable by the user running Hue. + ## xmlsec_binary=/usr/local/bin/xmlsec1 + + # Entity ID for Hue acting as service provider. + # Can also accept a pattern where '' will be replaced with server URL base. + ## entity_id="/saml2/metadata/" + + # Create users from SSO on login. + ## create_users_on_login=true + + # Required attributes to ask for from IdP. + # This requires a comma separated list. + ## required_attributes=uid + + # Optional attributes to ask for from IdP. + # This requires a comma separated list. + ## optional_attributes= + + # IdP metadata in the form of a file. This is generally an XML file containing metadata that the Identity Provider generates. + ## metadata_file= + + # Private key to encrypt metadata with. + ## key_file= + + # Signed certificate to send along with encrypted metadata. + ## cert_file= + + # A mapping from attributes in the response from the IdP to django user attributes. + ## user_attribute_mapping={'uid':'username'} + + # Have Hue initiated authn requests be signed and provide a certificate. + ## authn_requests_signed=false + + # Have Hue initiated logout requests be signed and provide a certificate. + ## logout_requests_signed=false + + ## Username can be sourced from 'attributes' or 'nameid'. + ## username_source=attributes + + # Performs the logout or not. + ## logout_enabled=true + + +########################################################################### +# Settings to configure OPENID +########################################################################### + +[libopenid] + # (Required) OpenId SSO endpoint url. + ## server_endpoint_url=https://www.google.com/accounts/o8/id + + # OpenId 1.1 identity url prefix to be used instead of SSO endpoint url + # This is only supported if you are using an OpenId 1.1 endpoint + ## identity_url_prefix=https://app.onelogin.com/openid/your_company.com/ + + # Create users from OPENID on login. + ## create_users_on_login=true + + # Use email for username + ## use_email_for_username=true + + +########################################################################### +# Settings to configure OAuth +########################################################################### + +[liboauth] + # NOTE: + # To work, each of the active (i.e. uncommented) service must have + # applications created on the social network. + # Then the "consumer key" and "consumer secret" must be provided here. + # + # The addresses where to do so are: + # Twitter: https://dev.twitter.com/apps + # Google+ : https://cloud.google.com/ + # Facebook: https://developers.facebook.com/apps + # Linkedin: https://www.linkedin.com/secure/developer + # + # Additionnaly, the following must be set in the application settings: + # Twitter: Callback URL (aka Redirect URL) must be set to http://YOUR_HUE_IP_OR_DOMAIN_NAME/oauth/social_login/oauth_authenticated + # Google+ : CONSENT SCREEN must have email address + # Facebook: Sandbox Mode must be DISABLED + # Linkedin: "In OAuth User Agreement", r_emailaddress is REQUIRED + + # The Consumer key of the application + ## consumer_key_twitter= + ## consumer_key_google= + ## consumer_key_facebook= + ## consumer_key_linkedin= + + # The Consumer secret of the application + ## consumer_secret_twitter= + ## consumer_secret_google= + ## consumer_secret_facebook= + ## consumer_secret_linkedin= + + # The Request token URL + ## request_token_url_twitter=https://api.twitter.com/oauth/request_token + ## request_token_url_google=https://accounts.google.com/o/oauth2/auth + ## request_token_url_linkedin=https://www.linkedin.com/uas/oauth2/authorization + ## request_token_url_facebook=https://graph.facebook.com/oauth/authorize + + # The Access token URL + ## access_token_url_twitter=https://api.twitter.com/oauth/access_token + ## access_token_url_google=https://accounts.google.com/o/oauth2/token + ## access_token_url_facebook=https://graph.facebook.com/oauth/access_token + ## access_token_url_linkedin=https://api.linkedin.com/uas/oauth2/accessToken + + # The Authenticate URL + ## authenticate_url_twitter=https://api.twitter.com/oauth/authorize + ## authenticate_url_google=https://www.googleapis.com/oauth2/v1/userinfo?access_token= + ## authenticate_url_facebook=https://graph.facebook.com/me?access_token= + ## authenticate_url_linkedin=https://api.linkedin.com/v1/people/~:(email-address)?format=json&oauth2_access_token= + + # Username Map. Json Hash format. + # Replaces username parts in order to simplify usernames obtained + # Example: {"@sub1.domain.com":"_S1", "@sub2.domain.com":"_S2"} + # converts 'email@sub1.domain.com' to 'email_S1' + ## username_map={} + + # Whitelisted domains (only applies to Google OAuth). CSV format. + ## whitelisted_domains_google= + +########################################################################### +# Settings for the RDBMS application +########################################################################### + +[librdbms] + # The RDBMS app can have any number of databases configured in the databases + # section. A database is known by its section name + # (IE sqlite, mysql, psql, and oracle in the list below). + + [[databases]] + # sqlite configuration. + ## [[[sqlite]]] + # Name to show in the UI. + ## nice_name=SQLite + + # For SQLite, name defines the path to the database. + ## name=/opt/mapr/hue/hue-3.8.1/desktop/desktop.db + + # Database backend to use. + ## engine=sqlite + + # Database options to send to the server when connecting. + # https://docs.djangoproject.com/en/1.4/ref/databases/ + ## options={} + + # mysql, oracle, or postgresql configuration. +{%- if db_host and rdbms_name %} + [[[mysql]]] + # Name to show in the UI. + nice_name="MySQL DB" + + # For MySQL and PostgreSQL, name is the name of the database. + # For Oracle, Name is instance of the Oracle server. For express edition + # this is 'xe' by default. + name={{ rdbms_name }} + + # Database backend to use. This can be: + # 1. mysql + # 2. postgresql + # 3. oracle + engine=mysql + + # IP or hostname of the database to connect to. + host={{ db_host }} + + # Port the database server is listening to. Defaults are: + # 1. MySQL: 3306 + # 2. PostgreSQL: 5432 + # 3. Oracle Express Edition: 1521 + port=3306 + + # Username to authenticate with when connecting to the database. + user={{ rdbms_user }} + + # Password matching the username to authenticate with when + # connecting to the database. + password={{ rdbms_password }} + + # Database options to send to the server when connecting. + # https://docs.djangoproject.com/en/1.4/ref/databases/ + ## options={} +{%- endif %} + +########################################################################### +# Settings to configure your Hadoop cluster. +########################################################################### + +[hadoop] + + # Configuration for HDFS NameNode + # ------------------------------------------------------------------------ + [[hdfs_clusters]] + # HA support by using HttpFs + + [[[default]]] + # Enter the filesystem uri + fs_defaultfs=maprfs:/// + + # NameNode logical name. + ## logical_name= + + # Use WebHdfs/HttpFs as the communication mechanism. + # Domain should be the NameNode or HttpFs host. + # Default port is 14000 for HttpFs. + webhdfs_url=http://{{ webhdfs_url | default("localhost", True) }}:14000/webhdfs/v1 + + # Change this if your HDFS cluster is secured + security_enabled=${security_enabled} + + # Security mechanism of authentication none/GSSAPI/MAPR-SECURITY + mechanism=${mechanism} + + # In secure mode (HTTPS), if SSL certificates from YARN Rest APIs + # have to be verified against certificate authority + ## ssl_cert_ca_verify=True + + # File size restriction for viewing file (float) + # '1.0' - default 1 GB file size restriction + # '0' - no file size restrictions + # >0 - set file size restriction in gigabytes, ex. 0.5, 1.0, 1.2... + ## file_size=1.0 + + # Directory of the Hadoop configuration + ## hadoop_conf_dir=$HADOOP_CONF_DIR when set or '/etc/hadoop/conf' + +{% if yarn_mode %} + # Configuration for YARN (MR2) + # ------------------------------------------------------------------------ + [[yarn_clusters]] + + [[[default]]] + # Enter the host on which you are running the ResourceManager + resourcemanager_host={{ rm_host | default("localhost", True) }} + + # The port where the ResourceManager IPC listens on + resourcemanager_port={{ rm_port | default("8032", True) }} + + # Whether to submit jobs to this cluster + submit_to={{ "True" if yarn_mode else "False" }} + + # Change this if your YARN cluster is secured + ## security_enabled=false + + # Security mechanism of authentication none/GSSAPI/MAPR-SECURITY + ## mechanism=MAPR-SECURITY + + # URL of the ResourceManager API + resourcemanager_api_url=http://{{ rm_host | default("localhost", True) }}:{{ rm_api_port | default("8088", True)}} + + # URL of the ProxyServer API + proxy_api_url=http://{{ rm_proxy_api | default(rm_host, True) }}:{{ rm_proxy_port | default("8088", True)}} + + # URL of the HistoryServer API + history_server_api_url=http://{{ hs_host | default(rm_host, True) }}:{{ hs_api_port | default("19888", True)}} + + # In secure mode (HTTPS), if SSL certificates from from YARN Rest APIs + # have to be verified against certificate authority + ## ssl_cert_ca_verify=True + + # HA support by specifying multiple clusters + # e.g. + + {% for rm in ha_rms %} + [[[ha{{ loop.index }}]]] + # Enter the host on which you are running the failover Resource Manager + resourcemanager_api_url=http://{{ rm }}:8088 + logical_name=ha-rm-{{ loop.index }} + submit_to=True + {% endfor %} + +{%- endif %} + +{%- if not yarn_mode %} + # Configuration for MapReduce (MR1) + # ------------------------------------------------------------------------ + [[mapred_clusters]] + + [[[default]]] + # Enter the host on which you are running the Hadoop JobTracker + jobtracker_host={{ jt_host | default("localhost", True) }} + + # The port where the JobTracker IPC listens on + jobtracker_port={{ jt_port | default("9001", True) }} + + # JobTracker logical name for HA + ## logical_name= + + # Thrift plug-in port for the JobTracker + thrift_port={{ jt_plugin_port | default("9290", True) }} + + # Whether to submit jobs to this cluster + submit_to={{ "False" if yarn_mode else "True" }} + + # Change this if your MapReduce cluster is secured + security_enabled=${security_enabled} + + # Security mechanism of authentication none/GSSAPI/MAPR-SECURITY + mechanism=${mechanism} + + {%- if fo_jobtracker_host %} + # HA support by specifying multiple clusters + # e.g. + + # [[[ha]]] + # Enter the logical name of the JobTrackers + ## logical_name=my-jt-name + {%- endif %} +{%- endif %} + +########################################################################### +# Settings to configure the Filebrowser app +########################################################################### + +[filebrowser] + # Location on local filesystem where the uploaded archives are temporary stored. + ## archive_upload_tempdir=/tmp + +########################################################################### +# Settings to configure liboozie +########################################################################### + +[liboozie] + # The URL where the Oozie service runs on. This is required in order for + # users to submit jobs. + oozie_url=http://{{ oozie_host | default("localhost", True) }}:{{ oozie_port | default("11000", True) }}/oozie + + # Requires FQDN in oozie_url if enabled + ## security_enabled=false + + # Location on HDFS where the workflows/coordinator are deployed when submitted. + ## remote_deployement_dir=/oozie/deployments + + # Security mechanism of authentication none/GSSAPI/MAPR-SECURITY + mechanism=${mechanism} + +########################################################################### +# Settings to configure the Oozie app +########################################################################### + +[oozie] + # Location on local FS where the examples are stored. + ## local_data_dir=..../examples + + # Location on local FS where the data for the examples is stored. + ## sample_data_dir=...thirdparty/sample_data + + # Location on HDFS where the oozie examples and workflows are stored. + remote_data_dir=/oozie/workspaces + + # Maximum of Oozie workflows or coodinators to retrieve in one API call. + ## oozie_jobs_count=100 + + # Use Cron format for defining the frequency of a Coordinator instead of the old frequency number/unit. + ## enable_cron_scheduling=true + + +########################################################################### +# Settings to configure Beeswax with Hive +########################################################################### + +{%- if hive_host %} +[beeswax] + + # Host where HiveServer2 is running. + # If Kerberos security is enabled, use fully-qualified domain name (FQDN). + hive_server_host={{ hive_host | default("localhost", True) }} + + # Port where HiveServer2 Thrift server runs on. + hive_server_port={{ hive_port | default("10000", True) }} + + # Hive configuration directory, where hive-site.xml is located + hive_conf_dir={{ hive_conf_dir | default("/opt/mapr/hive/hive-1.0/conf", True) }} + + # Timeout in seconds for thrift calls to Hive service + ## server_conn_timeout=120 + + # Security mechanism of authentication none/GSSAPI/MAPR-SECURITY + mechanism=${mechanism} + + # Path to HiveServer2 start script + hive_server_bin=/opt/mapr/hive/hive-{{ hive_version | default("1.0", True) }}/bin/hiveserver2 + + # Choose whether to use the old GetLog() thrift call from before Hive 0.14 to retrieve the logs. + # If false, use the FetchResults() thrift call from Hive 1.0 or more instead. + ## use_get_log_api=false + + # Set a LIMIT clause when browsing a partitioned table. + # A positive value will be set as the LIMIT. If 0 or negative, do not set any limit. + ## browse_partitioned_table_limit=250 + + # A limit to the number of rows that can be downloaded from a query. + # A value of -1 means there will be no limit. + # A maximum of 65,000 is applied to XLS downloads. + ## download_row_limit=1000000 + + # Hue will try to close the Hive query when the user leaves the editor page. + # This will free all the query resources in HiveServer2, but also make its results inaccessible. + ## close_queries=false + + # Thrift version to use when communicating with HiveServer2. + # New column format is from version 7. + ## thrift_version=7 + + [[ssl]] + # Path to Certificate Authority certificates. + ## cacerts=/etc/hue/cacerts.pem + + # Choose whether Hue should validate certificates received from the server. + ## validate=true +{%- endif %} + +########################################################################### +# Settings to configure Impala +########################################################################### +{%- if impala_host %} +[impala] + # Host of the Impala Server (one of the Impalad) + server_host={{ impala_host | default("localhost", True) }} + + # Port of the Impala Server + server_port={{ impala_port | default("21050", True) }} + + # Kerberos principal + ## impala_principal=impala/hostname.foo.com + + # Turn on/off impersonation mechanism when talking to Impala + impersonation_enabled=False + + # Number of initial rows of a result set to ask Impala to cache in order + # to support re-fetching them for downloading them. + # Set to 0 for disabling the option and backward compatibility. + querycache_rows=0 + + # Timeout in seconds for thrift calls + ## server_conn_timeout=120 + + # Hue will try to close the Impala query when the user leaves the editor page. + # This will free all the query resources in Impala, but also make its results inaccessible. + ## close_queries=true + + # If QUERY_TIMEOUT_S > 0, the query will be timed out (i.e. cancelled) if Impala does not do any work + # (compute or send back results) for that query within QUERY_TIMEOUT_S seconds. + ## query_timeout_s=0 + + [[ssl]] + # SSL communication enabled for this server. + ## enabled=false + + # Path to Certificate Authority certificates. + ## cacerts=/etc/hue/cacerts.pem + + # Choose whether Hue should validate certificates received from the server. + ## validate=true +{%- endif %} + +########################################################################### +# Settings to configure Pig +########################################################################### + +[pig] + # Location of piggybank.jar on local filesystem. + ## local_sample_dir=/opt/mapr/pig/pig-0.12/contrib/piggybank/java/ + + # Location piggybank.jar will be copied to in HDFS. + ## remote_data_dir=/oozie/pig/examples + + +########################################################################### +# Settings to configure Sqoop +########################################################################### +{%- if sqoop_host %} +[sqoop] + # For autocompletion, fill out the librdbms section. + + # Sqoop server URL + server_url=http://{{ sqoop_host | default("localhost", True) }}:{{ sqoop_port | default("12000", True) }}/sqoop + # Change this if your cluster is secured + security_enabled=${security_enabled} + + # Security mechanism of authentication none/GSSAPI/MAPR-SECURITY + mechanism=${mechanism} + +{%- endif %} + +########################################################################### +# Settings to configure Proxy +########################################################################### + +[proxy] + # Comma-separated list of regular expressions, + # which match 'host:port' of requested proxy target. + ## whitelist=(localhost|127\.0\.0\.1):(50030|50070|50060|50075) + + # Comma-separated list of regular expressions, + # which match any prefix of 'host:port/path' of requested proxy target. + # This does not support matching GET parameters. + ## blacklist= + + +########################################################################### +# Settings to configure HBase Browser +########################################################################### + +{%- if hbase_host %} +[hbase] + # Comma-separated list of HBase Thrift servers for clusters in the format of '(name|host:port)'. + # Use full hostname with security. + hbase_clusters=(HBase|{{ hbase_host | default("localhost", True) }}:{{ hbase_port | default("9090", True) }}) + + # HBase configuration directory, where hbase-site.xml is located. + hbase_conf_dir=/opt/mapr/hbase/hbase-{{ hbase_version | default("0.98.12", True) }}/conf + + # Hard limit of rows or columns per row fetched before truncating. + ## truncate_limit = 500 + + # 'buffered' is the default of the HBase Thrift Server and supports security. + # 'framed' can be used to chunk up responses, + # which is useful when used in conjunction with the nonblocking server in Thrift. + ## thrift_transport=buffered + + # Security mechanism of authentication none/GSSAPI/MAPR-SECURITY + mechanism=${mechanism} +{%- endif %} + + +########################################################################### +# Settings to configure Solr Search +########################################################################### + +[search] + + # URL of the Solr Server + ## solr_url=http://localhost:8983/solr/ + + # Requires FQDN in solr_url if enabled + security_enabled=${security_enabled} + + ## Query sent when no term is entered + ## empty_query=*:* + + +########################################################################### +# Settings to configure Solr Indexer +########################################################################### + +[indexer] + + # Location of the solrctl binary. + ## solrctl_path=/usr/bin/solrctl + + # Zookeeper ensemble. + ## solr_zk_ensemble=localhost:2181/solr + + +########################################################################### +# Settings to configure Job Designer +########################################################################### + +[jobsub] + + # Location on local FS where examples and template are stored. + ## local_data_dir=..../data + + # Location on local FS where sample data is stored + ## sample_data_dir=...thirdparty/sample_data + + +########################################################################### +# Settings to configure Job Browser. +########################################################################### + +[jobbrowser] + # Share submitted jobs information with all users. If set to false, + # submitted jobs are visible only to the owner and administrators. + ## share_jobs=true + + +########################################################################### +# Settings to configure the Zookeeper application. +########################################################################### + +[zookeeper] + + [[clusters]] + + [[[default]]] + # Zookeeper ensemble. Comma separated list of Host/Port. + # e.g. localhost:5181,node2_ip@:5181,node3_ip@:5181 + host_ports={{ zk_hosts_with_port | default("localhost:5181", True) }} + + # The URL of the REST contrib service (required for znode browsing). + ## rest_url=http://localhost:9999 + + # Name of Kerberos principal when using security. + ## principal_name=zookeeper + + +########################################################################### +# Settings to configure the Spark application. +########################################################################### + +[spark] + # URL of the REST Spark Job Server. + ## server_url=http://localhost:8090/ + + # IP or hostname of livy server. + ## livy_server_host=0.0.0.0 + + # List of available types of snippets + ## languages='[{"name": "Scala", "type": "scala"},{"name": "Python", "type": "python"},{"name": "Impala SQL", "type": "impala"},{"name": "Hive SQL", "type": "hive"},{"name": "Text", "type": "text"}]' + + +########################################################################### +# Settings for the User Admin application +########################################################################### + +[useradmin] + # The name of the default user group that users will be a member of + ## default_user_group=default + + [[password_policy]] + # Set password policy to all users. The default policy requires password to be at least 8 characters long, + # and contain both uppercase and lowercase letters, numbers, and special characters. + + ## is_enabled=false + ## pwd_regex="^(?=.*?[A-Z])(?=(.*[a-z]){1,})(?=(.*[\d]){1,})(?=(.*[\W_]){1,}).{8,}$" + ## pwd_hint="The password must be at least 8 characters long, and must contain both uppercase and lowercase letters, at least one number, and at least one special character." + ## pwd_error_message="The password must be at least 8 characters long, and must contain both uppercase and lowercase letters, at least one number, and at least one special character." + +########################################################################### +# Settings for the Sentry lib +########################################################################### + +[libsentry] + # Hostname or IP of server. + ## hostname=localhost + + # Port the sentry service is running on. + ## port=8038 + + # Sentry configuration directory, where sentry-site.xml is located. + ## sentry_conf_dir=/opt/mapr/sentry/sentry-1.4.0/conf diff --git a/sahara/plugins/mapr/util/validation_utils.py b/sahara/plugins/mapr/util/validation_utils.py index 1310a9ea..e6b88b46 100644 --- a/sahara/plugins/mapr/util/validation_utils.py +++ b/sahara/plugins/mapr/util/validation_utils.py @@ -38,6 +38,22 @@ class LessThanCountException(e.InvalidComponentCountException): self.message = LessThanCountException.MESSAGE % args +class MoreThanCountException(e.InvalidComponentCountException): + MESSAGE = _("Hadoop cluster should contain at most" + " %(expected_count)d %(component)s component(s)." + " Actual %(component)s count is %(actual_count)d") + + def __init__(self, component, expected_count, count): + super(MoreThanCountException, self).__init__( + component, expected_count, count) + args = { + "expected_count": expected_count, + "component": component, + "actual_count": count, + } + self.message = MoreThanCountException.MESSAGE % args + + class EvenCountException(ex.SaharaException): MESSAGE = _("Hadoop cluster should contain odd number of %(component)s" " but %(actual_count)s found.") @@ -99,6 +115,16 @@ def at_least(count, component): return ft.partial(validate, component=component, count=count) +def at_most(count, component): + def validate(cluster_context, component, count): + actual_count = cluster_context.get_instances_count(component) + if actual_count > count: + raise MoreThanCountException( + component.ui_name, count, actual_count) + + return ft.partial(validate, component=component, count=count) + + def exactly(count, component): def validate(cluster_context, component, count): actual_count = cluster_context.get_instances_count(component) diff --git a/sahara/plugins/mapr/versions/v5_0_0_mrv1/version_handler.py b/sahara/plugins/mapr/versions/v5_0_0_mrv1/version_handler.py index 664417ef..08de8c07 100644 --- a/sahara/plugins/mapr/versions/v5_0_0_mrv1/version_handler.py +++ b/sahara/plugins/mapr/versions/v5_0_0_mrv1/version_handler.py @@ -55,6 +55,7 @@ class VersionHandler(bvh.BaseVersionHandler): mahout.MahoutV010(), oozie.OozieV410(), hue.HueV370(), + hue.HueV381(), hbase.HBaseV0989(), hbase.HBaseV09812(), drill.DrillV11(), diff --git a/sahara/plugins/mapr/versions/v5_0_0_mrv2/version_handler.py b/sahara/plugins/mapr/versions/v5_0_0_mrv2/version_handler.py index e9d17cb6..d8274a39 100644 --- a/sahara/plugins/mapr/versions/v5_0_0_mrv2/version_handler.py +++ b/sahara/plugins/mapr/versions/v5_0_0_mrv2/version_handler.py @@ -55,6 +55,7 @@ class VersionHandler(bvh.BaseVersionHandler): mahout.MahoutV010(), oozie.OozieV410(), hue.HueV370(), + hue.HueV381(), hbase.HBaseV0989(), hbase.HBaseV09812(), drill.DrillV11(),