# Copyright (c) 2015, MapR Technologies # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import abc import os from oslo_log import log as logging import six from sahara import conductor from sahara import context from sahara.i18n import _LI from sahara.i18n import _LW import sahara.plugins.mapr.abstract.configurer as ac from sahara.plugins.mapr.domain import distro as d import sahara.plugins.mapr.services.management.management as mng import sahara.plugins.mapr.services.mapreduce.mapreduce as mr from sahara.plugins.mapr.services.maprfs import maprfs from sahara.plugins.mapr.services.mysql import mysql import sahara.plugins.mapr.services.yarn.yarn as yarn import sahara.plugins.mapr.util.general as util from sahara.topology import topology_helper as th import sahara.utils.configs as sahara_configs import sahara.utils.files as files LOG = logging.getLogger(__name__) conductor = conductor.API _MAPR_HOME = '/opt/mapr' _JAVA_HOME = '/usr/java/jdk1.7.0_51' _CONFIGURE_SH_TIMEOUT = 600 _SET_MODE_CMD = 'maprcli cluster mapreduce set -mode ' _TOPO_SCRIPT = 'plugins/mapr/resources/topology.sh' INSTALL_JAVA_SCRIPT = 'plugins/mapr/resources/install_java.sh' INSTALL_SCALA_SCRIPT = 'plugins/mapr/resources/install_scala.sh' INSTALL_MYSQL_CLIENT = 'plugins/mapr/resources/install_mysql_client.sh' ADD_MAPR_REPO_SCRIPT = 'plugins/mapr/resources/add_mapr_repo.sh' ADD_SECURITY_REPO_SCRIPT = 'plugins/mapr/resources/add_security_repos.sh' SERVICE_INSTALL_PRIORITY = [ mng.Management(), yarn.YARNv251(), yarn.YARNv241(), yarn.YARNv270(), mr.MapReduce(), maprfs.MapRFS(), ] @six.add_metaclass(abc.ABCMeta) class BaseConfigurer(ac.AbstractConfigurer): def configure(self, cluster_context, instances=None): instances = instances or cluster_context.get_instances() self._configure_ssh_connection(cluster_context, instances) self._install_mapr_repo(cluster_context, instances) if not cluster_context.is_prebuilt: self._prepare_bare_image(cluster_context, instances) self._install_services(cluster_context, instances) self._configure_topology(cluster_context, instances) self._configure_database(cluster_context, instances) self._configure_services(cluster_context, instances) self._configure_sh_cluster(cluster_context, instances) self._set_cluster_mode(cluster_context) self._write_config_files(cluster_context, instances) self._configure_environment(cluster_context, instances) self._update_cluster_info(cluster_context) def update(self, cluster_context, instances=None): LOG.debug('Configuring existing instances') instances = instances or cluster_context.get_instances() existing = cluster_context.existing_instances() self._configure_topology(cluster_context, existing) if cluster_context.has_control_nodes(instances): self._configure_sh_cluster(cluster_context, existing) self._post_configure_sh(cluster_context, existing) self._write_config_files(cluster_context, existing) self._update_services(cluster_context, existing) self._restart_services(cluster_context) self._update_cluster_info(cluster_context) LOG.info(_LI('Existing instances successfully configured')) def _configure_services(self, cluster_context, instances): for service in cluster_context.cluster_services: service.configure(cluster_context, instances) def _install_services(self, cluster_context, instances): for service in self._service_install_sequence(cluster_context): service.install(cluster_context, instances) def _service_install_sequence(self, cluster_context): def key(service): if service in SERVICE_INSTALL_PRIORITY: return SERVICE_INSTALL_PRIORITY.index(service) return -1 return sorted(cluster_context.cluster_services, key=key, reverse=True) def _prepare_bare_image(self, cluster_context, instances): LOG.debug('Preparing bare image') if d.UBUNTU == cluster_context.distro: LOG.debug("Installing security repos") util.execute_on_instances( instances, util.run_script, ADD_SECURITY_REPO_SCRIPT, 'root') d_name = cluster_context.distro.name LOG.debug('Installing Java') util.execute_on_instances( instances, util.run_script, INSTALL_JAVA_SCRIPT, 'root', d_name) LOG.debug('Installing Scala') util.execute_on_instances( instances, util.run_script, INSTALL_SCALA_SCRIPT, 'root', d_name) LOG.debug('Installing MySQL client') util.execute_on_instances( instances, util.run_script, INSTALL_MYSQL_CLIENT, 'root', d_name) LOG.debug('Bare images successfully prepared') def _configure_topology(self, context, instances): def write_file(instance, path, data): with instance.remote() as r: r.write_file_to(path, data, run_as_root=True) LOG.debug('Configuring cluster topology') is_node_aware = context.is_node_aware if is_node_aware: topo = th.generate_topology_map(context.cluster, is_node_aware) topo = '\n'.join(['%s %s' % i for i in six.iteritems(topo)]) + '\n' data_path = '%s/topology.data' % context.mapr_home script = files.get_file_text(_TOPO_SCRIPT) script_path = '%s/topology.sh' % context.mapr_home util.execute_on_instances(instances, write_file, data_path, topo) util.execute_on_instances( instances, util.write_file, script_path, script, '+x', 'root') else: LOG.debug('Data locality is disabled.') LOG.info(_LI('Cluster topology successfully configured')) def _write_config_files(self, cluster_context, instances): LOG.debug('Writing config files') def get_node_groups(instances): return util.unique_list(instances, lambda i: i.node_group) for ng in get_node_groups(instances): ng_services = cluster_context.get_cluster_services(ng) ng_user_configs = ng.configuration() ng_default_configs = cluster_context.get_services_configs_dict( ng_services) ng_configs = sahara_configs.merge_configs( ng_default_configs, ng_user_configs) ng_config_files = dict() for service in ng_services: service_conf_files = service.get_config_files( cluster_context=cluster_context, configs=ng_configs[service.ui_name], instance=ng.instances[0] ) LOG.debug('Rendering {ui_name} config files'.format( ui_name=service.ui_name)) for conf_file in service_conf_files: ng_config_files.update({ conf_file.remote_path: conf_file.render() }) ng_instances = filter(lambda i: i in instances, ng.instances) self._write_ng_config_files(ng_instances, ng_config_files) LOG.debug('Config files successfully wrote') def _write_ng_config_files(self, instances, conf_files): with context.ThreadGroup() as tg: for instance in instances: tg.spawn('write-config-files-%s' % instance.id, self._write_config_files_instance, instance, conf_files) def _configure_environment(self, cluster_context, instances): self.configure_general_environment(cluster_context, instances) self._post_install_services(cluster_context, instances) def _configure_database(self, cluster_context, instances): mysql_instance = mysql.MySQL.get_db_instance(cluster_context) distro_name = cluster_context.distro.name mysql.MySQL.install_mysql(mysql_instance, distro_name) mysql.MySQL.start_mysql_server(cluster_context) mysql.MySQL.create_databases(cluster_context, instances) @staticmethod def _write_config_files_instance(instance, config_files): paths = six.iterkeys(config_files) with instance.remote() as r: for path in paths: r.execute_command('mkdir -p ' + os.path.dirname(path), run_as_root=True) r.write_files_to(config_files, run_as_root=True) def _post_install_services(self, cluster_context, instances): LOG.debug('Executing service post install hooks') for s in cluster_context.cluster_services: s.post_install(cluster_context, instances) LOG.info(_LI('Post install hooks execution successfully executed')) def _update_cluster_info(self, cluster_context): LOG.debug('Updating UI information.') info = {} for service in cluster_context.cluster_services: for title, node_process, url_template in service.ui_info: removed = cluster_context.removed_instances(node_process) instances = cluster_context.get_instances(node_process) instances = [i for i in instances if i not in removed] if len(instances) == 1: display_name_template = "%(title)s" else: display_name_template = "%(title)s %(index)s" for index, instance in enumerate(instances, start=1): args = {"title": title, "index": index} display_name = display_name_template % args url = url_template % instance.management_ip info.update({display_name: {"WebUI": url}}) ctx = context.ctx() conductor.cluster_update(ctx, cluster_context.cluster, {'info': info}) def configure_general_environment(self, cluster_context, instances=None): LOG.debug('Executing post configure hooks') if not instances: instances = cluster_context.get_instances() def set_user_password(instance): LOG.debug('Setting password for user "mapr"') if self.mapr_user_exists(instance): with instance.remote() as r: r.execute_command( 'echo "%s:%s"|chpasswd' % ('mapr', 'mapr'), run_as_root=True) else: LOG.warning(_LW('User "mapr" does not exists')) def create_home_mapr(instance): target_path = '/home/mapr' LOG.debug("Creating home directory for user 'mapr'") args = {'path': target_path} cmd = 'mkdir -p %(path)s && chown mapr:mapr %(path)s' % args if self.mapr_user_exists(instance): with instance.remote() as r: r.execute_command(cmd, run_as_root=True) else: LOG.warning(_LW('User "mapr" does not exists')) util.execute_on_instances(instances, set_user_password) util.execute_on_instances(instances, create_home_mapr) def _configure_sh_cluster(self, cluster_context, instances): LOG.debug('Executing configure.sh') if not instances: instances = cluster_context.get_instances() script = cluster_context.configure_sh db_specs = dict(mysql.MySQL.METRICS_SPECS._asdict()) db_specs.update({ 'host': mysql.MySQL.get_db_instance(cluster_context).internal_ip, 'port': mysql.MySQL.MYSQL_SERVER_PORT, }) with context.ThreadGroup() as tg: for instance in instances: tg.spawn('configure-sh-%s' % instance.id, self._configure_sh_instance, cluster_context, instance, script, db_specs) LOG.debug('Executing configure.sh successfully completed') def _configure_sh_instance(self, context, instance, command, specs): if not self.mapr_user_exists(instance): command += ' --create-user' if context.check_for_process(instance, mng.METRICS): command += (' -d %(host)s:%(port)s -du %(user)s -dp %(password)s ' '-ds %(db_name)s') % specs with instance.remote() as r: r.execute_command('sudo -i ' + command, timeout=_CONFIGURE_SH_TIMEOUT) def _configure_ssh_connection(self, cluster_context, instances): def keep_alive_connection(instance): echo_param = 'echo "KeepAlive yes" >> ~/.ssh/config' echo_timeout = 'echo "ServerAliveInterval 60" >> ~/.ssh/config' with instance.remote() as r: r.execute_command(echo_param) r.execute_command(echo_timeout) util.execute_on_instances(instances, keep_alive_connection) def mapr_user_exists(self, instance): with instance.remote() as r: ec, __ = r.execute_command( "id -u mapr", run_as_root=True, raise_when_error=False) return ec == 0 def post_start(self, c_context, instances=None): instances = instances or c_context.get_instances() LOG.debug('Executing service post start hooks') for service in c_context.cluster_services: updated = c_context.filter_instances(instances, service=service) service.post_start(c_context, updated) LOG.info(_LI('Post start hooks successfully executed')) def _set_cluster_mode(self, cluster_context): cluster_mode = cluster_context.cluster_mode if not cluster_mode: return cmd = 'maprcli cluster mapreduce set -mode %s' % cluster_mode util.execute_command(cluster_context.get_instances(), cmd, 'mapr') def _install_mapr_repo(self, cluster_context, instances): d_name = cluster_context.distro.name util.execute_on_instances( instances, util.run_script, ADD_MAPR_REPO_SCRIPT, 'root', d_name, **cluster_context.mapr_repos) def _update_services(self, c_context, instances): for service in c_context.cluster_services: updated = c_context.filter_instances(instances, service=service) service.update(c_context, updated) def _restart_services(self, cluster_context): restart = cluster_context.should_be_restarted for service, instances in six.iteritems(restart): service.restart(util.unique_list(instances)) def _post_configure_sh(self, cluster_context, instances): LOG.debug('Executing post configure.sh hooks') for service in cluster_context.cluster_services: service.post_configure_sh(cluster_context, instances) LOG.info(_LI('Post configure.sh hooks successfully executed'))