From 92c3baab01db99cec7f36c09eb4d12f89d6d09f2 Mon Sep 17 00:00:00 2001 From: Artem Osadchyi Date: Wed, 18 Feb 2015 18:00:48 +0200 Subject: [PATCH] Add Drill support for MapR plugin Change-Id: I78e199f360125e72d74c3f8ce07e7d315856bd68 Implements: blueprint mapr-drill --- .../plugins/mapr/abstract/cluster_context.py | 8 +++ .../plugins/mapr/base/base_cluster_context.py | 9 +++- .../plugins/mapr/services/drill/__init__.py | 0 sahara/plugins/mapr/services/drill/drill.py | 52 +++++++++++++++++++ sahara/plugins/mapr/util/commands.py | 6 +++ .../mapr/versions/v3_1_1/version_handler.py | 2 + .../versions/v4_0_1_mrv1/version_handler.py | 2 + .../versions/v4_0_1_mrv2/version_handler.py | 2 + 8 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 sahara/plugins/mapr/services/drill/__init__.py create mode 100644 sahara/plugins/mapr/services/drill/drill.py diff --git a/sahara/plugins/mapr/abstract/cluster_context.py b/sahara/plugins/mapr/abstract/cluster_context.py index 451d0290..cedd5e4f 100644 --- a/sahara/plugins/mapr/abstract/cluster_context.py +++ b/sahara/plugins/mapr/abstract/cluster_context.py @@ -24,6 +24,14 @@ class AbstractClusterContext(object): def mapr_home(self): return + @abc.abstractproperty + def configure_sh_path(self): + return + + @abc.abstractproperty + def configure_sh(self): + return + @abc.abstractproperty def hadoop_version(self): return diff --git a/sahara/plugins/mapr/base/base_cluster_context.py b/sahara/plugins/mapr/base/base_cluster_context.py index a38aff6c..53f773ba 100644 --- a/sahara/plugins/mapr/base/base_cluster_context.py +++ b/sahara/plugins/mapr/base/base_cluster_context.py @@ -59,6 +59,7 @@ class BaseClusterContext(cc.AbstractClusterContext): self._oozie_server = None self._oozie_http = None self._some_instance = None + self._configure_sh_path = None self._configure_sh = None self._mapr_db = None self._hadoop_home = None @@ -158,6 +159,12 @@ class BaseClusterContext(cc.AbstractClusterContext): self._mapr_db = '-noDB' if not mapr_db else '' return self._mapr_db + @property + def configure_sh_path(self): + if not self._configure_sh_path: + self._configure_sh_path = '%s/server/configure.sh' % self.mapr_home + return self._configure_sh_path + @property def configure_sh(self): if not self._configure_sh: @@ -167,7 +174,7 @@ class BaseClusterContext(cc.AbstractClusterContext): ' -Z %(zookeepers)s' ' -no-autostart -f %(m7)s') args = { - 'script_path': '/opt/mapr/server/configure.sh', + 'script_path': self.configure_sh_path, 'cluster_name': self.cluster.name, 'cldbs': self.get_cldb_nodes_ip(), 'zookeepers': self.get_zookeeper_nodes_ip(), diff --git a/sahara/plugins/mapr/services/drill/__init__.py b/sahara/plugins/mapr/services/drill/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sahara/plugins/mapr/services/drill/drill.py b/sahara/plugins/mapr/services/drill/drill.py new file mode 100644 index 00000000..b5c136e5 --- /dev/null +++ b/sahara/plugins/mapr/services/drill/drill.py @@ -0,0 +1,52 @@ +# Copyright (c) 2015, MapR Technologies +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import six + +import sahara.plugins.mapr.domain.node_process as np +import sahara.plugins.mapr.domain.service as s +import sahara.plugins.mapr.util.commands as cmd +import sahara.plugins.mapr.util.validation_utils as vu + + +DRILL = np.NodeProcess( + name='drill-bits', + ui_name='Drill', + package='mapr-drill', + open_ports=[] +) + + +@six.add_metaclass(s.Single) +class Drill(s.Service): + def __init__(self): + super(Drill, self).__init__() + self._name = 'drill' + self._ui_name = 'Drill' + self._version = '0.7' + self._node_processes = [DRILL] + self._ui_info = [('Drill', DRILL, 'http://%s:8047')] + self._validation_rules = [vu.at_least(1, DRILL)] + + def install(self, cluster_context, instances): + # Drill requires running cluster + pass + + def post_start(self, cluster_context, instances): + instances = instances or cluster_context.get_instances(DRILL) + super(Drill, self).install(cluster_context, instances) + for instance in instances: + cmd.chown(instance, 'mapr:mapr', self.service_dir(cluster_context)) + cmd.re_configure_sh(instance, cluster_context) diff --git a/sahara/plugins/mapr/util/commands.py b/sahara/plugins/mapr/util/commands.py index 01264858..c215b628 100644 --- a/sahara/plugins/mapr/util/commands.py +++ b/sahara/plugins/mapr/util/commands.py @@ -17,3 +17,9 @@ def chown(instance, owner, path, run_as_root=True): cmd = 'chown -R %(owner)s %(path)s' % {'owner': owner, 'path': path} with instance.remote() as r: r.execute_command(cmd, run_as_root=run_as_root) + + +def re_configure_sh(instance, cluster_context): + with instance.remote() as r: + command = '%s -R' % cluster_context.configure_sh_path + r.execute_command(command, run_as_root=True) diff --git a/sahara/plugins/mapr/versions/v3_1_1/version_handler.py b/sahara/plugins/mapr/versions/v3_1_1/version_handler.py index 1c447b14..7079b308 100644 --- a/sahara/plugins/mapr/versions/v3_1_1/version_handler.py +++ b/sahara/plugins/mapr/versions/v3_1_1/version_handler.py @@ -14,6 +14,7 @@ from sahara.plugins.mapr.base import base_version_handler as bvh +from sahara.plugins.mapr.services.drill import drill from sahara.plugins.mapr.services.flume import flume from sahara.plugins.mapr.services.hbase import hbase from sahara.plugins.mapr.services.hive import hive @@ -56,6 +57,7 @@ class VersionHandler(bvh.BaseVersionHandler): pig.Pig(), swift.Swift(), flume.Flume(), + drill.Drill(), ] def get_context(self, cluster, added=None, removed=None): diff --git a/sahara/plugins/mapr/versions/v4_0_1_mrv1/version_handler.py b/sahara/plugins/mapr/versions/v4_0_1_mrv1/version_handler.py index 03794955..649d54ca 100644 --- a/sahara/plugins/mapr/versions/v4_0_1_mrv1/version_handler.py +++ b/sahara/plugins/mapr/versions/v4_0_1_mrv1/version_handler.py @@ -14,6 +14,7 @@ from sahara.plugins.mapr.base import base_version_handler as bvh +from sahara.plugins.mapr.services.drill import drill from sahara.plugins.mapr.services.flume import flume from sahara.plugins.mapr.services.hbase import hbase from sahara.plugins.mapr.services.hive import hive @@ -55,6 +56,7 @@ class VersionHandler(bvh.BaseVersionHandler): swift.Swift(), mapreduce.MapReduce(), flume.Flume(), + drill.Drill(), ] def get_context(self, cluster, added=None, removed=None): diff --git a/sahara/plugins/mapr/versions/v4_0_1_mrv2/version_handler.py b/sahara/plugins/mapr/versions/v4_0_1_mrv2/version_handler.py index 43881ba8..f4081c47 100644 --- a/sahara/plugins/mapr/versions/v4_0_1_mrv2/version_handler.py +++ b/sahara/plugins/mapr/versions/v4_0_1_mrv2/version_handler.py @@ -14,6 +14,7 @@ from sahara.plugins.mapr.base import base_version_handler as bvh +from sahara.plugins.mapr.services.drill import drill from sahara.plugins.mapr.services.flume import flume from sahara.plugins.mapr.services.hbase import hbase from sahara.plugins.mapr.services.hive import hive @@ -55,6 +56,7 @@ class VersionHandler(bvh.BaseVersionHandler): swift.Swift(), yarn.YARNv241(), flume.Flume(), + drill.Drill(), ] def get_context(self, cluster, added=None, removed=None):