From 5013fe747dcba0f204f0a5ec55997f91465c0c7b Mon Sep 17 00:00:00 2001 From: Artem Osadchyi Date: Mon, 3 Aug 2015 15:01:52 +0300 Subject: [PATCH] Added support for MapR v5.0.0 MapR plugin supports 5.0.0 version Change-Id: I520649ca1fae61abe09155320e58e4655e1940da Implements: blueprint mapr-500 --- .../sahara-ci/mapr-5.0.0.mrv2.yaml.mako | 47 +++++++++++++ .../mapr/versions/v5_0_0_mrv1/__init__.py | 0 .../mapr/versions/v5_0_0_mrv1/context.py | 49 +++++++++++++ .../versions/v5_0_0_mrv1/version_handler.py | 69 ++++++++++++++++++ .../mapr/versions/v5_0_0_mrv2/__init__.py | 0 .../mapr/versions/v5_0_0_mrv2/context.py | 65 +++++++++++++++++ .../versions/v5_0_0_mrv2/version_handler.py | 70 +++++++++++++++++++ 7 files changed, 300 insertions(+) create mode 100644 etc/scenario/sahara-ci/mapr-5.0.0.mrv2.yaml.mako create mode 100644 sahara/plugins/mapr/versions/v5_0_0_mrv1/__init__.py create mode 100644 sahara/plugins/mapr/versions/v5_0_0_mrv1/context.py create mode 100644 sahara/plugins/mapr/versions/v5_0_0_mrv1/version_handler.py create mode 100644 sahara/plugins/mapr/versions/v5_0_0_mrv2/__init__.py create mode 100644 sahara/plugins/mapr/versions/v5_0_0_mrv2/context.py create mode 100644 sahara/plugins/mapr/versions/v5_0_0_mrv2/version_handler.py diff --git a/etc/scenario/sahara-ci/mapr-5.0.0.mrv2.yaml.mako b/etc/scenario/sahara-ci/mapr-5.0.0.mrv2.yaml.mako new file mode 100644 index 00000000..f2ddc639 --- /dev/null +++ b/etc/scenario/sahara-ci/mapr-5.0.0.mrv2.yaml.mako @@ -0,0 +1,47 @@ +clusters: + - plugin_name: mapr + plugin_version: 5.0.0.mrv2 + image: ${mapr_500mrv2_image} + node_group_templates: + - name: master + flavor: ${ci_flavor_id} + node_processes: + - Metrics + - Webserver + - ZooKeeper + - HTTPFS + - Oozie + - FileServer + - CLDB + - Flume + - Hue + - NodeManager + - HistoryServer + - ResourceManager + - HiveServer2 + - HiveMetastore + - Sqoop2-Client + - Sqoop2-Server + auto_security_group: true + volumes_per_node: 2 + volumes_size: 20 + - name: worker + flavor: ${ci_flavor_id} + node_processes: + - NodeManager + - FileServer + auto_security_group: true + volumes_per_node: 2 + volumes_size: 20 + cluster_template: + name: mapr500mrv2 + node_group_templates: + master: 1 + worker: 3 + cluster: + name: ${cluster_name} + scaling: + - operation: add + node_group: worker + size: 1 + edp_jobs_flow: mapr diff --git a/sahara/plugins/mapr/versions/v5_0_0_mrv1/__init__.py b/sahara/plugins/mapr/versions/v5_0_0_mrv1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sahara/plugins/mapr/versions/v5_0_0_mrv1/context.py b/sahara/plugins/mapr/versions/v5_0_0_mrv1/context.py new file mode 100644 index 00000000..428d2fc7 --- /dev/null +++ b/sahara/plugins/mapr/versions/v5_0_0_mrv1/context.py @@ -0,0 +1,49 @@ +# Copyright (c) 2015, MapR Technologies +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import sahara.plugins.mapr.base.base_cluster_context as bc +import sahara.plugins.mapr.services.mapreduce.mapreduce as mr + + +class Context(bc.BaseClusterContext): + def __init__(self, cluster, version_handler, added=None, removed=None): + super(Context, self).__init__(cluster, version_handler, added, removed) + self._hadoop_version = mr.MapReduce().version + self._hadoop_lib = None + self._hadoop_conf = None + self._resource_manager_uri = "maprfs:///" + self._cluster_mode = mr.MapReduce.cluster_mode + self._node_aware = True + self._mapr_version = '5.0.0' + self._ubuntu_ecosystem_repo = ( + "http://package.mapr.com/releases/ecosystem-5.x/ubuntu binary/") + self._centos_ecosystem_repo = ( + "http://package.mapr.com/releases/ecosystem-5.x/redhat") + + @property + def hadoop_lib(self): + if not self._hadoop_lib: + self._hadoop_lib = "%s/lib" % self.hadoop_home + return self._hadoop_lib + + @property + def hadoop_conf(self): + if not self._hadoop_conf: + self._hadoop_conf = "%s/conf" % self.hadoop_home + return self._hadoop_conf + + @property + def resource_manager_uri(self): + return self._resource_manager_uri diff --git a/sahara/plugins/mapr/versions/v5_0_0_mrv1/version_handler.py b/sahara/plugins/mapr/versions/v5_0_0_mrv1/version_handler.py new file mode 100644 index 00000000..664417ef --- /dev/null +++ b/sahara/plugins/mapr/versions/v5_0_0_mrv1/version_handler.py @@ -0,0 +1,69 @@ +# Copyright (c) 2015, MapR Technologies +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from sahara.plugins.mapr.base import base_version_handler as bvh +from sahara.plugins.mapr.services.drill import drill +from sahara.plugins.mapr.services.flume import flume +from sahara.plugins.mapr.services.hbase import hbase +from sahara.plugins.mapr.services.hive import hive +from sahara.plugins.mapr.services.httpfs import httpfs +from sahara.plugins.mapr.services.hue import hue +from sahara.plugins.mapr.services.impala import impala +from sahara.plugins.mapr.services.mahout import mahout +from sahara.plugins.mapr.services.management import management as mng +from sahara.plugins.mapr.services.mapreduce import mapreduce +from sahara.plugins.mapr.services.maprfs import maprfs +from sahara.plugins.mapr.services.oozie import oozie +from sahara.plugins.mapr.services.pig import pig +from sahara.plugins.mapr.services.sqoop import sqoop2 +from sahara.plugins.mapr.services.swift import swift +import sahara.plugins.mapr.versions.v4_0_2_mrv1.context as c + + +version = "5.0.0.mrv1" + + +class VersionHandler(bvh.BaseVersionHandler): + def __init__(self): + super(VersionHandler, self).__init__() + self._version = version + self._required_services = [ + mapreduce.MapReduce(), + maprfs.MapRFS(), + mng.Management(), + oozie.Oozie(), + ] + self._services = [ + hive.HiveV013(), + hive.HiveV10(), + impala.ImpalaV141(), + pig.PigV014(), + flume.Flume(), + sqoop2.Sqoop2(), + mahout.MahoutV010(), + oozie.OozieV410(), + hue.HueV370(), + hbase.HBaseV0989(), + hbase.HBaseV09812(), + drill.DrillV11(), + mapreduce.MapReduce(), + maprfs.MapRFS(), + mng.Management(), + httpfs.HttpFS(), + swift.Swift(), + ] + + def get_context(self, cluster, added=None, removed=None): + return c.Context(cluster, self, added, removed) diff --git a/sahara/plugins/mapr/versions/v5_0_0_mrv2/__init__.py b/sahara/plugins/mapr/versions/v5_0_0_mrv2/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sahara/plugins/mapr/versions/v5_0_0_mrv2/context.py b/sahara/plugins/mapr/versions/v5_0_0_mrv2/context.py new file mode 100644 index 00000000..ebda7eb9 --- /dev/null +++ b/sahara/plugins/mapr/versions/v5_0_0_mrv2/context.py @@ -0,0 +1,65 @@ +# Copyright (c) 2015, MapR Technologies +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import sahara.plugins.mapr.base.base_cluster_context as bc +import sahara.plugins.mapr.services.yarn.yarn as yarn + + +class Context(bc.BaseClusterContext): + def __init__(self, cluster, version_handler, added=None, removed=None): + super(Context, self).__init__(cluster, version_handler, added, removed) + self._hadoop_version = yarn.YARNv270().version + self._hadoop_lib = None + self._hadoop_conf = None + self._cluster_mode = yarn.YARNv270.cluster_mode + self._node_aware = True + self._resource_manager_uri = None + self._mapr_version = "5.0.0" + self._ubuntu_ecosystem_repo = ( + "http://package.mapr.com/releases/ecosystem-5.x/ubuntu binary/") + self._centos_ecosystem_repo = ( + "http://package.mapr.com/releases/ecosystem-5.x/redhat") + + @property + def hadoop_lib(self): + if not self._hadoop_lib: + self._hadoop_lib = "%s/share/hadoop/common" % self.hadoop_home + return self._hadoop_lib + + @property + def hadoop_conf(self): + if not self._hadoop_conf: + self._hadoop_conf = "%s/etc/hadoop" % self.hadoop_home + return self._hadoop_conf + + @property + def resource_manager_uri(self): + # FIXME(aosadchyi): Wait for RM HA to work properly + if not self._resource_manager_uri: + ip = self.get_instance(yarn.RESOURCE_MANAGER).internal_ip + self._resource_manager_uri = "%s:8032" % ip + return self._resource_manager_uri + + @property + def configure_sh(self): + if not self._configure_sh: + f = "%(base)s -RM %(resource_manager)s -HS %(history_server)s" + args = { + "base": super(Context, self).configure_sh, + "resource_manager": self.get_resourcemanager_ip(), + "history_server": self.get_historyserver_ip(), + } + self._configure_sh = f % args + return self._configure_sh diff --git a/sahara/plugins/mapr/versions/v5_0_0_mrv2/version_handler.py b/sahara/plugins/mapr/versions/v5_0_0_mrv2/version_handler.py new file mode 100644 index 00000000..e9d17cb6 --- /dev/null +++ b/sahara/plugins/mapr/versions/v5_0_0_mrv2/version_handler.py @@ -0,0 +1,70 @@ +# Copyright (c) 2015, MapR Technologies +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import sahara.plugins.mapr.base.base_version_handler as bvh +from sahara.plugins.mapr.services.drill import drill +from sahara.plugins.mapr.services.flume import flume +from sahara.plugins.mapr.services.hbase import hbase +from sahara.plugins.mapr.services.hive import hive +from sahara.plugins.mapr.services.httpfs import httpfs +from sahara.plugins.mapr.services.hue import hue +from sahara.plugins.mapr.services.impala import impala +from sahara.plugins.mapr.services.mahout import mahout +from sahara.plugins.mapr.services.management import management as mng +from sahara.plugins.mapr.services.maprfs import maprfs +from sahara.plugins.mapr.services.oozie import oozie +from sahara.plugins.mapr.services.pig import pig +from sahara.plugins.mapr.services.sqoop import sqoop2 +from sahara.plugins.mapr.services.swift import swift +from sahara.plugins.mapr.services.yarn import yarn +import sahara.plugins.mapr.versions.v4_0_2_mrv2.context as c + + +version = "5.0.0.mrv2" + + +class VersionHandler(bvh.BaseVersionHandler): + def __init__(self): + super(VersionHandler, self).__init__() + self._version = version + self._required_services = [ + yarn.YARNv270(), + maprfs.MapRFS(), + mng.Management(), + oozie.Oozie(), + ] + self._services = [ + hive.HiveV013(), + hive.HiveV10(), + impala.ImpalaV141(), + pig.PigV014(), + flume.Flume(), + sqoop2.Sqoop2(), + mahout.MahoutV010(), + oozie.OozieV410(), + hue.HueV370(), + hbase.HBaseV0989(), + hbase.HBaseV09812(), + drill.DrillV11(), + yarn.YARNv270(), + maprfs.MapRFS(), + mng.Management(), + impala.ImpalaV141(), + httpfs.HttpFS(), + swift.Swift(), + ] + + def get_context(self, cluster, added=None, removed=None): + return c.Context(cluster, self, added, removed)