Merge "Added support for MapR v5.0.0"

etc/scenario/sahara-ci/mapr-5.0.0.mrv2.yaml.mako

@@ -0,0 +1,47 @@
+clusters:
+  - plugin_name: mapr
+    plugin_version: 5.0.0.mrv2
+    image: ${mapr_500mrv2_image}
+    node_group_templates:
+      - name: master
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - Metrics
+          - Webserver
+          - ZooKeeper
+          - HTTPFS
+          - Oozie
+          - FileServer
+          - CLDB
+          - Flume
+          - Hue
+          - NodeManager
+          - HistoryServer
+          - ResourceManager
+          - HiveServer2
+          - HiveMetastore
+          - Sqoop2-Client
+          - Sqoop2-Server
+        auto_security_group: true
+        volumes_per_node: 2
+        volumes_size: 20
+      - name: worker
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - NodeManager
+          - FileServer
+        auto_security_group: true
+        volumes_per_node: 2
+        volumes_size: 20
+    cluster_template:
+      name: mapr500mrv2
+      node_group_templates:
+        master: 1
+        worker: 3
+    cluster:
+      name: ${cluster_name}
+    scaling:
+      - operation: add
+        node_group: worker
+        size: 1
+    edp_jobs_flow: mapr

sahara/plugins/mapr/versions/v5_0_0_mrv1/context.py

@@ -0,0 +1,49 @@
+# Copyright (c) 2015, MapR Technologies
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import sahara.plugins.mapr.base.base_cluster_context as bc
+import sahara.plugins.mapr.services.mapreduce.mapreduce as mr
+
+
+class Context(bc.BaseClusterContext):
+    def __init__(self, cluster, version_handler, added=None, removed=None):
+        super(Context, self).__init__(cluster, version_handler, added, removed)
+        self._hadoop_version = mr.MapReduce().version
+        self._hadoop_lib = None
+        self._hadoop_conf = None
+        self._resource_manager_uri = "maprfs:///"
+        self._cluster_mode = mr.MapReduce.cluster_mode
+        self._node_aware = True
+        self._mapr_version = '5.0.0'
+        self._ubuntu_ecosystem_repo = (
+            "http://package.mapr.com/releases/ecosystem-5.x/ubuntu binary/")
+        self._centos_ecosystem_repo = (
+            "http://package.mapr.com/releases/ecosystem-5.x/redhat")
+
+    @property
+    def hadoop_lib(self):
+        if not self._hadoop_lib:
+            self._hadoop_lib = "%s/lib" % self.hadoop_home
+        return self._hadoop_lib
+
+    @property
+    def hadoop_conf(self):
+        if not self._hadoop_conf:
+            self._hadoop_conf = "%s/conf" % self.hadoop_home
+        return self._hadoop_conf
+
+    @property
+    def resource_manager_uri(self):
+        return self._resource_manager_uri

sahara/plugins/mapr/versions/v5_0_0_mrv1/version_handler.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2015, MapR Technologies
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+from sahara.plugins.mapr.base import base_version_handler as bvh
+from sahara.plugins.mapr.services.drill import drill
+from sahara.plugins.mapr.services.flume import flume
+from sahara.plugins.mapr.services.hbase import hbase
+from sahara.plugins.mapr.services.hive import hive
+from sahara.plugins.mapr.services.httpfs import httpfs
+from sahara.plugins.mapr.services.hue import hue
+from sahara.plugins.mapr.services.impala import impala
+from sahara.plugins.mapr.services.mahout import mahout
+from sahara.plugins.mapr.services.management import management as mng
+from sahara.plugins.mapr.services.mapreduce import mapreduce
+from sahara.plugins.mapr.services.maprfs import maprfs
+from sahara.plugins.mapr.services.oozie import oozie
+from sahara.plugins.mapr.services.pig import pig
+from sahara.plugins.mapr.services.sqoop import sqoop2
+from sahara.plugins.mapr.services.swift import swift
+import sahara.plugins.mapr.versions.v4_0_2_mrv1.context as c
+
+
+version = "5.0.0.mrv1"
+
+
+class VersionHandler(bvh.BaseVersionHandler):
+    def __init__(self):
+        super(VersionHandler, self).__init__()
+        self._version = version
+        self._required_services = [
+            mapreduce.MapReduce(),
+            maprfs.MapRFS(),
+            mng.Management(),
+            oozie.Oozie(),
+        ]
+        self._services = [
+            hive.HiveV013(),
+            hive.HiveV10(),
+            impala.ImpalaV141(),
+            pig.PigV014(),
+            flume.Flume(),
+            sqoop2.Sqoop2(),
+            mahout.MahoutV010(),
+            oozie.OozieV410(),
+            hue.HueV370(),
+            hbase.HBaseV0989(),
+            hbase.HBaseV09812(),
+            drill.DrillV11(),
+            mapreduce.MapReduce(),
+            maprfs.MapRFS(),
+            mng.Management(),
+            httpfs.HttpFS(),
+            swift.Swift(),
+        ]
+
+    def get_context(self, cluster, added=None, removed=None):
+        return c.Context(cluster, self, added, removed)

sahara/plugins/mapr/versions/v5_0_0_mrv2/context.py

@@ -0,0 +1,65 @@
+# Copyright (c) 2015, MapR Technologies
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import sahara.plugins.mapr.base.base_cluster_context as bc
+import sahara.plugins.mapr.services.yarn.yarn as yarn
+
+
+class Context(bc.BaseClusterContext):
+    def __init__(self, cluster, version_handler, added=None, removed=None):
+        super(Context, self).__init__(cluster, version_handler, added, removed)
+        self._hadoop_version = yarn.YARNv270().version
+        self._hadoop_lib = None
+        self._hadoop_conf = None
+        self._cluster_mode = yarn.YARNv270.cluster_mode
+        self._node_aware = True
+        self._resource_manager_uri = None
+        self._mapr_version = "5.0.0"
+        self._ubuntu_ecosystem_repo = (
+            "http://package.mapr.com/releases/ecosystem-5.x/ubuntu binary/")
+        self._centos_ecosystem_repo = (
+            "http://package.mapr.com/releases/ecosystem-5.x/redhat")
+
+    @property
+    def hadoop_lib(self):
+        if not self._hadoop_lib:
+            self._hadoop_lib = "%s/share/hadoop/common" % self.hadoop_home
+        return self._hadoop_lib
+
+    @property
+    def hadoop_conf(self):
+        if not self._hadoop_conf:
+            self._hadoop_conf = "%s/etc/hadoop" % self.hadoop_home
+        return self._hadoop_conf
+
+    @property
+    def resource_manager_uri(self):
+        # FIXME(aosadchyi): Wait for RM HA to work properly
+        if not self._resource_manager_uri:
+            ip = self.get_instance(yarn.RESOURCE_MANAGER).internal_ip
+            self._resource_manager_uri = "%s:8032" % ip
+        return self._resource_manager_uri
+
+    @property
+    def configure_sh(self):
+        if not self._configure_sh:
+            f = "%(base)s -RM %(resource_manager)s -HS %(history_server)s"
+            args = {
+                "base": super(Context, self).configure_sh,
+                "resource_manager": self.get_resourcemanager_ip(),
+                "history_server": self.get_historyserver_ip(),
+            }
+            self._configure_sh = f % args
+        return self._configure_sh

sahara/plugins/mapr/versions/v5_0_0_mrv2/version_handler.py

@@ -0,0 +1,70 @@
+# Copyright (c) 2015, MapR Technologies
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import sahara.plugins.mapr.base.base_version_handler as bvh
+from sahara.plugins.mapr.services.drill import drill
+from sahara.plugins.mapr.services.flume import flume
+from sahara.plugins.mapr.services.hbase import hbase
+from sahara.plugins.mapr.services.hive import hive
+from sahara.plugins.mapr.services.httpfs import httpfs
+from sahara.plugins.mapr.services.hue import hue
+from sahara.plugins.mapr.services.impala import impala
+from sahara.plugins.mapr.services.mahout import mahout
+from sahara.plugins.mapr.services.management import management as mng
+from sahara.plugins.mapr.services.maprfs import maprfs
+from sahara.plugins.mapr.services.oozie import oozie
+from sahara.plugins.mapr.services.pig import pig
+from sahara.plugins.mapr.services.sqoop import sqoop2
+from sahara.plugins.mapr.services.swift import swift
+from sahara.plugins.mapr.services.yarn import yarn
+import sahara.plugins.mapr.versions.v4_0_2_mrv2.context as c
+
+
+version = "5.0.0.mrv2"
+
+
+class VersionHandler(bvh.BaseVersionHandler):
+    def __init__(self):
+        super(VersionHandler, self).__init__()
+        self._version = version
+        self._required_services = [
+            yarn.YARNv270(),
+            maprfs.MapRFS(),
+            mng.Management(),
+            oozie.Oozie(),
+        ]
+        self._services = [
+            hive.HiveV013(),
+            hive.HiveV10(),
+            impala.ImpalaV141(),
+            pig.PigV014(),
+            flume.Flume(),
+            sqoop2.Sqoop2(),
+            mahout.MahoutV010(),
+            oozie.OozieV410(),
+            hue.HueV370(),
+            hbase.HBaseV0989(),
+            hbase.HBaseV09812(),
+            drill.DrillV11(),
+            yarn.YARNv270(),
+            maprfs.MapRFS(),
+            mng.Management(),
+            impala.ImpalaV141(),
+            httpfs.HttpFS(),
+            swift.Swift(),
+        ]
+
+    def get_context(self, cluster, added=None, removed=None):
+        return c.Context(cluster, self, added, removed)