Update vanilla plugin to the latest version

New vanilla supports following features:

 * Oozie of 4.2.0 version
 * YARN, MapReduce, HDFS of latest 2.7.1 version.

Also required templates for testing were added.

Implements blueprint: support-vanilla-2-7-1

Change-Id: Iae884d9f044ba1fe64b6d4ff23fa6ebabdbb41d9

MANIFEST.in

@@ -34,6 +34,7 @@ include sahara/plugins/vanilla/v1_2_1/resources/*.sh
 include sahara/plugins/vanilla/v1_2_1/resources/*.sql
 include sahara/plugins/vanilla/v1_2_1/resources/*.xml
 include sahara/plugins/vanilla/v2_6_0/resources/*.xml
+include sahara/plugins/vanilla/v2_7_1/resources/*.xml
 include sahara/plugins/hdp/versions/version_1_3_2/resources/*.template
 include sahara/plugins/hdp/versions/version_1_3_2/resources/*.json
 include sahara/plugins/hdp/versions/version_1_3_2/resources/*.sh

etc/scenario/sahara-ci/vanilla-2.7.1.yaml.mako

@@ -0,0 +1,71 @@
+clusters:
+  - plugin_name: vanilla
+    plugin_version: 2.7.1
+    image: ${vanilla_two_seven_one_image}
+    node_group_templates:
+      - name: worker-dn-nm
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - datanode
+          - nodemanager
+        volumes_per_node: 2
+        volumes_size: 2
+        auto_security_group: true
+      - name: worker-nm
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - nodemanager
+        auto_security_group: true
+      - name: worker-dn
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - datanode
+        volumes_per_node: 2
+        volumes_size: 2
+        auto_security_group: true
+      - name: master-rm-nn-hvs
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - namenode
+          - resourcemanager
+          - hiveserver
+          - nodemanager
+        auto_security_group: true
+      - name: master-oo-hs-sn
+        flavor: ${ci_flavor_id}
+        node_processes:
+          - oozie
+          - historyserver
+          - secondarynamenode
+          - nodemanager
+        auto_security_group: true
+    cluster_template:
+      name: vanilla271
+      node_group_templates:
+        master-rm-nn-hvs: 1
+        master-oo-hs-sn: 1
+        worker-dn-nm: 2
+        worker-dn: 1
+        worker-nm: 1
+      cluster_configs:
+        HDFS:
+          dfs.replication: 1
+    cluster:
+      name: ${cluster_name}
+    scaling:
+      - operation: resize
+        node_group: worker-dn-nm
+        size: 1
+      - operation: resize
+        node_group: worker-dn
+        size: 0
+      - operation: resize
+        node_group: worker-nm
+        size: 0
+      - operation: add
+        node_group: worker-dn
+        size: 1
+      - operation: add
+        node_group: worker-nm
+        size: 2
+    edp_jobs_flow: hadoop_2

sahara/plugins/vanilla/v2_7_1/config_helper.py

@@ -0,0 +1,98 @@
+# Copyright (c) 2015 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+from oslo_log import log as logging
+
+from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper
+from sahara.utils import xmlutils as x
+
+CONF = cfg.CONF
+CONF.import_opt("enable_data_locality", "sahara.topology.topology_helper")
+
+LOG = logging.getLogger(__name__)
+
+CORE_DEFAULT = x.load_hadoop_xml_defaults(
+    'plugins/vanilla/v2_7_1/resources/core-default.xml')
+
+HDFS_DEFAULT = x.load_hadoop_xml_defaults(
+    'plugins/vanilla/v2_7_1/resources/hdfs-default.xml')
+
+MAPRED_DEFAULT = x.load_hadoop_xml_defaults(
+    'plugins/vanilla/v2_7_1/resources/mapred-default.xml')
+
+YARN_DEFAULT = x.load_hadoop_xml_defaults(
+    'plugins/vanilla/v2_7_1/resources/yarn-default.xml')
+
+OOZIE_DEFAULT = x.load_hadoop_xml_defaults(
+    'plugins/vanilla/v2_7_1/resources/oozie-default.xml')
+
+HIVE_DEFAULT = x.load_hadoop_xml_defaults(
+    'plugins/vanilla/v2_7_1/resources/hive-default.xml')
+
+XML_CONFS = {
+    "Hadoop": [CORE_DEFAULT],
+    "HDFS": [HDFS_DEFAULT],
+    "YARN": [YARN_DEFAULT],
+    "MapReduce": [MAPRED_DEFAULT],
+    "JobFlow": [OOZIE_DEFAULT],
+    "Hive": [HIVE_DEFAULT]
+}
+
+ENV_CONFS = {
+    "YARN": {
+        'ResourceManager Heap Size': 1024,
+        'NodeManager Heap Size': 1024
+    },
+    "HDFS": {
+        'NameNode Heap Size': 1024,
+        'SecondaryNameNode Heap Size': 1024,
+        'DataNode Heap Size': 1024
+    },
+    "MapReduce": {
+        'JobHistoryServer Heap Size': 1024
+    },
+    "JobFlow": {
+        'Oozie Heap Size': 1024
+    }
+}
+
+
+# Initialise plugin Hadoop configurations
+PLUGIN_XML_CONFIGS = c_helper.init_xml_configs(XML_CONFS)
+PLUGIN_ENV_CONFIGS = c_helper.init_env_configs(ENV_CONFS)
+
+
+def _init_all_configs():
+    configs = []
+    configs.extend(PLUGIN_XML_CONFIGS)
+    configs.extend(PLUGIN_ENV_CONFIGS)
+    configs.extend(c_helper.PLUGIN_GENERAL_CONFIGS)
+    return configs
+
+
+PLUGIN_CONFIGS = _init_all_configs()
+
+
+def get_plugin_configs():
+    return PLUGIN_CONFIGS
+
+
+def get_xml_configs():
+    return PLUGIN_XML_CONFIGS
+
+
+def get_env_configs():
+    return ENV_CONFS

sahara/plugins/vanilla/v2_7_1/edp_engine.py

@@ -0,0 +1,35 @@
+# Copyright (c) 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from sahara.plugins.vanilla import confighints_helper as ch_helper
+from sahara.plugins.vanilla.hadoop2 import edp_engine
+from sahara.utils import edp
+
+
+class EdpOozieEngine(edp_engine.EdpOozieEngine):
+    @staticmethod
+    def get_possible_job_config(job_type):
+        if edp.compare_job_type(job_type, edp.JOB_TYPE_HIVE):
+            return {'job_config': ch_helper.get_possible_hive_config_from(
+                    'plugins/vanilla/v2_7_1/resources/hive-default.xml')}
+        if edp.compare_job_type(job_type,
+                                edp.JOB_TYPE_MAPREDUCE,
+                                edp.JOB_TYPE_MAPREDUCE_STREAMING):
+            return {'job_config': ch_helper.get_possible_mapreduce_config_from(
+                    'plugins/vanilla/v2_7_1/resources/mapred-default.xml')}
+        if edp.compare_job_type(job_type, edp.JOB_TYPE_PIG):
+            return {'job_config': ch_helper.get_possible_pig_config_from(
+                    'plugins/vanilla/v2_7_1/resources/mapred-default.xml')}
+        return edp_engine.EdpOozieEngine.get_possible_job_config(job_type)

sahara/plugins/vanilla/v2_7_1/resources/README.rst

@@ -0,0 +1,28 @@
+Apache Hadoop Configurations for Sahara
+=======================================
+
+This directory contains default XML configuration files:
+
+* core-default.xml
+* hdfs-default.xml
+* mapred-default.xml
+* yarn-default.xml
+* oozie-default.xml
+* hive-default.xml
+
+These files are applied for Sahara's plugin of Apache Hadoop version 2.7.1
+
+
+Files were taken from here:
+
+* `core-default.xml <https://github.com/apache/hadoop/blob/release-2.7.1/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml>`_
+* `hdfs-default.xml <https://github.com/apache/hadoop/blob/release-2.7.1/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml>`_
+* `yarn-default.xml <https://github.com/apache/hadoop/blob/release-2.7.1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml>`_
+* `mapred-default.xml <https://github.com/apache/hadoop/blob/release-2.7.1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml>`_
+* `oozie-default.xml <https://github.com/apache/oozie/blob/release-4.2.0/core/src/main/resources/oozie-default.xml>`_
+* `hive-default.xml <https://github.com/apache/hive/blob/release-0.11.0/conf/hive-default.xml.template>`_
+
+XML configs are used to expose default Hadoop configurations to the users
+through Sahara's REST API. It allows users to override some config values which
+will be pushed to the provisioned VMs running Hadoop services as part of
+appropriate xml config.

sahara/plugins/vanilla/v2_7_1/versionhandler.py

@@ -0,0 +1,140 @@
+# Copyright (c) 2015 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from oslo_config import cfg
+from oslo_log import log as logging
+
+from sahara import conductor
+from sahara import context
+from sahara.plugins import utils
+from sahara.plugins.vanilla import abstractversionhandler as avm
+from sahara.plugins.vanilla.hadoop2 import config as c
+from sahara.plugins.vanilla.hadoop2 import recommendations_utils as ru
+from sahara.plugins.vanilla.hadoop2 import run_scripts as run
+from sahara.plugins.vanilla.hadoop2 import scaling as sc
+from sahara.plugins.vanilla.hadoop2 import starting_scripts as s_scripts
+from sahara.plugins.vanilla.hadoop2 import validation as vl
+from sahara.plugins.vanilla import utils as vu
+from sahara.plugins.vanilla.v2_7_1 import config_helper as c_helper
+from sahara.plugins.vanilla.v2_7_1 import edp_engine
+
+
+conductor = conductor.API
+LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
+
+
+class VersionHandler(avm.AbstractVersionHandler):
+    def __init__(self):
+        self.pctx = {
+            'env_confs': c_helper.get_env_configs(),
+            'all_confs': c_helper.get_plugin_configs()
+        }
+
+    def get_plugin_configs(self):
+        return self.pctx['all_confs']
+
+    def get_node_processes(self):
+        return {
+            "Hadoop": [],
+            "MapReduce": ["historyserver"],
+            "HDFS": ["namenode", "datanode", "secondarynamenode"],
+            "YARN": ["resourcemanager", "nodemanager"],
+            "JobFlow": ["oozie"],
+            "Hive": ["hiveserver"]
+        }
+
+    def validate(self, cluster):
+        vl.validate_cluster_creating(self.pctx, cluster)
+
+    def update_infra(self, cluster):
+        pass
+
+    def configure_cluster(self, cluster):
+        c.configure_cluster(self.pctx, cluster)
+
+    def start_cluster(self, cluster):
+        s_scripts.start_namenode(cluster)
+        s_scripts.start_secondarynamenode(cluster)
+        s_scripts.start_resourcemanager(cluster)
+
+        run.start_dn_nm_processes(utils.get_instances(cluster))
+        run.await_datanodes(cluster)
+
+        s_scripts.start_historyserver(cluster)
+        s_scripts.start_oozie(self.pctx, cluster)
+        s_scripts.start_hiveserver(self.pctx, cluster)
+
+        self._set_cluster_info(cluster)
+
+    def decommission_nodes(self, cluster, instances):
+        sc.decommission_nodes(self.pctx, cluster, instances)
+
+    def validate_scaling(self, cluster, existing, additional):
+        vl.validate_additional_ng_scaling(cluster, additional)
+        vl.validate_existing_ng_scaling(self.pctx, cluster, existing)
+
+    def scale_cluster(self, cluster, instances):
+        sc.scale_cluster(self.pctx, cluster, instances)
+
+    def _set_cluster_info(self, cluster):
+        nn = vu.get_namenode(cluster)
+        rm = vu.get_resourcemanager(cluster)
+        hs = vu.get_historyserver(cluster)
+        oo = vu.get_oozie(cluster)
+
+        info = {}
+
+        if rm:
+            info['YARN'] = {
+                'Web UI': 'http://%s:%s' % (rm.management_ip, '8088'),
+                'ResourceManager': 'http://%s:%s' % (rm.management_ip, '8032')
+            }
+
+        if nn:
+            info['HDFS'] = {
+                'Web UI': 'http://%s:%s' % (nn.management_ip, '50070'),
+                'NameNode': 'hdfs://%s:%s' % (nn.hostname(), '9000')
+            }
+
+        if oo:
+            info['JobFlow'] = {
+                'Oozie': 'http://%s:%s' % (oo.management_ip, '11000')
+            }
+
+        if hs:
+            info['MapReduce JobHistory Server'] = {
+                'Web UI': 'http://%s:%s' % (hs.management_ip, '19888')
+            }
+
+        ctx = context.ctx()
+        conductor.cluster_update(ctx, cluster, {'info': info})
+
+    def get_edp_engine(self, cluster, job_type):
+        if job_type in edp_engine.EdpOozieEngine.get_supported_job_types():
+            return edp_engine.EdpOozieEngine(cluster)
+        return None
+
+    def get_edp_job_types(self):
+        return edp_engine.EdpOozieEngine.get_supported_job_types()
+
+    def get_edp_config_hints(self, job_type):
+        return edp_engine.EdpOozieEngine.get_possible_job_config(job_type)
+
+    def get_open_ports(self, node_group):
+        return c.get_open_ports(node_group)
+
+    def recommend_configs(self, cluster):
+        ru.recommend_configs(cluster, self.get_plugin_configs())