From b5bb9f97dc55b3606aabf0e772d1b4e057342226 Mon Sep 17 00:00:00 2001 From: Michael Ionkin Date: Wed, 16 Mar 2016 19:03:08 +0300 Subject: [PATCH] Add hadoop openstack swift jar to ambari cluster This patch adds our custom hadoop swiftfs implementation to Ambari cluster instances after their start in order to allow usage of swift with Keystone API v3. p.s. required jar file should be previously saved in the /opt folder of the base images. Closes-bug: 1558064 Change-Id: Ie6df4a542a16b4417b505b4a621e8b4b921364d3 --- doc/source/userdoc/hadoop-swift.rst | 11 +++++--- ...swift-jar-for-ambari-4439913b01d42468.yaml | 4 +++ sahara/plugins/ambari/deploy.py | 26 +++++++++++++++++++ sahara/plugins/ambari/plugin.py | 5 +++- sahara/plugins/spark/config_helper.py | 2 +- sahara/tests/unit/service/edp/spark/base.py | 2 +- 6 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 releasenotes/notes/hadoop-swift-jar-for-ambari-4439913b01d42468.yaml diff --git a/doc/source/userdoc/hadoop-swift.rst b/doc/source/userdoc/hadoop-swift.rst index 3ca9238e..b27aece3 100644 --- a/doc/source/userdoc/hadoop-swift.rst +++ b/doc/source/userdoc/hadoop-swift.rst @@ -17,6 +17,11 @@ the most current features enabled. * The latest compiled version of the jar for this component can be downloaded from http://sahara-files.mirantis.com/hadoop-swift/hadoop-swift-latest.jar +Now the latest version of this jar (which uses Keystone API v3) is used in +the plugins' images automatically during build of these images. But for +Ambari plugin we need to explicitly put this jar into /opt directory of the +base image **before** cluster launching. + Hadoop patching --------------- You may build the jar file yourself by choosing the latest patch from the @@ -25,9 +30,9 @@ provided. Or you may get the latest jar pre-built from the CDN at http://sahara-files.mirantis.com/hadoop-swift/hadoop-swift-latest.jar You will need to put this file into the hadoop libraries -(e.g. /usr/lib/share/hadoop/lib) on each job-tracker and task-tracker node -for Hadoop 1.x, or each ResourceManager and NodeManager node for Hadoop 2.x -in the cluster. +(e.g. /usr/lib/share/hadoop/lib, it depends on the plugin which you use) on +each job-tracker and task-tracker node for Hadoop 1.x, or each ResourceManager +and NodeManager node for Hadoop 2.x in the cluster. Hadoop configurations --------------------- diff --git a/releasenotes/notes/hadoop-swift-jar-for-ambari-4439913b01d42468.yaml b/releasenotes/notes/hadoop-swift-jar-for-ambari-4439913b01d42468.yaml new file mode 100644 index 00000000..060f5a78 --- /dev/null +++ b/releasenotes/notes/hadoop-swift-jar-for-ambari-4439913b01d42468.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - This patch adds ability to work with swift by using Keystone + API v3 diff --git a/sahara/plugins/ambari/deploy.py b/sahara/plugins/ambari/deploy.py index 2c1e2c6d..fa1a2eb3 100644 --- a/sahara/plugins/ambari/deploy.py +++ b/sahara/plugins/ambari/deploy.py @@ -22,6 +22,7 @@ from oslo_utils import uuidutils from sahara import conductor from sahara import context +from sahara.i18n import _LW from sahara.plugins.ambari import client as ambari_client from sahara.plugins.ambari import common as p_common from sahara.plugins.ambari import configs @@ -388,3 +389,28 @@ def _wait_all_processes_removed(cluster, instance): if not hdp_processes: return context.sleep(5) + + +def add_hadoop_swift_jar(instances): + new_jar = "/opt/hadoop-openstack.jar" + for inst in instances: + with inst.remote() as r: + code, out = r.execute_command("test -f %s" % new_jar, + raise_when_error=False) + if code == 0: + # get ambari hadoop version (e.g.: 2.7.1.2.3.4.0-3485) + code, amb_hadoop_version = r.execute_command( + "sudo hadoop version | grep 'Hadoop' | awk '{print $2}'") + amb_hadoop_version = amb_hadoop_version.strip() + # get special code of ambari hadoop version(e.g.:2.3.4.0-3485) + amb_code = '.'.join(amb_hadoop_version.split('.')[3:]) + origin_jar = ( + "/usr/hdp/%s/hadoop-mapreduce/hadoop-openstack-%s.jar" % ( + amb_code, amb_hadoop_version)) + r.execute_command("sudo cp %s %s" % (new_jar, origin_jar)) + else: + LOG.warning(_LW("The {jar_file} file cannot be found " + "in the {dir} directory so Keystone API v3 " + "is not enabled for this cluster.") + .format(jar_file="hadoop-openstack.jar", + dir="/opt")) diff --git a/sahara/plugins/ambari/plugin.py b/sahara/plugins/ambari/plugin.py index 5477fb75..5ce42197 100644 --- a/sahara/plugins/ambari/plugin.py +++ b/sahara/plugins/ambari/plugin.py @@ -89,7 +89,9 @@ class AmbariPluginProvider(p.ProvisioningPluginBase): def start_cluster(self, cluster): self._set_cluster_info(cluster) deploy.start_cluster(cluster) - swift_helper.install_ssl_certs(plugin_utils.get_instances(cluster)) + cluster_instances = plugin_utils.get_instances(cluster) + swift_helper.install_ssl_certs(cluster_instances) + deploy.add_hadoop_swift_jar(cluster_instances) def _set_cluster_info(self, cluster): ambari_ip = plugin_utils.get_instance( @@ -180,6 +182,7 @@ class AmbariPluginProvider(p.ProvisioningPluginBase): deploy.manage_config_groups(cluster, instances) deploy.manage_host_components(cluster, instances) swift_helper.install_ssl_certs(instances) + deploy.add_hadoop_swift_jar(instances) def decommission_nodes(self, cluster, instances): deploy.decommission_hosts(cluster, instances) diff --git a/sahara/plugins/spark/config_helper.py b/sahara/plugins/spark/config_helper.py index 5ddc507f..6ff2647a 100644 --- a/sahara/plugins/spark/config_helper.py +++ b/sahara/plugins/spark/config_helper.py @@ -44,7 +44,7 @@ XML_CONFS = { } _default_executor_classpath = ":".join( - ['/usr/lib/hadoop/hadoop-swift.jar']) + ['/usr/lib/hadoop-mapreduce/hadoop-openstack.jar']) SPARK_CONFS = { 'Spark': { diff --git a/sahara/tests/unit/service/edp/spark/base.py b/sahara/tests/unit/service/edp/spark/base.py index 503bbd93..a75f3e9a 100644 --- a/sahara/tests/unit/service/edp/spark/base.py +++ b/sahara/tests/unit/service/edp/spark/base.py @@ -42,7 +42,7 @@ class TestSpark(base.SaharaTestCase): self.spark_pid = "12345" self.spark_home = "/opt/spark" self.workflow_dir = "/wfdir" - self.driver_cp = "/usr/lib/hadoop/hadoop-swift.jar:" + self.driver_cp = "/usr/lib/hadoop-mapreduce/hadoop-openstack.jar:" def test_get_pid_and_inst_id(self): '''Test parsing of job ids