diff --git a/doc/source/userdoc/hadoop-swift.rst b/doc/source/userdoc/hadoop-swift.rst index 3ca9238e..b27aece3 100644 --- a/doc/source/userdoc/hadoop-swift.rst +++ b/doc/source/userdoc/hadoop-swift.rst @@ -17,6 +17,11 @@ the most current features enabled. * The latest compiled version of the jar for this component can be downloaded from http://sahara-files.mirantis.com/hadoop-swift/hadoop-swift-latest.jar +Now the latest version of this jar (which uses Keystone API v3) is used in +the plugins' images automatically during build of these images. But for +Ambari plugin we need to explicitly put this jar into /opt directory of the +base image **before** cluster launching. + Hadoop patching --------------- You may build the jar file yourself by choosing the latest patch from the @@ -25,9 +30,9 @@ provided. Or you may get the latest jar pre-built from the CDN at http://sahara-files.mirantis.com/hadoop-swift/hadoop-swift-latest.jar You will need to put this file into the hadoop libraries -(e.g. /usr/lib/share/hadoop/lib) on each job-tracker and task-tracker node -for Hadoop 1.x, or each ResourceManager and NodeManager node for Hadoop 2.x -in the cluster. +(e.g. /usr/lib/share/hadoop/lib, it depends on the plugin which you use) on +each job-tracker and task-tracker node for Hadoop 1.x, or each ResourceManager +and NodeManager node for Hadoop 2.x in the cluster. Hadoop configurations --------------------- diff --git a/releasenotes/notes/hadoop-swift-jar-for-ambari-4439913b01d42468.yaml b/releasenotes/notes/hadoop-swift-jar-for-ambari-4439913b01d42468.yaml new file mode 100644 index 00000000..060f5a78 --- /dev/null +++ b/releasenotes/notes/hadoop-swift-jar-for-ambari-4439913b01d42468.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - This patch adds ability to work with swift by using Keystone + API v3 diff --git a/sahara/plugins/ambari/deploy.py b/sahara/plugins/ambari/deploy.py index 2c1e2c6d..fa1a2eb3 100644 --- a/sahara/plugins/ambari/deploy.py +++ b/sahara/plugins/ambari/deploy.py @@ -22,6 +22,7 @@ from oslo_utils import uuidutils from sahara import conductor from sahara import context +from sahara.i18n import _LW from sahara.plugins.ambari import client as ambari_client from sahara.plugins.ambari import common as p_common from sahara.plugins.ambari import configs @@ -388,3 +389,28 @@ def _wait_all_processes_removed(cluster, instance): if not hdp_processes: return context.sleep(5) + + +def add_hadoop_swift_jar(instances): + new_jar = "/opt/hadoop-openstack.jar" + for inst in instances: + with inst.remote() as r: + code, out = r.execute_command("test -f %s" % new_jar, + raise_when_error=False) + if code == 0: + # get ambari hadoop version (e.g.: 2.7.1.2.3.4.0-3485) + code, amb_hadoop_version = r.execute_command( + "sudo hadoop version | grep 'Hadoop' | awk '{print $2}'") + amb_hadoop_version = amb_hadoop_version.strip() + # get special code of ambari hadoop version(e.g.:2.3.4.0-3485) + amb_code = '.'.join(amb_hadoop_version.split('.')[3:]) + origin_jar = ( + "/usr/hdp/%s/hadoop-mapreduce/hadoop-openstack-%s.jar" % ( + amb_code, amb_hadoop_version)) + r.execute_command("sudo cp %s %s" % (new_jar, origin_jar)) + else: + LOG.warning(_LW("The {jar_file} file cannot be found " + "in the {dir} directory so Keystone API v3 " + "is not enabled for this cluster.") + .format(jar_file="hadoop-openstack.jar", + dir="/opt")) diff --git a/sahara/plugins/ambari/plugin.py b/sahara/plugins/ambari/plugin.py index 5477fb75..5ce42197 100644 --- a/sahara/plugins/ambari/plugin.py +++ b/sahara/plugins/ambari/plugin.py @@ -89,7 +89,9 @@ class AmbariPluginProvider(p.ProvisioningPluginBase): def start_cluster(self, cluster): self._set_cluster_info(cluster) deploy.start_cluster(cluster) - swift_helper.install_ssl_certs(plugin_utils.get_instances(cluster)) + cluster_instances = plugin_utils.get_instances(cluster) + swift_helper.install_ssl_certs(cluster_instances) + deploy.add_hadoop_swift_jar(cluster_instances) def _set_cluster_info(self, cluster): ambari_ip = plugin_utils.get_instance( @@ -180,6 +182,7 @@ class AmbariPluginProvider(p.ProvisioningPluginBase): deploy.manage_config_groups(cluster, instances) deploy.manage_host_components(cluster, instances) swift_helper.install_ssl_certs(instances) + deploy.add_hadoop_swift_jar(instances) def decommission_nodes(self, cluster, instances): deploy.decommission_hosts(cluster, instances) diff --git a/sahara/plugins/spark/config_helper.py b/sahara/plugins/spark/config_helper.py index 5ddc507f..6ff2647a 100644 --- a/sahara/plugins/spark/config_helper.py +++ b/sahara/plugins/spark/config_helper.py @@ -44,7 +44,7 @@ XML_CONFS = { } _default_executor_classpath = ":".join( - ['/usr/lib/hadoop/hadoop-swift.jar']) + ['/usr/lib/hadoop-mapreduce/hadoop-openstack.jar']) SPARK_CONFS = { 'Spark': { diff --git a/sahara/tests/unit/service/edp/spark/base.py b/sahara/tests/unit/service/edp/spark/base.py index 503bbd93..a75f3e9a 100644 --- a/sahara/tests/unit/service/edp/spark/base.py +++ b/sahara/tests/unit/service/edp/spark/base.py @@ -42,7 +42,7 @@ class TestSpark(base.SaharaTestCase): self.spark_pid = "12345" self.spark_home = "/opt/spark" self.workflow_dir = "/wfdir" - self.driver_cp = "/usr/lib/hadoop/hadoop-swift.jar:" + self.driver_cp = "/usr/lib/hadoop-mapreduce/hadoop-openstack.jar:" def test_get_pid_and_inst_id(self): '''Test parsing of job ids