Merge "Register SSL cert in Java keystore to access to swift via SSL"

sahara/config.py

@@ -131,6 +131,7 @@ def list_opts():
     from sahara.service.edp import job_utils
     from sahara.service.heat import heat_engine
     from sahara.service import periodic
+    from sahara.swift import swift_helper
     from sahara.utils import cluster_progress_ops as cpo
     from sahara.utils.openstack import base
     from sahara.utils.openstack import heat
@@ -177,7 +178,9 @@ def list_opts():
         (keystone.keystone_group.name,
          itertools.chain(keystone.ssl_opts)),
         (base.retries.name,
-         itertools.chain(base.opts))
+         itertools.chain(base.opts)),
+        (swift_helper.public_endpoint_cert_group.name,
+         itertools.chain(swift_helper.opts))
     ]
 
 

sahara/plugins/ambari/plugin.py

@@ -24,6 +24,7 @@ from sahara.plugins.ambari import edp_engine
 from sahara.plugins.ambari import validation
 from sahara.plugins import provisioning as p
 from sahara.plugins import utils as plugin_utils
+from sahara.swift import swift_helper
 
 
 conductor = conductor.API
@@ -84,6 +85,7 @@ class AmbariPluginProvider(p.ProvisioningPluginBase):
     def start_cluster(self, cluster):
         self._set_cluster_info(cluster)
         deploy.start_cluster(cluster)
+        swift_helper.install_ssl_certs(plugin_utils.get_instances(cluster))
 
     def _set_cluster_info(self, cluster):
         ambari_ip = plugin_utils.get_instance(

sahara/plugins/cdh/plugin_utils.py

@@ -27,6 +27,7 @@ from sahara.i18n import _
 from sahara.plugins.cdh import commands as cmd
 from sahara.plugins import recommendations_utils as ru
 from sahara.plugins import utils as u
+from sahara.swift import swift_helper
 from sahara.utils import cluster_progress_ops as cpo
 from sahara.utils import edp as edp_u
 from sahara.utils import poll_utils
@@ -254,6 +255,7 @@ class AbstractPluginUtils(object):
                 for i in instances:
                     tg.spawn('cdh-swift-conf-%s' % i.instance_name,
                              self._configure_swift_to_inst, i)
+            swift_helper.install_ssl_certs(instances)
 
     @cpo.event_wrapper(True)
     def _configure_swift_to_inst(self, instance):

sahara/plugins/spark/plugin.py

@@ -32,6 +32,7 @@ from sahara.plugins.spark import run_scripts as run
 from sahara.plugins.spark import scaling as sc
 from sahara.plugins.spark import shell_engine
 from sahara.plugins import utils
+from sahara.swift import swift_helper
 from sahara.topology import topology_helper as th
 from sahara.utils import cluster_progress_ops as cpo
 from sahara.utils import files as f
@@ -153,6 +154,7 @@ class SparkProvider(p.ProvisioningPluginBase):
 
         # start spark nodes
         self.start_spark(cluster)
+        swift_helper.install_ssl_certs(utils.get_instances(cluster))
 
         LOG.info(_LI('Cluster has been started successfully'))
         self._set_cluster_info(cluster)
@@ -447,6 +449,7 @@ class SparkProvider(p.ProvisioningPluginBase):
                         'datanode' in instance.node_group.node_processes]
         self._start_datanode_processes(dn_instances)
 
+        swift_helper.install_ssl_certs(instances)
         run.start_spark_master(r_master, self._spark_home(cluster))
         LOG.info(_LI("Spark master service has been restarted"))
 

sahara/plugins/vanilla/hadoop2/scaling.py

@@ -20,6 +20,7 @@ from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper
 from sahara.plugins.vanilla.hadoop2 import run_scripts as run
 from sahara.plugins.vanilla.hadoop2 import utils as pu
 from sahara.plugins.vanilla import utils as vu
+from sahara.swift import swift_helper
 from sahara.utils import cluster_progress_ops as cpo
 from sahara.utils import poll_utils
 
@@ -37,6 +38,7 @@ def scale_cluster(pctx, cluster, instances):
 
     config.configure_topology_data(pctx, cluster)
     run.start_dn_nm_processes(instances)
+    swift_helper.install_ssl_certs(instances)
 
 
 def _get_instances_with_service(instances, service):

sahara/plugins/vanilla/v2_6_0/versionhandler.py

@@ -31,6 +31,8 @@ from sahara.plugins.vanilla.hadoop2 import validation as vl
 from sahara.plugins.vanilla import utils as vu
 from sahara.plugins.vanilla.v2_6_0 import config_helper as c_helper
 from sahara.plugins.vanilla.v2_6_0 import edp_engine
+from sahara.swift import swift_helper
+from sahara.utils import cluster as cluster_utils
 
 
 conductor = conductor.API
@@ -82,6 +84,8 @@ class VersionHandler(avm.AbstractVersionHandler):
         s_scripts.start_oozie(self.pctx, cluster)
         s_scripts.start_hiveserver(self.pctx, cluster)
 
+        swift_helper.install_ssl_certs(cluster_utils.get_instances(cluster))
+
         self._set_cluster_info(cluster)
 
     def decommission_nodes(self, cluster, instances):

sahara/plugins/vanilla/v2_7_1/versionhandler.py

@@ -29,6 +29,8 @@ from sahara.plugins.vanilla.hadoop2 import validation as vl
 from sahara.plugins.vanilla import utils as vu
 from sahara.plugins.vanilla.v2_7_1 import config_helper as c_helper
 from sahara.plugins.vanilla.v2_7_1 import edp_engine
+from sahara.swift import swift_helper
+from sahara.utils import cluster as cluster_utils
 
 
 conductor = conductor.API
@@ -77,6 +79,8 @@ class VersionHandler(avm.AbstractVersionHandler):
         s_scripts.start_oozie(self.pctx, cluster)
         s_scripts.start_hiveserver(self.pctx, cluster)
 
+        swift_helper.install_ssl_certs(cluster_utils.get_instances(cluster))
+
         self._set_cluster_info(cluster)
 
     def decommission_nodes(self, cluster, instances):

sahara/swift/swift_helper.py

@@ -32,6 +32,21 @@ HADOOP_SWIFT_REGION = 'fs.swift.service.sahara.region'
 HADOOP_SWIFT_TRUST_ID = 'fs.swift.service.sahara.trust.id'
 HADOOP_SWIFT_DOMAIN_NAME = 'fs.swift.service.sahara.domain.name'
 
+opts = [
+    cfg.StrOpt("public_identity_ca_file",
+               help=("Location of ca certificate file to use for identity "
+                     "client requests via public endpoint")),
+    cfg.StrOpt("public_object_store_ca_file",
+               help=("Location of ca certificate file to use for object-store "
+                     "client requests via public endpoint"))
+]
+
+public_endpoint_cert_group = cfg.OptGroup(
+    name="object_store_access", title="Auth options for Swift access from VM")
+
+CONF.register_group(public_endpoint_cert_group)
+CONF.register_opts(opts, group=public_endpoint_cert_group)
+
 
 def retrieve_tenant():
     return context.current().tenant_name
@@ -55,3 +70,33 @@ def get_swift_configs():
 
 def read_default_swift_configs():
     return x.load_hadoop_xml_defaults('swift/resources/conf-template.xml')
+
+
+def install_ssl_certs(instances):
+    certs = []
+    if CONF.object_store_access.public_identity_ca_file:
+        certs.append(CONF.object_store_access.public_identity_ca_file)
+    if CONF.object_store_access.public_object_store_ca_file:
+        certs.append(CONF.object_store_access.public_object_store_ca_file)
+    if not certs:
+        return
+    with context.ThreadGroup() as tg:
+        for inst in instances:
+            tg.spawn("configure-ssl-cert-%s" % inst.instance_id,
+                     _install_ssl_certs, inst, certs)
+
+
+def _install_ssl_certs(instance, certs):
+    register_cmd = (
+        "sudo su - -c \"keytool -import -alias sahara-%d -keystore "
+        "`cut -f2 -d \\\"=\\\" /etc/profile.d/99-java.sh | head -1`"
+        "/lib/security/cacerts -file /tmp/cert.pem -noprompt -storepass "
+        "changeit\"")
+    with instance.remote() as r:
+        for idx, cert in enumerate(certs):
+            data = open(cert).read()
+            r.write_file_to("/tmp/cert.pem", data)
+            try:
+                r.execute_command(register_cmd % idx)
+            finally:
+                r.execute_command("rm /tmp/cert.pem")