From d7f17930919644bb44858956f690f9383de35d17 Mon Sep 17 00:00:00 2001 From: Michael Lelyakin Date: Wed, 3 Aug 2016 16:02:18 +0000 Subject: [PATCH] Remove hardcoded password from db schema This patch remove hardcoded password from file "plugins/ "vanilla/hadoop2/resources/create_hive_db.sql". Now we use castellan service to store random-generated password. Closes-bug: 1498035 Change-Id: Ib354ef9d24df4eb19788b1cd7dbc495d0dada55a co-authored-by: Michael Ionkin --- ...d-password-from-hive-eb923b518974e853.yaml | 5 +++ sahara/plugins/vanilla/hadoop2/config.py | 5 ++- .../hadoop2/resources/create_hive_db.sql | 8 ++--- sahara/plugins/vanilla/hadoop2/run_scripts.py | 3 ++ sahara/plugins/vanilla/hadoop2/utils.py | 17 +++++++++ .../plugins/vanilla/hadoop2/test_utils.py | 36 +++++++++++++++++++ 6 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 releasenotes/notes/remove-hardcoded-password-from-hive-eb923b518974e853.yaml diff --git a/releasenotes/notes/remove-hardcoded-password-from-hive-eb923b518974e853.yaml b/releasenotes/notes/remove-hardcoded-password-from-hive-eb923b518974e853.yaml new file mode 100644 index 00000000..598c7620 --- /dev/null +++ b/releasenotes/notes/remove-hardcoded-password-from-hive-eb923b518974e853.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - Fixed issues with hardcoded password during starting hive process, + bug 1498035. + diff --git a/sahara/plugins/vanilla/hadoop2/config.py b/sahara/plugins/vanilla/hadoop2/config.py index 64e7f529..b624abe6 100644 --- a/sahara/plugins/vanilla/hadoop2/config.py +++ b/sahara/plugins/vanilla/hadoop2/config.py @@ -23,6 +23,7 @@ from sahara.i18n import _LW from sahara.plugins import utils from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper from sahara.plugins.vanilla.hadoop2 import oozie_helper as o_helper +from sahara.plugins.vanilla.hadoop2 import utils as u from sahara.plugins.vanilla import utils as vu from sahara.service.castellan import utils as key_manager from sahara.swift import swift_helper as swift @@ -162,6 +163,8 @@ def _get_hadoop_configs(pctx, instance): hive_hostname = vu.get_instance_hostname(vu.get_hiveserver(cluster)) if hive_hostname: + hive_pass = u.get_hive_password(cluster) + hive_cfg = { 'hive.warehouse.subdir.inherit.perms': True, 'javax.jdo.option.ConnectionURL': @@ -175,7 +178,7 @@ def _get_hadoop_configs(pctx, instance): 'javax.jdo.option.ConnectionDriverName': 'com.mysql.jdbc.Driver', 'javax.jdo.option.ConnectionUserName': 'hive', - 'javax.jdo.option.ConnectionPassword': 'pass', + 'javax.jdo.option.ConnectionPassword': hive_pass, 'datanucleus.autoCreateSchema': 'false', 'datanucleus.fixedDatastore': 'true', 'hive.metastore.uris': 'thrift://%s:9083' % hive_hostname, diff --git a/sahara/plugins/vanilla/hadoop2/resources/create_hive_db.sql b/sahara/plugins/vanilla/hadoop2/resources/create_hive_db.sql index 741800dc..b1c2459c 100644 --- a/sahara/plugins/vanilla/hadoop2/resources/create_hive_db.sql +++ b/sahara/plugins/vanilla/hadoop2/resources/create_hive_db.sql @@ -1,9 +1,9 @@ CREATE DATABASE metastore; USE metastore; SOURCE /opt/hive/scripts/metastore/upgrade/mysql/hive-schema-0.10.0.mysql.sql; -CREATE USER 'hive'@'localhost' IDENTIFIED BY 'pass'; +CREATE USER 'hive'@'localhost' IDENTIFIED BY '{{password}}'; REVOKE ALL PRIVILEGES, GRANT OPTION FROM 'hive'@'localhost'; -GRANT ALL PRIVILEGES ON metastore.* TO 'hive'@'localhost' IDENTIFIED BY 'pass'; -GRANT ALL PRIVILEGES ON metastore.* TO 'hive'@'%' IDENTIFIED BY 'pass'; +GRANT ALL PRIVILEGES ON metastore.* TO 'hive'@'localhost' IDENTIFIED BY '{{password}}'; +GRANT ALL PRIVILEGES ON metastore.* TO 'hive'@'%' IDENTIFIED BY '{{password}}'; FLUSH PRIVILEGES; -exit \ No newline at end of file +exit diff --git a/sahara/plugins/vanilla/hadoop2/run_scripts.py b/sahara/plugins/vanilla/hadoop2/run_scripts.py index f41e3c6f..3ed149ab 100644 --- a/sahara/plugins/vanilla/hadoop2/run_scripts.py +++ b/sahara/plugins/vanilla/hadoop2/run_scripts.py @@ -23,6 +23,7 @@ from sahara.i18n import _LI from sahara.plugins import utils as pu from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper from sahara.plugins.vanilla.hadoop2 import oozie_helper +from sahara.plugins.vanilla.hadoop2 import utils as u from sahara.plugins.vanilla import utils as vu from sahara.utils import cluster_progress_ops as cpo from sahara.utils import edp @@ -232,6 +233,8 @@ def start_hiveserver_process(pctx, instance): 'plugins/vanilla/hadoop2/resources/create_hive_db.sql' ) + sql_script = sql_script.replace( + '{{password}}', u.get_hive_password(instance.cluster)) r.write_file_to('/tmp/create_hive_db.sql', sql_script) _hive_create_db(r) _hive_metastore_start(r) diff --git a/sahara/plugins/vanilla/hadoop2/utils.py b/sahara/plugins/vanilla/hadoop2/utils.py index fb1f4ee0..b5d5c6b6 100644 --- a/sahara/plugins/vanilla/hadoop2/utils.py +++ b/sahara/plugins/vanilla/hadoop2/utils.py @@ -71,3 +71,20 @@ def delete_oozie_password(cluster): castellan.delete_secret(extra['oozie_pass_id']) else: LOG.warning(_LW("Cluster hasn't Oozie password")) + + +def get_hive_password(cluster): + cluster = conductor.cluster_get(context.ctx(), cluster) + extra = cluster.extra.to_dict() + if 'hive_pass_id' not in extra: + extra['hive_pass_id'] = u.generate_random_password() + conductor.cluster_update(context.ctx(), cluster, {'extra': extra}) + return castellan.get_secret(extra['hive_pass_id']) + + +def delete_hive_password(cluster): + extra = cluster.extra.to_dict() + if 'hive_pass_id' in extra: + castellan.delete_secret(extra['hive_pass_id']) + else: + LOG.warning(_LW("Cluster hasn't hive password")) diff --git a/sahara/tests/unit/plugins/vanilla/hadoop2/test_utils.py b/sahara/tests/unit/plugins/vanilla/hadoop2/test_utils.py index 65056888..98b4d32b 100644 --- a/sahara/tests/unit/plugins/vanilla/hadoop2/test_utils.py +++ b/sahara/tests/unit/plugins/vanilla/hadoop2/test_utils.py @@ -103,3 +103,39 @@ class UtilsTestCase(base.SaharaTestCase): cluster.extra.to_dict.return_value = {"oozie_pass_id": "31415926"} u.delete_oozie_password(cluster) delete_secret.assert_called_once_with("31415926") + + @mock.patch('sahara.conductor.API.cluster_get') + @mock.patch('sahara.service.castellan.utils.get_secret') + @mock.patch('sahara.service.castellan.utils.store_secret') + @mock.patch('sahara.conductor.API.cluster_update') + def test_get_hive_password(self, cluster_update, + store_secret, get_secret, conductor): + cluster = mock.MagicMock() + cluster.extra.to_dict.return_value = {"hive_pass_id": "31415926"} + + conductor.return_value = cluster + + get_secret.return_value = "hive_pass" + result = u.get_hive_password(cluster) + + get_secret.assert_called_once_with("31415926") + self.assertEqual('hive_pass', result) + + cluster.extra.to_dict.return_value = {} + + store_secret.return_value = 'hive_pass' + result = u.get_hive_password(cluster) + self.assertEqual('hive_pass', result) + + @mock.patch('sahara.service.castellan.utils.delete_secret') + def test_delete_hive_password(self, delete_secret): + cluster = mock.MagicMock() + + cluster.extra.to_dict.return_value = {} + u.delete_hive_password(cluster) + delete_secret.assert_not_called() + + cluster.extra.to_dict.return_value = {"hive_pass_id": "31415926"} + + u.delete_hive_password(cluster) + delete_secret.assert_called_once_with("31415926")