From 423d80498b77ca0e5e401e3e7619cd8247adb438 Mon Sep 17 00:00:00 2001 From: Michael McCune Date: Mon, 11 Jan 2016 10:09:10 -0500 Subject: [PATCH] add helper functions for key manager this change adds a utils module to the castellan service package. this module contains 3 wrapper functions to help reduce the overhead for working with castellan. * add sahara.service.castellan.utils module * fixup previous usages of the castellan key manager Change-Id: I6ad4e98ab41788022104ad2886e0ab74e4061ec3 Partial-Implements: blueprint improved-secret-storage --- sahara/conductor/manager.py | 39 ++++++-------- sahara/plugins/vanilla/hadoop2/config.py | 8 ++- sahara/service/castellan/utils.py | 54 +++++++++++++++++++ .../edp/binary_retrievers/internal_swift.py | 16 ++---- .../workflow_creator/workflow_factory.py | 18 +++---- sahara/service/edp/spark/engine.py | 8 ++- sahara/utils/proxy.py | 27 ++++------ 7 files changed, 98 insertions(+), 72 deletions(-) create mode 100644 sahara/service/castellan/utils.py diff --git a/sahara/conductor/manager.py b/sahara/conductor/manager.py index 7d4d72c2..cafa9799 100644 --- a/sahara/conductor/manager.py +++ b/sahara/conductor/manager.py @@ -17,12 +17,11 @@ import copy -from castellan.common.objects import passphrase -from castellan import key_manager from oslo_config import cfg from sahara.conductor import resource as r from sahara.db import base as db_base +from sahara.service.castellan import utils as key_manager from sahara.service import shares from sahara.utils import configs from sahara.utils import crypto @@ -395,9 +394,8 @@ class ConductorManager(db_base.Base): # to store the password. if (values.get('credentials') and values['credentials'].get('password')): - key = passphrase.Passphrase(values['credentials']['password']) - password = key_manager.API().store(context, key) - values['credentials']['password'] = password + values['credentials']['password'] = key_manager.store_secret( + values['credentials']['password'], context) return self.db.data_source_create(context, values) def data_source_destroy(self, context, data_source): @@ -411,8 +409,8 @@ class ConductorManager(db_base.Base): ds_record = self.data_source_get(context, data_source) if (ds_record.get('credentials') and ds_record['credentials'].get('password')): - key_manager.API().delete(context, - ds_record['credentials']['password']) + key_manager.delete_secret( + ds_record['credentials']['password'], context) return self.db.data_source_destroy(context, data_source) def data_source_update(self, context, id, values): @@ -437,14 +435,13 @@ class ConductorManager(db_base.Base): ds_record = self.data_source_get(context, id) if (ds_record.get('credentials') and ds_record['credentials'].get('password')): - key_manager.API().delete(context, - ds_record['credentials']['password']) + key_manager.delete_secret( + ds_record['credentials']['password'], context) # next we create the new key. if (values.get('credentials') and values['credentials'].get('password')): - key = passphrase.Passphrase(values['credentials']['password']) - password = key_manager.API().store(context, key) - values['credentials']['password'] = password + values['credentials']['password'] = key_manager.store_secret( + values['credentials']['password'], context) return self.db.data_source_update(context, values) # JobExecution ops @@ -542,9 +539,8 @@ class ConductorManager(db_base.Base): # if credentials are being passed in, we use the key_manager # to store the password. if values.get('extra') and values['extra'].get('password'): - key = passphrase.Passphrase(values['extra']['password']) - password = key_manager.API().store(context, key) - values['extra']['password'] = password + values['extra']['password'] = key_manager.store_secret( + values['extra']['password'], context) return self.db.job_binary_create(context, values) def job_binary_destroy(self, context, job_binary): @@ -557,8 +553,8 @@ class ConductorManager(db_base.Base): CONF.use_domain_for_proxy_users): jb_record = self.job_binary_get(context, job_binary) if jb_record.get('extra') and jb_record['extra'].get('password'): - key_manager.API().delete(context, - jb_record['extra']['password']) + key_manager.delete_secret(jb_record['extra']['password'], + context) self.db.job_binary_destroy(context, job_binary) def job_binary_update(self, context, id, values): @@ -579,13 +575,12 @@ class ConductorManager(db_base.Base): # uuid, and delete it. jb_record = self.job_binary_get(context, id) if jb_record.get('extra') and jb_record['extra'].get('password'): - key_manager.API().delete(context, - jb_record['extra']['password']) + key_manager.delete_secret(jb_record['extra']['password'], + context) # next we create the new key. if values.get('extra') and values['extra'].get('password'): - key = passphrase.Passphrase(values['extra']['password']) - password = key_manager.API().store(context, key) - values['extra']['password'] = password + values['extra']['password'] = key_manager.store_secret( + values['extra']['password'], context) return self.db.job_binary_update(context, values) # JobBinaryInternal ops diff --git a/sahara/plugins/vanilla/hadoop2/config.py b/sahara/plugins/vanilla/hadoop2/config.py index 94543401..37c88e8f 100644 --- a/sahara/plugins/vanilla/hadoop2/config.py +++ b/sahara/plugins/vanilla/hadoop2/config.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from castellan import key_manager from oslo_config import cfg from oslo_log import log as logging import six @@ -25,6 +24,7 @@ from sahara.plugins import utils from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper from sahara.plugins.vanilla.hadoop2 import oozie_helper as o_helper from sahara.plugins.vanilla import utils as vu +from sahara.service.castellan import utils as key_manager from sahara.swift import swift_helper as swift from sahara.topology import topology_helper as th from sahara.utils import cluster_progress_ops as cpo @@ -183,12 +183,10 @@ def _get_hadoop_configs(pctx, instance): proxy_configs = cluster.cluster_configs.get('proxy_configs') if proxy_configs and c_helper.is_swift_enabled(pctx, cluster): - key = key_manager.API().get(context.current(), - proxy_configs['proxy_password']) - password = key.get_encoded() hive_cfg.update({ swift.HADOOP_SWIFT_USERNAME: proxy_configs['proxy_username'], - swift.HADOOP_SWIFT_PASSWORD: password, + swift.HADOOP_SWIFT_PASSWORD: key_manager.get_secret( + proxy_configs['proxy_password']), swift.HADOOP_SWIFT_TRUST_ID: proxy_configs['proxy_trust_id'], swift.HADOOP_SWIFT_DOMAIN_NAME: CONF.proxy_user_domain_name }) diff --git a/sahara/service/castellan/utils.py b/sahara/service/castellan/utils.py new file mode 100644 index 00000000..e14133e9 --- /dev/null +++ b/sahara/service/castellan/utils.py @@ -0,0 +1,54 @@ +# Copyright (c) 2016 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from castellan.common.objects import passphrase +from castellan import key_manager + +from sahara import context + + +def delete_secret(id, ctx=None): + '''delete a secret from the external key manager + + if no context is provided, the current context is used. + + ''' + if ctx is None: + ctx = context.current() + key_manager.API().delete(ctx, id) + + +def get_secret(id, ctx=None): + '''get a secret associated with an id + + if no context is provided, the current context is used. + + ''' + if ctx is None: + ctx = context.current() + key = key_manager.API().get(ctx, id) + return key.get_encoded() + + +def store_secret(secret, ctx=None): + '''store a secret and return its identifier + + if no context is provided, the current context is used. + + ''' + if ctx is None: + ctx = context.current() + key = passphrase.Passphrase(secret) + return key_manager.API().store(ctx, key) diff --git a/sahara/service/edp/binary_retrievers/internal_swift.py b/sahara/service/edp/binary_retrievers/internal_swift.py index 7f9b1b98..2b7f7d8e 100644 --- a/sahara/service/edp/binary_retrievers/internal_swift.py +++ b/sahara/service/edp/binary_retrievers/internal_swift.py @@ -14,14 +14,13 @@ # limitations under the License. import functools -from castellan import key_manager from oslo_config import cfg import six import swiftclient -import sahara.context as context import sahara.exceptions as ex from sahara.i18n import _ +from sahara.service.castellan import utils as key_manager from sahara.swift import utils as su from sahara.utils.openstack import swift as sw @@ -82,19 +81,14 @@ def _validate_job_binary_url(f): def get_raw_data(job_binary, proxy_configs=None): conn_kwargs = {} if proxy_configs: - key = key_manager.API().get(context.current(), - proxy_configs.get('proxy_password')) - password = key.get_encoded() conn_kwargs.update(username=proxy_configs.get('proxy_username'), - password=password, + password=key_manager.get_secret( + proxy_configs.get('proxy_password')), trust_id=proxy_configs.get('proxy_trust_id')) else: - key = key_manager.API().get(context.current(), - job_binary.extra.get('password')) - password = key.get_encoded() conn_kwargs.update(username=job_binary.extra.get('user'), - password=password) - + password=key_manager.get_secret( + job_binary.extra.get('password'))) conn = sw.client(**conn_kwargs) return _get_raw_data(job_binary, conn) diff --git a/sahara/service/edp/oozie/workflow_creator/workflow_factory.py b/sahara/service/edp/oozie/workflow_creator/workflow_factory.py index 151a841e..1d818178 100644 --- a/sahara/service/edp/oozie/workflow_creator/workflow_factory.py +++ b/sahara/service/edp/oozie/workflow_creator/workflow_factory.py @@ -13,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from castellan import key_manager from oslo_config import cfg import six from sahara import conductor as c from sahara import context +from sahara.service.castellan import utils as key_manager from sahara.service.edp.oozie.workflow_creator import hive_workflow from sahara.service.edp.oozie.workflow_creator import java_workflow from sahara.service.edp.oozie.workflow_creator import mapreduce_workflow @@ -100,11 +100,10 @@ class BaseFactory(object): configs = {} if proxy_configs: - key = key_manager.API().get( - context.current(), proxy_configs.get('proxy_password')) configs[sw.HADOOP_SWIFT_USERNAME] = proxy_configs.get( 'proxy_username') - configs[sw.HADOOP_SWIFT_PASSWORD] = key.get_encoded() + configs[sw.HADOOP_SWIFT_PASSWORD] = key_manager.get_secret( + proxy_configs.get('proxy_password')) configs[sw.HADOOP_SWIFT_TRUST_ID] = proxy_configs.get( 'proxy_trust_id') configs[sw.HADOOP_SWIFT_DOMAIN_NAME] = CONF.proxy_user_domain_name @@ -115,9 +114,8 @@ class BaseFactory(object): if "user" in src.credentials: configs[sw.HADOOP_SWIFT_USERNAME] = src.credentials['user'] if "password" in src.credentials: - key = key_manager.API().get( - context.current(), src.credentials['password']) - configs[sw.HADOOP_SWIFT_PASSWORD] = key.get_encoded() + configs[sw.HADOOP_SWIFT_PASSWORD] = ( + key_manager.get_secret(src.credentials['password'])) break return configs @@ -225,12 +223,10 @@ class JavaFactory(BaseFactory): configs = {} if proxy_configs: - key = key_manager.API().get( - context.current(), proxy_configs.get('proxy_password')) - password = key.get_encoded() configs[sw.HADOOP_SWIFT_USERNAME] = proxy_configs.get( 'proxy_username') - configs[sw.HADOOP_SWIFT_PASSWORD] = password + configs[sw.HADOOP_SWIFT_PASSWORD] = key_manager.get_secret( + proxy_configs.get('proxy_password')) configs[sw.HADOOP_SWIFT_TRUST_ID] = proxy_configs.get( 'proxy_trust_id') configs[sw.HADOOP_SWIFT_DOMAIN_NAME] = CONF.proxy_user_domain_name diff --git a/sahara/service/edp/spark/engine.py b/sahara/service/edp/spark/engine.py index 3ef4dfd7..0563a5b8 100644 --- a/sahara/service/edp/spark/engine.py +++ b/sahara/service/edp/spark/engine.py @@ -17,7 +17,6 @@ import os import uuid -from castellan import key_manager from oslo_config import cfg import six @@ -25,6 +24,7 @@ from sahara import conductor as c from sahara import context from sahara import exceptions as e from sahara.i18n import _ +from sahara.service.castellan import utils as key_manager from sahara.service.edp import base_engine from sahara.service.edp.binary_retrievers import dispatch from sahara.service.edp import hdfs_helper as h @@ -117,12 +117,10 @@ class SparkJobEngine(base_engine.JobEngine): proxy_configs = job_configs.get('proxy_configs') configs = {} if proxy_configs: - key = key_manager.API().get( - context.current(), proxy_configs.get('proxy_password')) - password = key.get_encoded() configs[sw.HADOOP_SWIFT_USERNAME] = proxy_configs.get( 'proxy_username') - configs[sw.HADOOP_SWIFT_PASSWORD] = password + configs[sw.HADOOP_SWIFT_PASSWORD] = key_manager.get_secret( + proxy_configs.get('proxy_password')) configs[sw.HADOOP_SWIFT_TRUST_ID] = proxy_configs.get( 'proxy_trust_id') configs[sw.HADOOP_SWIFT_DOMAIN_NAME] = CONF.proxy_user_domain_name diff --git a/sahara/utils/proxy.py b/sahara/utils/proxy.py index e0714afb..deccab9a 100644 --- a/sahara/utils/proxy.py +++ b/sahara/utils/proxy.py @@ -15,8 +15,6 @@ import uuid -from castellan.common.objects import passphrase -from castellan import key_manager from oslo_config import cfg from oslo_log import log as logging import six @@ -25,6 +23,7 @@ from sahara import conductor as c from sahara import context from sahara import exceptions as ex from sahara.i18n import _ +from sahara.service.castellan import utils as key_manager from sahara.service.edp import job_utils from sahara.service import trusts as t from sahara.swift import utils as su @@ -62,8 +61,7 @@ def create_proxy_user_for_job_execution(job_execution): ''' username = 'job_{0}'.format(job_execution.id) - key = passphrase.Passphrase(proxy_user_create(username)) - password = key_manager.API().store(context.current(), key) + password = key_manager.store_secret(proxy_user_create(username)) current_user = k.auth() proxy_user = k.auth_for_proxy(username, password) trust_id = t.create_trust(trustor=current_user, @@ -88,17 +86,14 @@ def delete_proxy_user_for_job_execution(job_execution): proxy_configs = job_execution.job_configs.get('proxy_configs') if proxy_configs is not None: proxy_username = proxy_configs.get('proxy_username') - key = key_manager.API().get( - context.current(), proxy_configs.get('proxy_password')) - proxy_password = key.get_encoded() proxy_trust_id = proxy_configs.get('proxy_trust_id') proxy_user = k.auth_for_proxy(proxy_username, - proxy_password, + key_manager.get_secret( + proxy_configs.get('proxy_password')), proxy_trust_id) t.delete_trust(proxy_user, proxy_trust_id) proxy_user_delete(proxy_username) - key_manager.API().delete(context.current(), - proxy_configs.get('proxy_password')) + key_manager.delete_secret(proxy_configs.get('proxy_password')) update = job_execution.job_configs.to_dict() del update['proxy_configs'] return update @@ -114,8 +109,7 @@ def create_proxy_user_for_cluster(cluster): if cluster.cluster_configs.get('proxy_configs'): return cluster username = 'cluster_{0}'.format(cluster.id) - key = passphrase.Passphrase(proxy_user_create(username)) - password = key_manager.API().store(context.current(), key) + password = key_manager.store_secret(proxy_user_create(username)) current_user = k.auth() proxy_user = k.auth_for_proxy(username, password) trust_id = t.create_trust(trustor=current_user, @@ -139,17 +133,14 @@ def delete_proxy_user_for_cluster(cluster): proxy_configs = cluster.cluster_configs.get('proxy_configs') if proxy_configs is not None: proxy_username = proxy_configs.get('proxy_username') - key = key_manager.API().get( - context.current(), proxy_configs.get('proxy_password')) - proxy_password = key.get_encoded() proxy_trust_id = proxy_configs.get('proxy_trust_id') proxy_user = k.auth_for_proxy(proxy_username, - proxy_password, + key_manager.get_secret( + proxy_configs.get('proxy_password')), proxy_trust_id) t.delete_trust(proxy_user, proxy_trust_id) proxy_user_delete(proxy_username) - key_manager.API().delete(context.current(), - proxy_configs.get('proxy_password')) + key_manager.delete_secret(proxy_configs.get('proxy_password')) update = {'cluster_configs': cluster.cluster_configs.to_dict()} del update['cluster_configs']['proxy_configs'] conductor.cluster_update(context.ctx(), cluster, update)