openstack/deb-sahara
Code Issues Proposed changes

add helper functions for key manager

Browse Source 
this change adds a utils module to the castellan service package. this
module contains 3 wrapper functions to help reduce the overhead for
working with castellan.

* add sahara.service.castellan.utils module
* fixup previous usages of the castellan key manager

Change-Id: I6ad4e98ab41788022104ad2886e0ab74e4061ec3
Partial-Implements: blueprint improved-secret-storage
This commit is contained in:
Michael McCune 2016-01-11 10:09:10 -05:00
parent 7c3ebf703a
commit 423d80498b
7 changed files with 98 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
sahara/conductor/manager.py
View File

@ -17,12 +17,11 @@
import copy
from castellan.common.objects import passphrase
from castellan import key_manager
from oslo_config import cfg
from sahara.conductor import resource as r
from sahara.db import base as db_base
from sahara.service.castellan import utils as key_manager
from sahara.service import shares
from sahara.utils import configs
from sahara.utils import crypto
@ -395,9 +394,8 @@ class ConductorManager(db_base.Base):
# to store the password.
if (values.get('credentials') and
values['credentials'].get('password')):
key = passphrase.Passphrase(values['credentials']['password'])
password = key_manager.API().store(context, key)
values['credentials']['password'] = password
values['credentials']['password'] = key_manager.store_secret(
values['credentials']['password'], context)
return self.db.data_source_create(context, values)
def data_source_destroy(self, context, data_source):
@ -411,8 +409,8 @@ class ConductorManager(db_base.Base):
ds_record = self.data_source_get(context, data_source)
if (ds_record.get('credentials') and
ds_record['credentials'].get('password')):
key_manager.API().delete(context,
ds_record['credentials']['password'])
key_manager.delete_secret(
ds_record['credentials']['password'], context)
return self.db.data_source_destroy(context, data_source)
def data_source_update(self, context, id, values):
@ -437,14 +435,13 @@ class ConductorManager(db_base.Base):
ds_record = self.data_source_get(context, id)
if (ds_record.get('credentials') and
ds_record['credentials'].get('password')):
key_manager.API().delete(context,
ds_record['credentials']['password'])
key_manager.delete_secret(
ds_record['credentials']['password'], context)
# next we create the new key.
if (values.get('credentials') and
values['credentials'].get('password')):
key = passphrase.Passphrase(values['credentials']['password'])
password = key_manager.API().store(context, key)
values['credentials']['password'] = password
values['credentials']['password'] = key_manager.store_secret(
values['credentials']['password'], context)
return self.db.data_source_update(context, values)
# JobExecution ops
@ -542,9 +539,8 @@ class ConductorManager(db_base.Base):
# if credentials are being passed in, we use the key_manager
# to store the password.
if values.get('extra') and values['extra'].get('password'):
key = passphrase.Passphrase(values['extra']['password'])
password = key_manager.API().store(context, key)
values['extra']['password'] = password
values['extra']['password'] = key_manager.store_secret(
values['extra']['password'], context)
return self.db.job_binary_create(context, values)
def job_binary_destroy(self, context, job_binary):
@ -557,8 +553,8 @@ class ConductorManager(db_base.Base):
CONF.use_domain_for_proxy_users):
jb_record = self.job_binary_get(context, job_binary)
if jb_record.get('extra') and jb_record['extra'].get('password'):
key_manager.API().delete(context,
jb_record['extra']['password'])
key_manager.delete_secret(jb_record['extra']['password'],
context)
self.db.job_binary_destroy(context, job_binary)
def job_binary_update(self, context, id, values):
@ -579,13 +575,12 @@ class ConductorManager(db_base.Base):
# uuid, and delete it.
jb_record = self.job_binary_get(context, id)
if jb_record.get('extra') and jb_record['extra'].get('password'):
key_manager.API().delete(context,
jb_record['extra']['password'])
key_manager.delete_secret(jb_record['extra']['password'],
context)
# next we create the new key.
if values.get('extra') and values['extra'].get('password'):
key = passphrase.Passphrase(values['extra']['password'])
password = key_manager.API().store(context, key)
values['extra']['password'] = password
values['extra']['password'] = key_manager.store_secret(
values['extra']['password'], context)
return self.db.job_binary_update(context, values)
# JobBinaryInternal ops

8
sahara/plugins/vanilla/hadoop2/config.py
View File

@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from castellan import key_manager
from oslo_config import cfg
from oslo_log import log as logging
import six
@ -25,6 +24,7 @@ from sahara.plugins import utils
from sahara.plugins.vanilla.hadoop2 import config_helper as c_helper
from sahara.plugins.vanilla.hadoop2 import oozie_helper as o_helper
from sahara.plugins.vanilla import utils as vu
from sahara.service.castellan import utils as key_manager
from sahara.swift import swift_helper as swift
from sahara.topology import topology_helper as th
from sahara.utils import cluster_progress_ops as cpo
@ -183,12 +183,10 @@ def _get_hadoop_configs(pctx, instance):
proxy_configs = cluster.cluster_configs.get('proxy_configs')
if proxy_configs and c_helper.is_swift_enabled(pctx, cluster):
key = key_manager.API().get(context.current(),
proxy_configs['proxy_password'])
password = key.get_encoded()
hive_cfg.update({
swift.HADOOP_SWIFT_USERNAME: proxy_configs['proxy_username'],
swift.HADOOP_SWIFT_PASSWORD: password,
swift.HADOOP_SWIFT_PASSWORD: key_manager.get_secret(
proxy_configs['proxy_password']),
swift.HADOOP_SWIFT_TRUST_ID: proxy_configs['proxy_trust_id'],
swift.HADOOP_SWIFT_DOMAIN_NAME: CONF.proxy_user_domain_name
})

54
sahara/service/castellan/utils.py Normal file
View File

@ -0,0 +1,54 @@
# Copyright (c) 2016 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from castellan.common.objects import passphrase
from castellan import key_manager
from sahara import context
def delete_secret(id, ctx=None):
'''delete a secret from the external key manager
if no context is provided, the current context is used.
'''
if ctx is None:
ctx = context.current()
key_manager.API().delete(ctx, id)
def get_secret(id, ctx=None):
'''get a secret associated with an id
if no context is provided, the current context is used.
'''
if ctx is None:
ctx = context.current()
key = key_manager.API().get(ctx, id)
return key.get_encoded()
def store_secret(secret, ctx=None):
'''store a secret and return its identifier
if no context is provided, the current context is used.
'''
if ctx is None:
ctx = context.current()
key = passphrase.Passphrase(secret)
return key_manager.API().store(ctx, key)

16
sahara/service/edp/binary_retrievers/internal_swift.py
View File

@ -14,14 +14,13 @@
# limitations under the License.
import functools
from castellan import key_manager
from oslo_config import cfg
import six
import swiftclient
import sahara.context as context
import sahara.exceptions as ex
from sahara.i18n import _
from sahara.service.castellan import utils as key_manager
from sahara.swift import utils as su
from sahara.utils.openstack import swift as sw
@ -82,19 +81,14 @@ def _validate_job_binary_url(f):
def get_raw_data(job_binary, proxy_configs=None):
conn_kwargs = {}
if proxy_configs:
key = key_manager.API().get(context.current(),
proxy_configs.get('proxy_password'))
password = key.get_encoded()
conn_kwargs.update(username=proxy_configs.get('proxy_username'),
password=password,
password=key_manager.get_secret(
proxy_configs.get('proxy_password')),
trust_id=proxy_configs.get('proxy_trust_id'))
else:
key = key_manager.API().get(context.current(),
job_binary.extra.get('password'))
password = key.get_encoded()
conn_kwargs.update(username=job_binary.extra.get('user'),
password=password)
password=key_manager.get_secret(
job_binary.extra.get('password')))
conn = sw.client(**conn_kwargs)
return _get_raw_data(job_binary, conn)

18
sahara/service/edp/oozie/workflow_creator/workflow_factory.py
View File

@ -13,12 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from castellan import key_manager
from oslo_config import cfg
import six
from sahara import conductor as c
from sahara import context
from sahara.service.castellan import utils as key_manager
from sahara.service.edp.oozie.workflow_creator import hive_workflow
from sahara.service.edp.oozie.workflow_creator import java_workflow
from sahara.service.edp.oozie.workflow_creator import mapreduce_workflow
@ -100,11 +100,10 @@ class BaseFactory(object):
configs = {}
if proxy_configs:
key = key_manager.API().get(
context.current(), proxy_configs.get('proxy_password'))
configs[sw.HADOOP_SWIFT_USERNAME] = proxy_configs.get(
'proxy_username')
configs[sw.HADOOP_SWIFT_PASSWORD] = key.get_encoded()
configs[sw.HADOOP_SWIFT_PASSWORD] = key_manager.get_secret(
proxy_configs.get('proxy_password'))
configs[sw.HADOOP_SWIFT_TRUST_ID] = proxy_configs.get(
'proxy_trust_id')
configs[sw.HADOOP_SWIFT_DOMAIN_NAME] = CONF.proxy_user_domain_name
@ -115,9 +114,8 @@ class BaseFactory(object):
if "user" in src.credentials:
configs[sw.HADOOP_SWIFT_USERNAME] = src.credentials['user']
if "password" in src.credentials:
key = key_manager.API().get(
context.current(), src.credentials['password'])
configs[sw.HADOOP_SWIFT_PASSWORD] = key.get_encoded()
configs[sw.HADOOP_SWIFT_PASSWORD] = (
key_manager.get_secret(src.credentials['password']))
break
return configs
@ -225,12 +223,10 @@ class JavaFactory(BaseFactory):
configs = {}
if proxy_configs:
key = key_manager.API().get(
context.current(), proxy_configs.get('proxy_password'))
password = key.get_encoded()
configs[sw.HADOOP_SWIFT_USERNAME] = proxy_configs.get(
'proxy_username')
configs[sw.HADOOP_SWIFT_PASSWORD] = password
configs[sw.HADOOP_SWIFT_PASSWORD] = key_manager.get_secret(
proxy_configs.get('proxy_password'))
configs[sw.HADOOP_SWIFT_TRUST_ID] = proxy_configs.get(
'proxy_trust_id')
configs[sw.HADOOP_SWIFT_DOMAIN_NAME] = CONF.proxy_user_domain_name

8
sahara/service/edp/spark/engine.py
View File

@ -17,7 +17,6 @@
import os
import uuid
from castellan import key_manager
from oslo_config import cfg
import six
@ -25,6 +24,7 @@ from sahara import conductor as c
from sahara import context
from sahara import exceptions as e
from sahara.i18n import _
from sahara.service.castellan import utils as key_manager
from sahara.service.edp import base_engine
from sahara.service.edp.binary_retrievers import dispatch
from sahara.service.edp import hdfs_helper as h
@ -117,12 +117,10 @@ class SparkJobEngine(base_engine.JobEngine):
proxy_configs = job_configs.get('proxy_configs')
configs = {}
if proxy_configs:
key = key_manager.API().get(
context.current(), proxy_configs.get('proxy_password'))
password = key.get_encoded()
configs[sw.HADOOP_SWIFT_USERNAME] = proxy_configs.get(
'proxy_username')
configs[sw.HADOOP_SWIFT_PASSWORD] = password
configs[sw.HADOOP_SWIFT_PASSWORD] = key_manager.get_secret(
proxy_configs.get('proxy_password'))
configs[sw.HADOOP_SWIFT_TRUST_ID] = proxy_configs.get(
'proxy_trust_id')
configs[sw.HADOOP_SWIFT_DOMAIN_NAME] = CONF.proxy_user_domain_name

27
sahara/utils/proxy.py
View File

@ -15,8 +15,6 @@
import uuid
from castellan.common.objects import passphrase
from castellan import key_manager
from oslo_config import cfg
from oslo_log import log as logging
import six
@ -25,6 +23,7 @@ from sahara import conductor as c
from sahara import context
from sahara import exceptions as ex
from sahara.i18n import _
from sahara.service.castellan import utils as key_manager
from sahara.service.edp import job_utils
from sahara.service import trusts as t
from sahara.swift import utils as su
@ -62,8 +61,7 @@ def create_proxy_user_for_job_execution(job_execution):
'''
username = 'job_{0}'.format(job_execution.id)
key = passphrase.Passphrase(proxy_user_create(username))
password = key_manager.API().store(context.current(), key)
password = key_manager.store_secret(proxy_user_create(username))
current_user = k.auth()
proxy_user = k.auth_for_proxy(username, password)
trust_id = t.create_trust(trustor=current_user,
@ -88,17 +86,14 @@ def delete_proxy_user_for_job_execution(job_execution):
proxy_configs = job_execution.job_configs.get('proxy_configs')
if proxy_configs is not None:
proxy_username = proxy_configs.get('proxy_username')
key = key_manager.API().get(
context.current(), proxy_configs.get('proxy_password'))
proxy_password = key.get_encoded()
proxy_trust_id = proxy_configs.get('proxy_trust_id')
proxy_user = k.auth_for_proxy(proxy_username,
proxy_password,
key_manager.get_secret(
proxy_configs.get('proxy_password')),
proxy_trust_id)
t.delete_trust(proxy_user, proxy_trust_id)
proxy_user_delete(proxy_username)
key_manager.API().delete(context.current(),
proxy_configs.get('proxy_password'))
key_manager.delete_secret(proxy_configs.get('proxy_password'))
update = job_execution.job_configs.to_dict()
del update['proxy_configs']
return update
@ -114,8 +109,7 @@ def create_proxy_user_for_cluster(cluster):
if cluster.cluster_configs.get('proxy_configs'):
return cluster
username = 'cluster_{0}'.format(cluster.id)
key = passphrase.Passphrase(proxy_user_create(username))
password = key_manager.API().store(context.current(), key)
password = key_manager.store_secret(proxy_user_create(username))
current_user = k.auth()
proxy_user = k.auth_for_proxy(username, password)
trust_id = t.create_trust(trustor=current_user,
@ -139,17 +133,14 @@ def delete_proxy_user_for_cluster(cluster):
proxy_configs = cluster.cluster_configs.get('proxy_configs')
if proxy_configs is not None:
proxy_username = proxy_configs.get('proxy_username')
key = key_manager.API().get(
context.current(), proxy_configs.get('proxy_password'))
proxy_password = key.get_encoded()
proxy_trust_id = proxy_configs.get('proxy_trust_id')
proxy_user = k.auth_for_proxy(proxy_username,
proxy_password,
key_manager.get_secret(
proxy_configs.get('proxy_password')),
proxy_trust_id)
t.delete_trust(proxy_user, proxy_trust_id)
proxy_user_delete(proxy_username)
key_manager.API().delete(context.current(),
proxy_configs.get('proxy_password'))
key_manager.delete_secret(proxy_configs.get('proxy_password'))
update = {'cluster_configs': cluster.cluster_configs.to_dict()}
del update['cluster_configs']['proxy_configs']
conductor.cluster_update(context.ctx(), cluster, update)