From d148dd4d551d7635b1015e868ceb54933c78acb1 Mon Sep 17 00:00:00 2001 From: Michael McCune Date: Thu, 7 May 2015 11:48:51 -0400 Subject: [PATCH] Initial key manager implementation This change adds the sahara key manager and converts the proxy passwords and swift passwords to use the castellan interface. * adding sahara key manager * adding castellan to requirements * removing barbicanclient from requirements * removing sahara.utils.keymgr and related tests * adding castellan wrapper configs to sahara list_opts * creating a castellan validate_config to help setup * updating documentation for castellan usage * fixing up tests to work with castellan * converting all proxy password usages to use castellan * converting job binaries to use castellan when user credentials are applied * converting data source to use castellan when user credentials are applied Change-Id: I8cb08a365c6175744970b1037501792fe1ddb0c7 Partial-Implements: blueprint improved-secret-storage Closes-Bug: #1431944 --- .../userdoc/advanced.configuration.guide.rst | 37 +++---- requirements.txt | 2 +- sahara/conductor/manager.py | 88 ++++++++++++++++- sahara/config.py | 8 +- sahara/main.py | 2 + sahara/plugins/vanilla/hadoop2/config.py | 6 +- sahara/service/castellan/__init__.py | 0 sahara/service/castellan/config.py | 53 ++++++++++ .../service/castellan/sahara_key_manager.py | 69 +++++++++++++ .../edp/binary_retrievers/internal_swift.py | 12 ++- .../workflow_creator/workflow_factory.py | 17 ++-- sahara/service/edp/spark/engine.py | 7 +- .../tests/unit/conductor/manager/test_edp.py | 13 +++ .../tests/unit/service/castellan/__init__.py | 0 .../castellan/test_sahara_key_manager.py | 44 +++++++++ .../binary_retrievers/test_internal_swift.py | 2 + .../unit/service/edp/test_job_manager.py | 2 + sahara/tests/unit/service/test_periodic.py | 2 + sahara/tests/unit/utils/test_keymgr.py | 80 ---------------- sahara/utils/keymgr.py | 96 ------------------- sahara/utils/proxy.py | 20 +++- 21 files changed, 348 insertions(+), 212 deletions(-) create mode 100644 sahara/service/castellan/__init__.py create mode 100644 sahara/service/castellan/config.py create mode 100644 sahara/service/castellan/sahara_key_manager.py create mode 100644 sahara/tests/unit/service/castellan/__init__.py create mode 100644 sahara/tests/unit/service/castellan/test_sahara_key_manager.py delete mode 100644 sahara/tests/unit/utils/test_keymgr.py delete mode 100644 sahara/utils/keymgr.py diff --git a/doc/source/userdoc/advanced.configuration.guide.rst b/doc/source/userdoc/advanced.configuration.guide.rst index 32130e93..fd6fcedf 100644 --- a/doc/source/userdoc/advanced.configuration.guide.rst +++ b/doc/source/userdoc/advanced.configuration.guide.rst @@ -186,39 +186,42 @@ These options will also be present in the generated sample configuration file. For instructions on creating the configuration file please see the :doc:`configuration.guide`. -External key manager usage (EXPERIMENTAL) ------------------------------------------ +External key manager usage +-------------------------- Sahara generates and stores several passwords during the course of operation. To harden sahara's usage of passwords it can be instructed to use an external key manager for storage and retrieval of these secrets. To enable this feature there must first be an OpenStack Key Manager service deployed -within the stack. Currently, the barbican project is the only key manager -supported by sahara. +within the stack. With a Key Manager service deployed on the stack, sahara must be configured -to enable the external storage of secrets. This is accomplished by editing -the sahara configuration file as follows: +to enable the external storage of secrets. Sahara uses the +`castellan `_ library +to interface with the OpenStack Key Manager service. This library provides +configurable access to a key manager. To configure sahara to use barbican as +the key manager, edit the sahara configuration file as follows: .. sourcecode:: cfg [DEFAULT] - use_external_key_manager=True + use_barbican_key_manager=True -.. TODO (mimccune) - this language should be removed once a new keystone authentication - section has been created in the configuration file. +Enabling the ``use_barbican_key_manager`` option will configure castellan +to use barbican as its key management implementation. By default it will +attempt to find barbican in the Identity service's service catalog. -Additionally, at this time there are two more values which must be provided -to ensure proper access for sahara to the Key Manager service. These are -the Identity domain for the administrative user and the domain for the -administrative project. By default these values will appear as: +For added control of the barbican server location, optional configuration +values may be added to specify the URL for the barbican API server. .. sourcecode:: cfg - [DEFAULT] - admin_user_domain_name=default - admin_project_domain_name=default + [castellan] + barbican_api_endpoint=http://{barbican controller IP:PORT}/ + barbican_api_version=v1 + +The specific values for the barbican endpoint will be dictated by the +IP address of the controller for your installation. With all of these values configured and the Key Manager service deployed, sahara will begin storing its secrets in the external manager. diff --git a/requirements.txt b/requirements.txt index c3819132..95f4f000 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ pbr>=1.6 alembic>=0.8.0 Babel>=1.3 +castellan>=0.3.1 # Apache-2.0 eventlet>=0.17.4 Flask<1.0,>=0.10 iso8601>=0.1.9 @@ -27,7 +28,6 @@ oslo.service>=1.0.0 # Apache-2.0 oslo.utils>=3.2.0 # Apache-2.0 paramiko>=1.13.0 requests!=2.9.0,>=2.8.1 -python-barbicanclient>=3.3.0 python-cinderclient>=1.3.1 python-keystoneclient!=1.8.0,>=1.6.0 python-manilaclient>=1.3.0 diff --git a/sahara/conductor/manager.py b/sahara/conductor/manager.py index 4d58973c..7d4d72c2 100644 --- a/sahara/conductor/manager.py +++ b/sahara/conductor/manager.py @@ -17,6 +17,10 @@ import copy +from castellan.common.objects import passphrase +from castellan import key_manager +from oslo_config import cfg + from sahara.conductor import resource as r from sahara.db import base as db_base from sahara.service import shares @@ -24,6 +28,8 @@ from sahara.utils import configs from sahara.utils import crypto +CONF = cfg.CONF + CLUSTER_DEFAULTS = { "cluster_configs": {}, "status": "undefined", @@ -381,20 +387,64 @@ class ConductorManager(db_base.Base): def data_source_create(self, context, values): """Create a Data Source from the values dictionary.""" + values = copy.deepcopy(values) values = _apply_defaults(values, DATA_SOURCE_DEFAULTS) values['tenant_id'] = context.tenant_id - + # if credentials are being passed in, we use the key_manager + # to store the password. + if (values.get('credentials') and + values['credentials'].get('password')): + key = passphrase.Passphrase(values['credentials']['password']) + password = key_manager.API().store(context, key) + values['credentials']['password'] = password return self.db.data_source_create(context, values) def data_source_destroy(self, context, data_source): """Destroy the Data Source or raise if it does not exist.""" + + # in cases where the credentials to access the data source are + # stored with the record and the external key manager is being + # used, we need to delete the key from the external manager. + if (CONF.use_barbican_key_manager and not + CONF.use_domain_for_proxy_users): + ds_record = self.data_source_get(context, data_source) + if (ds_record.get('credentials') and + ds_record['credentials'].get('password')): + key_manager.API().delete(context, + ds_record['credentials']['password']) return self.db.data_source_destroy(context, data_source) def data_source_update(self, context, id, values): """Update the Data Source or raise if it does not exist.""" + values = copy.deepcopy(values) values["id"] = id + # in cases where the credentials to access the data source are + # stored with the record and the external key manager is being + # used, we need to delete the old key from the manager and + # create a new one. the other option here would be to retrieve + # the previous key and check to see if it has changed, but it + # seems less expensive to just delete the old and create a new + # one. + # it should be noted that the jsonschema validation ensures that + # if the proxy domain is not in use then credentials must be + # sent with this record. + if (CONF.use_barbican_key_manager and not + CONF.use_domain_for_proxy_users): + # first we retrieve the original record to get the old key + # uuid, and delete it. + ds_record = self.data_source_get(context, id) + if (ds_record.get('credentials') and + ds_record['credentials'].get('password')): + key_manager.API().delete(context, + ds_record['credentials']['password']) + # next we create the new key. + if (values.get('credentials') and + values['credentials'].get('password')): + key = passphrase.Passphrase(values['credentials']['password']) + password = key_manager.API().store(context, key) + values['credentials']['password'] = password return self.db.data_source_update(context, values) # JobExecution ops @@ -489,10 +539,26 @@ class ConductorManager(db_base.Base): values = copy.deepcopy(values) values = _apply_defaults(values, JOB_BINARY_DEFAULTS) values['tenant_id'] = context.tenant_id + # if credentials are being passed in, we use the key_manager + # to store the password. + if values.get('extra') and values['extra'].get('password'): + key = passphrase.Passphrase(values['extra']['password']) + password = key_manager.API().store(context, key) + values['extra']['password'] = password return self.db.job_binary_create(context, values) def job_binary_destroy(self, context, job_binary): """Destroy the JobBinary or raise if it does not exist.""" + + # in cases where the credentials to access the job binary are + # stored with the record and the external key manager is being + # used, we need to delete the key from the external manager. + if (CONF.use_barbican_key_manager and not + CONF.use_domain_for_proxy_users): + jb_record = self.job_binary_get(context, job_binary) + if jb_record.get('extra') and jb_record['extra'].get('password'): + key_manager.API().delete(context, + jb_record['extra']['password']) self.db.job_binary_destroy(context, job_binary) def job_binary_update(self, context, id, values): @@ -500,6 +566,26 @@ class ConductorManager(db_base.Base): values = copy.deepcopy(values) values['id'] = id + # in cases where the credentials to access the job binary are + # stored with the record and the external key manager is being + # used, we need to delete the old key from the manager and + # create a new one. the other option here would be to retrieve + # the previous key and check to see if it has changed, but it + # seems less expensive to just delete the old and create a new + # one. + if (CONF.use_barbican_key_manager and not + CONF.use_domain_for_proxy_users): + # first we retrieve the original record to get the old key + # uuid, and delete it. + jb_record = self.job_binary_get(context, id) + if jb_record.get('extra') and jb_record['extra'].get('password'): + key_manager.API().delete(context, + jb_record['extra']['password']) + # next we create the new key. + if values.get('extra') and values['extra'].get('password'): + key = passphrase.Passphrase(values['extra']['password']) + password = key_manager.API().store(context, key) + values['extra']['password'] = password return self.db.job_binary_update(context, values) # JobBinaryInternal ops diff --git a/sahara/config.py b/sahara/config.py index 6d2a4447..785b1856 100644 --- a/sahara/config.py +++ b/sahara/config.py @@ -23,6 +23,7 @@ from oslo_log import log from sahara import exceptions as ex from sahara.i18n import _ from sahara.plugins import base as plugins_base +from sahara.service.castellan import config as castellan from sahara.topology import topology_helper from sahara.utils.notification import sender from sahara.utils.openstack import cinder @@ -163,7 +164,8 @@ def list_opts(): heat_engine.heat_engine_opts, templates.heat_engine_opts, sessions.sessions_opts, - ssh_remote.ssh_config_options)), + ssh_remote.ssh_config_options, + castellan.opts)), (poll_utils.timeouts.name, itertools.chain(poll_utils.timeouts_opts)), (api.conductor_group.name, @@ -183,7 +185,9 @@ def list_opts(): (base.retries.name, itertools.chain(base.opts)), (swift_helper.public_endpoint_cert_group.name, - itertools.chain(swift_helper.opts)) + itertools.chain(swift_helper.opts)), + (castellan.castellan_group.name, + itertools.chain(castellan.castellan_opts)) ] diff --git a/sahara/main.py b/sahara/main.py index 3bbcdaef..df947c48 100644 --- a/sahara/main.py +++ b/sahara/main.py @@ -28,6 +28,7 @@ from sahara.i18n import _LI from sahara.i18n import _LW from sahara.plugins import base as plugins_base from sahara.service import api as service_api +from sahara.service.castellan import config as castellan from sahara.service.edp import api as edp_api from sahara.service import ops as service_ops from sahara.service import periodic @@ -82,6 +83,7 @@ def setup_common(possible_topdir, service_name): # Validate other configurations (that may produce logs) here cinder.validate_config() + castellan.validate_config() if service_name != 'all-in-one' or cfg.CONF.enable_notifications: messaging.setup() diff --git a/sahara/plugins/vanilla/hadoop2/config.py b/sahara/plugins/vanilla/hadoop2/config.py index f62a7cca..94543401 100644 --- a/sahara/plugins/vanilla/hadoop2/config.py +++ b/sahara/plugins/vanilla/hadoop2/config.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from castellan import key_manager from oslo_config import cfg from oslo_log import log as logging import six @@ -182,9 +183,12 @@ def _get_hadoop_configs(pctx, instance): proxy_configs = cluster.cluster_configs.get('proxy_configs') if proxy_configs and c_helper.is_swift_enabled(pctx, cluster): + key = key_manager.API().get(context.current(), + proxy_configs['proxy_password']) + password = key.get_encoded() hive_cfg.update({ swift.HADOOP_SWIFT_USERNAME: proxy_configs['proxy_username'], - swift.HADOOP_SWIFT_PASSWORD: proxy_configs['proxy_password'], + swift.HADOOP_SWIFT_PASSWORD: password, swift.HADOOP_SWIFT_TRUST_ID: proxy_configs['proxy_trust_id'], swift.HADOOP_SWIFT_DOMAIN_NAME: CONF.proxy_user_domain_name }) diff --git a/sahara/service/castellan/__init__.py b/sahara/service/castellan/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sahara/service/castellan/config.py b/sahara/service/castellan/config.py new file mode 100644 index 00000000..38025736 --- /dev/null +++ b/sahara/service/castellan/config.py @@ -0,0 +1,53 @@ +# Copyright (c) 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from castellan import options as castellan +from oslo_config import cfg + +from sahara.utils.openstack import base as utils + + +opts = [ + cfg.BoolOpt('use_barbican_key_manager', default=False, + help='Enable the usage of the OpenStack Key Management ' + 'service provided by barbican.'), +] + +castellan_opts = [ + cfg.StrOpt('barbican_api_endpoint', + help='The endpoint to use for connecting to the barbican ' + 'api controller. By default, castellan will use the ' + 'URL from the service catalog.'), + cfg.StrOpt('barbican_api_version', default='v1', + help='Version of the barbican API, for example: "v1"'), +] + +castellan_group = cfg.OptGroup(name='castellan', + title='castellan key manager options') + +CONF = cfg.CONF +CONF.register_group(castellan_group) +CONF.register_opts(opts) +CONF.register_opts(castellan_opts, group=castellan_group) + + +def validate_config(): + if CONF.use_barbican_key_manager: + # NOTE (elmiko) there is no need to set the api_class as castellan + # uses barbican by default. + castellan.set_defaults(CONF, auth_endpoint=utils.retrieve_auth_url()) + else: + castellan.set_defaults(CONF, api_class='sahara.service.castellan.' + 'sahara_key_manager.SaharaKeyManager') diff --git a/sahara/service/castellan/sahara_key_manager.py b/sahara/service/castellan/sahara_key_manager.py new file mode 100644 index 00000000..572662ac --- /dev/null +++ b/sahara/service/castellan/sahara_key_manager.py @@ -0,0 +1,69 @@ +# Copyright (c) 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from castellan.common.objects import passphrase as key +from castellan.key_manager import key_manager as km + + +class SaharaKeyManager(km.KeyManager): + '''Sahara specific key manager + + This manager is a thin wrapper around the secret being stored. It is + intended for backward compatible use only. It will not store keys + or generate UUIDs but instead return the secret that is being stored. + This behavior allows Sahara to continue storing secrets in its database + while using the Castellan key manager abstraction. + ''' + def __init__(self, configuration=None): + pass + + def create_key(self, context, algorithm=None, length=0, + expiration=None, **kwargs): + '''creates a key + + algorithm, length, and expiration are unused by sahara keys. + ''' + return key.Passphrase(passphrase=kwargs.get('passphrase', '')) + + def create_key_pair(self, *args, **kwargs): + pass + + def store(self, context, key, expiration=None, **kwargs): + '''store a key + + in normal usage a store_key will return the UUID of the key as + dictated by the key manager. Sahara would then store this UUID in + its database to use for retrieval. As sahara is not actually using + a key manager in this context it will return the key's payload for + storage. + ''' + return key.get_encoded() + + def get(self, context, key_id, **kwargs): + '''get a key + + since sahara is not actually storing key UUIDs the key_id to this + function should actually be the key payload. this function will + simply return a new SaharaKey based on that value. + ''' + return key.Passphrase(passphrase=key_id) + + def delete(self, context, key_id, **kwargs): + '''delete a key + + as there is no external key manager, this function will not + perform any external actions. therefore, it won't change anything. + ''' + pass diff --git a/sahara/service/edp/binary_retrievers/internal_swift.py b/sahara/service/edp/binary_retrievers/internal_swift.py index 458dd25e..7f9b1b98 100644 --- a/sahara/service/edp/binary_retrievers/internal_swift.py +++ b/sahara/service/edp/binary_retrievers/internal_swift.py @@ -14,10 +14,12 @@ # limitations under the License. import functools +from castellan import key_manager from oslo_config import cfg import six import swiftclient +import sahara.context as context import sahara.exceptions as ex from sahara.i18n import _ from sahara.swift import utils as su @@ -80,12 +82,18 @@ def _validate_job_binary_url(f): def get_raw_data(job_binary, proxy_configs=None): conn_kwargs = {} if proxy_configs: + key = key_manager.API().get(context.current(), + proxy_configs.get('proxy_password')) + password = key.get_encoded() conn_kwargs.update(username=proxy_configs.get('proxy_username'), - password=proxy_configs.get('proxy_password'), + password=password, trust_id=proxy_configs.get('proxy_trust_id')) else: + key = key_manager.API().get(context.current(), + job_binary.extra.get('password')) + password = key.get_encoded() conn_kwargs.update(username=job_binary.extra.get('user'), - password=job_binary.extra.get('password')) + password=password) conn = sw.client(**conn_kwargs) return _get_raw_data(job_binary, conn) diff --git a/sahara/service/edp/oozie/workflow_creator/workflow_factory.py b/sahara/service/edp/oozie/workflow_creator/workflow_factory.py index 6076583d..151a841e 100644 --- a/sahara/service/edp/oozie/workflow_creator/workflow_factory.py +++ b/sahara/service/edp/oozie/workflow_creator/workflow_factory.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from castellan import key_manager from oslo_config import cfg import six @@ -99,10 +100,11 @@ class BaseFactory(object): configs = {} if proxy_configs: + key = key_manager.API().get( + context.current(), proxy_configs.get('proxy_password')) configs[sw.HADOOP_SWIFT_USERNAME] = proxy_configs.get( 'proxy_username') - configs[sw.HADOOP_SWIFT_PASSWORD] = proxy_configs.get( - 'proxy_password') + configs[sw.HADOOP_SWIFT_PASSWORD] = key.get_encoded() configs[sw.HADOOP_SWIFT_TRUST_ID] = proxy_configs.get( 'proxy_trust_id') configs[sw.HADOOP_SWIFT_DOMAIN_NAME] = CONF.proxy_user_domain_name @@ -113,8 +115,9 @@ class BaseFactory(object): if "user" in src.credentials: configs[sw.HADOOP_SWIFT_USERNAME] = src.credentials['user'] if "password" in src.credentials: - configs[ - sw.HADOOP_SWIFT_PASSWORD] = src.credentials['password'] + key = key_manager.API().get( + context.current(), src.credentials['password']) + configs[sw.HADOOP_SWIFT_PASSWORD] = key.get_encoded() break return configs @@ -222,10 +225,12 @@ class JavaFactory(BaseFactory): configs = {} if proxy_configs: + key = key_manager.API().get( + context.current(), proxy_configs.get('proxy_password')) + password = key.get_encoded() configs[sw.HADOOP_SWIFT_USERNAME] = proxy_configs.get( 'proxy_username') - configs[sw.HADOOP_SWIFT_PASSWORD] = proxy_configs.get( - 'proxy_password') + configs[sw.HADOOP_SWIFT_PASSWORD] = password configs[sw.HADOOP_SWIFT_TRUST_ID] = proxy_configs.get( 'proxy_trust_id') configs[sw.HADOOP_SWIFT_DOMAIN_NAME] = CONF.proxy_user_domain_name diff --git a/sahara/service/edp/spark/engine.py b/sahara/service/edp/spark/engine.py index 605f604e..3ef4dfd7 100644 --- a/sahara/service/edp/spark/engine.py +++ b/sahara/service/edp/spark/engine.py @@ -17,6 +17,7 @@ import os import uuid +from castellan import key_manager from oslo_config import cfg import six @@ -116,10 +117,12 @@ class SparkJobEngine(base_engine.JobEngine): proxy_configs = job_configs.get('proxy_configs') configs = {} if proxy_configs: + key = key_manager.API().get( + context.current(), proxy_configs.get('proxy_password')) + password = key.get_encoded() configs[sw.HADOOP_SWIFT_USERNAME] = proxy_configs.get( 'proxy_username') - configs[sw.HADOOP_SWIFT_PASSWORD] = proxy_configs.get( - 'proxy_password') + configs[sw.HADOOP_SWIFT_PASSWORD] = password configs[sw.HADOOP_SWIFT_TRUST_ID] = proxy_configs.get( 'proxy_trust_id') configs[sw.HADOOP_SWIFT_DOMAIN_NAME] = CONF.proxy_user_domain_name diff --git a/sahara/tests/unit/conductor/manager/test_edp.py b/sahara/tests/unit/conductor/manager/test_edp.py index 0ec87540..8dffad30 100644 --- a/sahara/tests/unit/conductor/manager/test_edp.py +++ b/sahara/tests/unit/conductor/manager/test_edp.py @@ -21,6 +21,7 @@ import testtools from sahara import context from sahara import exceptions as ex +from sahara.service.castellan import config as castellan import sahara.tests.unit.conductor.base as test_base from sahara.tests.unit.conductor.manager import test_clusters from sahara.utils import edp @@ -121,6 +122,10 @@ class DataSourceTest(test_base.ConductorManagerTestCase): lambda: SAMPLE_DATA_SOURCE ], *args, **kwargs) + def setUp(self): + super(DataSourceTest, self).setUp() + castellan.validate_config() + def test_crud_operation_create_list_delete(self): ctx = context.ctx() self.api.data_source_create(ctx, SAMPLE_DATA_SOURCE) @@ -319,6 +324,10 @@ class DataSourceTest(test_base.ConductorManagerTestCase): class JobExecutionTest(test_base.ConductorManagerTestCase): + def setUp(self): + super(JobExecutionTest, self).setUp() + castellan.validate_config() + def test_crud_operation_create_list_delete_update(self): ctx = context.ctx() job = self.api.job_create(ctx, SAMPLE_JOB) @@ -793,6 +802,10 @@ class JobBinaryTest(test_base.ConductorManagerTestCase): lambda: SAMPLE_JOB_BINARY ], *args, **kwargs) + def setUp(self): + super(JobBinaryTest, self).setUp() + castellan.validate_config() + def test_crud_operation_create_list_delete(self): ctx = context.ctx() diff --git a/sahara/tests/unit/service/castellan/__init__.py b/sahara/tests/unit/service/castellan/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sahara/tests/unit/service/castellan/test_sahara_key_manager.py b/sahara/tests/unit/service/castellan/test_sahara_key_manager.py new file mode 100644 index 00000000..416bc146 --- /dev/null +++ b/sahara/tests/unit/service/castellan/test_sahara_key_manager.py @@ -0,0 +1,44 @@ +# Copyright (c) 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from castellan.common.objects import passphrase as key + +from sahara.service.castellan import sahara_key_manager +from sahara.tests.unit import base + + +class SaharaKeyManagerTest(base.SaharaTestCase): + + def setUp(self): + super(SaharaKeyManagerTest, self).setUp() + self.k_m = sahara_key_manager.SaharaKeyManager() + self.ctx = None + + def test_create_key(self): + k = self.k_m.create_key(self.ctx, passphrase='super_secret') + self.assertEqual('super_secret', k.get_encoded()) + + k = self.k_m.create_key(self.ctx) + self.assertEqual('', k.get_encoded()) + + def test_store(self): + k = key.Passphrase(passphrase='super_secret') + k_id = self.k_m.store(self.ctx, k) + self.assertEqual('super_secret', k_id) + + def test_get(self): + k_id = 'super_secret' + k = self.k_m.get(self.ctx, k_id) + self.assertEqual(k_id, k.get_encoded()) diff --git a/sahara/tests/unit/service/edp/binary_retrievers/test_internal_swift.py b/sahara/tests/unit/service/edp/binary_retrievers/test_internal_swift.py index efbebdc5..f42af25c 100644 --- a/sahara/tests/unit/service/edp/binary_retrievers/test_internal_swift.py +++ b/sahara/tests/unit/service/edp/binary_retrievers/test_internal_swift.py @@ -16,6 +16,7 @@ import mock import sahara.exceptions as ex +from sahara.service.castellan import config as castellan from sahara.service.edp.binary_retrievers import internal_swift as i_s from sahara.tests.unit import base @@ -23,6 +24,7 @@ from sahara.tests.unit import base class TestInternalSwift(base.SaharaTestCase): def setUp(self): super(TestInternalSwift, self).setUp() + castellan.validate_config() def test__get_raw_data(self): client_instance = mock.Mock() diff --git a/sahara/tests/unit/service/edp/test_job_manager.py b/sahara/tests/unit/service/edp/test_job_manager.py index cd2e4c2d..99ff7f26 100644 --- a/sahara/tests/unit/service/edp/test_job_manager.py +++ b/sahara/tests/unit/service/edp/test_job_manager.py @@ -22,6 +22,7 @@ import testtools from sahara import conductor as cond from sahara import exceptions as ex from sahara.plugins import base as pb +from sahara.service.castellan import config as castellan from sahara.service.edp import job_manager from sahara.service.edp import job_utils from sahara.service.edp.oozie.workflow_creator import workflow_factory @@ -45,6 +46,7 @@ class TestJobManager(base.SaharaWithDbTestCase): super(TestJobManager, self).setUp() p.patch_minidom_writexml() pb.setup_plugins() + castellan.validate_config() @mock.patch('uuid.uuid4') @mock.patch('sahara.utils.remote.get_remote') diff --git a/sahara/tests/unit/service/test_periodic.py b/sahara/tests/unit/service/test_periodic.py index bd4f01d2..9b0aa161 100644 --- a/sahara/tests/unit/service/test_periodic.py +++ b/sahara/tests/unit/service/test_periodic.py @@ -20,6 +20,7 @@ from oslo_utils import timeutils from sahara.conductor import manager from sahara import context +from sahara.service.castellan import config as castellan import sahara.service.periodic as p import sahara.tests.unit.base as base from sahara.tests.unit.conductor.manager import test_clusters as tc @@ -32,6 +33,7 @@ class TestPeriodicBack(base.SaharaWithDbTestCase): def setUp(self): super(TestPeriodicBack, self).setUp() self.api = manager.ConductorManager() + castellan.validate_config() @mock.patch('sahara.service.edp.job_manager.get_job_status') def test_job_status_update(self, get_job_status): diff --git a/sahara/tests/unit/utils/test_keymgr.py b/sahara/tests/unit/utils/test_keymgr.py deleted file mode 100644 index c3b07c3a..00000000 --- a/sahara/tests/unit/utils/test_keymgr.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright (c) 2015 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock - -from sahara.tests.unit import base -from sahara.utils import keymgr - - -class TestKeymgrUtils(base.SaharaTestCase): - def setUp(self): - super(TestKeymgrUtils, self).setUp() - - @mock.patch('sahara.utils.openstack.barbican.client_for_admin') - def test_keymgr_delete_with_external(self, client_for_admin): - self.override_config('use_external_key_manager', True) - keyref = 'test_key_reference' - secrets_manager = mock.Mock() - secrets_manager.delete = mock.Mock() - client = mock.Mock(secrets=secrets_manager) - client_for_admin.return_value = client - keymgr.delete(keyref) - secrets_manager.delete.assert_called_with(keyref) - - def test_keymgr_get_no_external(self): - actual_key = 'test_key_super_secret' - # with no external key manager, get should return the argument - keyref = keymgr.get(actual_key) - self.assertEqual(actual_key, keyref) - - @mock.patch('sahara.utils.openstack.barbican.client_for_admin') - def test_keymgr_get_with_external(self, client_for_admin): - self.override_config('use_external_key_manager', True) - actual_key = 'test_key_super_secret' - keyref = 'test_key_reference' - secret = mock.Mock(payload=actual_key) - secrets_manager = mock.Mock() - secrets_manager.get = mock.Mock(return_value=secret) - client = mock.Mock(secrets=secrets_manager) - client_for_admin.return_value = client - # with external key manager, get should return a key from a reference - key = keymgr.get(keyref) - secrets_manager.get.assert_called_with(keyref) - self.assertEqual(actual_key, key) - - def test_keymgr_store_no_external(self): - actual_key = 'test_key_super_secret' - # with no external key manager, store should return the argument - keyref = keymgr.store(actual_key) - self.assertEqual(actual_key, keyref) - - @mock.patch('sahara.utils.openstack.barbican.client_for_admin') - def test_keymgr_store_with_external(self, client_for_admin): - self.override_config('use_external_key_manager', True) - key = 'test_key_super_secret' - actual_keyref = 'test_key_reference' - secret = mock.Mock() - secret.store = mock.Mock(return_value=actual_keyref) - secrets_manager = mock.Mock() - secrets_manager.create = mock.Mock(return_value=secret) - client = mock.Mock(secrets=secrets_manager) - client_for_admin.return_value = client - # with external key manager, store should return a key reference - keyref = keymgr.store(key) - secrets_manager.create.assert_called_with( - payload=key, payload_content_type='text/plain') - secret.store.assert_called_once_with() - self.assertEqual(actual_keyref, keyref) diff --git a/sahara/utils/keymgr.py b/sahara/utils/keymgr.py deleted file mode 100644 index 2a6d7661..00000000 --- a/sahara/utils/keymgr.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (c) 2015 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from oslo_config import cfg -from oslo_log import log as logging - -from sahara.utils.openstack import barbican - - -LOG = logging.getLogger(__name__) -CONF = cfg.CONF - -opts = [ - cfg.BoolOpt('use_external_key_manager', - default=False, - help='Enable Sahara to use an external key manager service ' - 'provided by the identity service catalog. Sahara will ' - 'store all keys with the manager service.') -] -CONF.register_opts(opts) - - -def delete(key_ref): - '''delete a key - - When this function is used without an external key manager it does - nothing to the provided reference. - - :param key_ref: The reference of the key to delete - - ''' - if CONF.use_external_key_manager: - client = barbican.client_for_admin() - client.secrets.delete(key_ref) - LOG.debug('Deleted key {key_ref}'.format(key_ref=key_ref)) - else: - LOG.debug('External key manager not enabled, key not deleted') - - -def get(key_ref): - '''retrieve a key - - When used with an external key manager this will retrieve the key - and return it as stored. - - When used without an external key manager it will return the argument - provided. - - :param key_ref: The reference of the key to retrieve - :returns: The retrieved key - - ''' - if CONF.use_external_key_manager: - client = barbican.client_for_admin() - key = client.secrets.get(key_ref) - LOG.debug('Retrieved key for {key_ref}'.format(key_ref=key_ref)) - payload = key.payload - return payload - else: - return key_ref - - -def store(key): - '''store a key - - When used with an external key manager this function will store the key - in the manager and return a reference provided by the manager. - - When used without an external manager this function will return the - argument provided. - - :param key: The key to store - :returns: A reference for the stored key - - ''' - if CONF.use_external_key_manager: - client = barbican.client_for_admin() - secret = client.secrets.create(payload=key, - payload_content_type='text/plain') - secret_ref = secret.store() - LOG.debug('Stored key as {key_ref}'.format(key_ref=secret_ref)) - return secret_ref - else: - return key diff --git a/sahara/utils/proxy.py b/sahara/utils/proxy.py index 4bbd9ae9..e0714afb 100644 --- a/sahara/utils/proxy.py +++ b/sahara/utils/proxy.py @@ -15,6 +15,8 @@ import uuid +from castellan.common.objects import passphrase +from castellan import key_manager from oslo_config import cfg from oslo_log import log as logging import six @@ -60,7 +62,8 @@ def create_proxy_user_for_job_execution(job_execution): ''' username = 'job_{0}'.format(job_execution.id) - password = proxy_user_create(username) + key = passphrase.Passphrase(proxy_user_create(username)) + password = key_manager.API().store(context.current(), key) current_user = k.auth() proxy_user = k.auth_for_proxy(username, password) trust_id = t.create_trust(trustor=current_user, @@ -85,13 +88,17 @@ def delete_proxy_user_for_job_execution(job_execution): proxy_configs = job_execution.job_configs.get('proxy_configs') if proxy_configs is not None: proxy_username = proxy_configs.get('proxy_username') - proxy_password = proxy_configs.get('proxy_password') + key = key_manager.API().get( + context.current(), proxy_configs.get('proxy_password')) + proxy_password = key.get_encoded() proxy_trust_id = proxy_configs.get('proxy_trust_id') proxy_user = k.auth_for_proxy(proxy_username, proxy_password, proxy_trust_id) t.delete_trust(proxy_user, proxy_trust_id) proxy_user_delete(proxy_username) + key_manager.API().delete(context.current(), + proxy_configs.get('proxy_password')) update = job_execution.job_configs.to_dict() del update['proxy_configs'] return update @@ -107,7 +114,8 @@ def create_proxy_user_for_cluster(cluster): if cluster.cluster_configs.get('proxy_configs'): return cluster username = 'cluster_{0}'.format(cluster.id) - password = proxy_user_create(username) + key = passphrase.Passphrase(proxy_user_create(username)) + password = key_manager.API().store(context.current(), key) current_user = k.auth() proxy_user = k.auth_for_proxy(username, password) trust_id = t.create_trust(trustor=current_user, @@ -131,13 +139,17 @@ def delete_proxy_user_for_cluster(cluster): proxy_configs = cluster.cluster_configs.get('proxy_configs') if proxy_configs is not None: proxy_username = proxy_configs.get('proxy_username') - proxy_password = proxy_configs.get('proxy_password') + key = key_manager.API().get( + context.current(), proxy_configs.get('proxy_password')) + proxy_password = key.get_encoded() proxy_trust_id = proxy_configs.get('proxy_trust_id') proxy_user = k.auth_for_proxy(proxy_username, proxy_password, proxy_trust_id) t.delete_trust(proxy_user, proxy_trust_id) proxy_user_delete(proxy_username) + key_manager.API().delete(context.current(), + proxy_configs.get('proxy_password')) update = {'cluster_configs': cluster.cluster_configs.to_dict()} del update['cluster_configs']['proxy_configs'] conductor.cluster_update(context.ctx(), cluster, update)