Add a custom filter method to scan wrapped dict results

Basic filtering is available via Resource._filter_fields and
Resource._children but partial filtering of fields is not supported.

This change allows a Resource to define custom sanitization methods
for individual fields. The specified method is applied after the
field has been generated.

For nested structures, particularly when elements are not Resource
types, it is easier to scan the field after generation and remove
specific elements than it is to modify the recursive descent already
implemented in to_wrapped_dict().

JobExecutions are filtered with this mechanism to set swift credential
values to empty strings and to remove the 'conf' section reported
for running Oozie actions.

Closes-Bug: #1300291
Change-Id: Ifb0b99a3e13d40306139b0d0021aead69e870205
This commit is contained in:
Trevor McKay 2014-04-02 18:03:24 -04:00
parent bedfb7f3b3
commit 3367d9c363
3 changed files with 51 additions and 6 deletions

View File

@ -28,6 +28,7 @@ import datetime
import six
from sahara.conductor import objects
from sahara.swift import swift_helper
from sahara.utils import types
@ -81,6 +82,7 @@ class Resource(types.FrozenDict):
_resource_name = 'resource'
_children = {}
_filter_fields = []
_sanitize_fields = {}
def __init__(self, dct):
super(Resource, self).__setattr__('_initial_dict', dct)
@ -147,7 +149,9 @@ class Resource(types.FrozenDict):
if refname in self._children:
childs_backref = self._children[refname][1]
dct[refname] = self._entity_to_dict(entity, childs_backref)
sanitize = self._sanitize_fields.get(refname)
if sanitize is not None:
dct[refname] = sanitize(self, dct[refname])
return dct
def _entity_to_dict(self, entity, childs_backref):
@ -213,8 +217,27 @@ class DataSource(Resource, objects.DataSource):
class JobExecution(Resource, objects.JobExecution):
def sanitize_job_configs(self, job_configs):
if 'configs' in job_configs:
configs = job_configs['configs']
if swift_helper.HADOOP_SWIFT_USERNAME in configs:
configs[swift_helper.HADOOP_SWIFT_USERNAME] = ""
if swift_helper.HADOOP_SWIFT_PASSWORD in configs:
configs[swift_helper.HADOOP_SWIFT_PASSWORD] = ""
return job_configs
def sanitize_info(self, info):
if 'actions' in info:
for d in info['actions']:
if 'conf' in d:
del d['conf']
return info
_resource_name = "job_execution"
_filter_fields = ['extra', 'job_configs']
_filter_fields = ['extra']
_sanitize_fields = {'job_configs': sanitize_job_configs,
'info': sanitize_info}
class JobBinary(Resource, objects.JobBinary):

View File

@ -26,6 +26,8 @@ LOG = logging.getLogger(__name__)
CONF = cfg.CONF
HADOOP_SWIFT_AUTH_URL = 'fs.swift.service.sahara.auth.url'
HADOOP_SWIFT_TENANT = 'fs.swift.service.sahara.tenant'
HADOOP_SWIFT_USERNAME = 'fs.swift.service.sahara.username'
HADOOP_SWIFT_PASSWORD = 'fs.swift.service.sahara.password'
def _retrieve_tenant():

View File

@ -18,6 +18,7 @@ import copy
import unittest2
from sahara.conductor import resource as r
from sahara.swift import swift_helper
from sahara.utils import types
@ -100,6 +101,8 @@ SAMPLE_JOB_EXECUTION = {
"extra": {},
"id": "1b0b1874-a261-4d1f-971a-a2cebadeba6c",
"info": {
"actions": [{"conf": "some stuff"},
{"conf": "more stuff"}],
"status": "Pending"
},
"input_id": "b5ddde55-594e-428f-9040-028be81eb3c2",
@ -109,8 +112,9 @@ SAMPLE_JOB_EXECUTION = {
"bill"
],
"configs": {
"fs.swift.service.savanna.password": "openstack",
"fs.swift.service.savanna.username": "admin"
swift_helper.HADOOP_SWIFT_PASSWORD: "openstack",
swift_helper.HADOOP_SWIFT_USERNAME: "admin",
"myfavoriteconfig": 1
}
},
"job_id": "d0f3e397-7bef-42f9-a4db-e5a96059246e",
@ -203,6 +207,22 @@ class TestResource(unittest2.TestCase):
def test_job_execution_filter_credentials(self):
job_exec = r.JobExecution(SAMPLE_JOB_EXECUTION)
wrapped_dict = job_exec.to_wrapped_dict()
self.assertIn('extra', job_exec)
self.assertIn(swift_helper.HADOOP_SWIFT_PASSWORD,
job_exec['job_configs']['configs'])
self.assertIn(swift_helper.HADOOP_SWIFT_USERNAME,
job_exec['job_configs']['configs'])
for a in job_exec['info']['actions']:
self.assertIn('conf', a)
wrapped_dict = job_exec.to_wrapped_dict()['job_execution']
self.assertNotIn('extra', wrapped_dict)
self.assertNotIn('job_configs', wrapped_dict)
configs = wrapped_dict['job_configs']['configs']
self.assertEqual("",
configs[swift_helper.HADOOP_SWIFT_PASSWORD])
self.assertEqual("",
configs[swift_helper.HADOOP_SWIFT_USERNAME])
for a in wrapped_dict['info']['actions']:
self.assertNotIn('conf', a)