Implement Date Filters for Secrets

This CR implements the spec for date filters.  The only difference
between the spec and this CR is the use of the alternative ISO format
(without the "Z") for specifying the dates.  This change was made to
have a more consistent API since the Zulu designation is not used
anywhere in the API where dates are shown to the user.  Additionaly the
libraries used for date-time parsing do not make use of the Zulu
designation either.

Implements: blueprint date-filters
DocImpact
APIImpact

Change-Id: Ic8fbe3d0e8b309bb192aaddf30291d1333756064
This commit is contained in:
Douglas Mendizábal 2016-07-06 12:01:35 -05:00
parent d89e93b290
commit 55912386d5
5 changed files with 427 additions and 21 deletions

View File

@ -10,6 +10,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from oslo_utils import timeutils
import pecan
from six.moves.urllib import parse
@ -49,6 +50,10 @@ def _secret_already_has_data():
pecan.abort(409, u._("Secret already has data, cannot modify it."))
def _bad_query_string_parameters():
pecan.abort(400, u._("URI provided invalid query string parameters."))
def _request_has_twsk_but_no_transport_key_id():
"""Throw exception for bad wrapping parameters.
@ -253,6 +258,60 @@ class SecretsController(controllers.ACLMixin):
self.secret_repo = repo.get_secret_repository()
self.quota_enforcer = quota.QuotaEnforcer('secrets', self.secret_repo)
def _is_valid_date_filter(self, date_filter):
filters = date_filter.split(',')
sorted_filters = dict()
try:
for filter in filters:
if filter.startswith('gt:'):
if sorted_filters.get('gt') or sorted_filters.get('gte'):
return False
sorted_filters['gt'] = timeutils.parse_isotime(filter[3:])
elif filter.startswith('gte:'):
if sorted_filters.get('gt') or sorted_filters.get(
'gte') or sorted_filters.get('eq'):
return False
sorted_filters['gte'] = timeutils.parse_isotime(filter[4:])
elif filter.startswith('lt:'):
if sorted_filters.get('lt') or sorted_filters.get('lte'):
return False
sorted_filters['lt'] = timeutils.parse_isotime(filter[3:])
elif filter.startswith('lte:'):
if sorted_filters.get('lt') or sorted_filters.get(
'lte') or sorted_filters.get('eq'):
return False
sorted_filters['lte'] = timeutils.parse_isotime(filter[4:])
elif sorted_filters.get('eq') or sorted_filters.get(
'gte') or sorted_filters.get('lte'):
return False
else:
sorted_filters['eq'] = timeutils.parse_isotime(filter)
except ValueError:
return False
return True
def _is_valid_sorting(self, sorting):
allowed_keys = ['algorithm', 'bit_length', 'created',
'expiration', 'mode', 'name', 'secret_type', 'status',
'updated']
allowed_directions = ['asc', 'desc']
sorted_keys = dict()
for sort in sorting.split(','):
if ':' in sort:
try:
key, direction = sort.split(':')
except ValueError:
return False
else:
key, direction = sort, 'asc'
if key not in allowed_keys or direction not in allowed_directions:
return False
if sorted_keys.get(key):
return False
else:
sorted_keys[key] = direction
return True
@pecan.expose()
def _lookup(self, secret_id, *remainder):
# NOTE(jaosorior): It's worth noting that even though this section
@ -293,22 +352,33 @@ class SecretsController(controllers.ACLMixin):
# the default should be used.
bits = 0
for date_filter in 'created', 'updated', 'expiration':
if kw.get(date_filter) and not self._is_valid_date_filter(
kw.get(date_filter)):
_bad_query_string_parameters()
if kw.get('sort') and not self._is_valid_sorting(kw.get('sort')):
_bad_query_string_parameters()
ctxt = controllers._get_barbican_context(pecan.request)
user_id = None
if ctxt:
user_id = ctxt.user
result = self.secret_repo.get_by_create_date(
result = self.secret_repo.get_secret_list(
external_project_id,
offset_arg=kw.get('offset', 0),
limit_arg=kw.get('limit', None),
limit_arg=kw.get('limit'),
name=name,
alg=kw.get('alg'),
mode=kw.get('mode'),
bits=bits,
suppress_exception=True,
acl_only=kw.get('acl_only', None),
user_id=user_id
acl_only=kw.get('acl_only'),
user_id=user_id,
created=kw.get('created'),
updated=kw.get('updated'),
expiration=kw.get('expiration'),
sort=kw.get('sort')
)
secrets, offset, limit, total = result

View File

@ -599,17 +599,19 @@ class ProjectRepo(BaseRepo):
class SecretRepo(BaseRepo):
"""Repository for the Secret entity."""
def get_by_create_date(self, external_project_id, offset_arg=None,
limit_arg=None, name=None, alg=None, mode=None,
bits=0, secret_type=None, suppress_exception=False,
session=None, acl_only=None, user_id=None):
def get_secret_list(self, external_project_id,
offset_arg=None, limit_arg=None,
name=None, alg=None, mode=None,
bits=0, secret_type=None, suppress_exception=False,
session=None, acl_only=None, user_id=None,
created=None, updated=None, expiration=None,
sort=None):
"""Returns a list of secrets
The returned secrets are ordered by the date they were created at
and paged based on the offset and limit fields. The external_project_id
is external-to-Barbican value assigned to the project by Keystone.
The list is scoped to secrets that are associated with the
external_project_id (e.g. Keystone Project ID), and filtered
using any provided filters.
"""
offset, limit = clean_paging_values(offset_arg, limit_arg)
session = self.get_session(session)
@ -631,6 +633,19 @@ class SecretRepo(BaseRepo):
query = query.filter(models.Secret.bit_length == bits)
if secret_type:
query = query.filter(models.Secret.secret_type == secret_type)
if created:
query = self._build_date_filter_query(query, 'created_at', created)
if updated:
query = self._build_date_filter_query(query, 'updated_at', updated)
if expiration:
query = self._build_date_filter_query(
query, 'expiration', expiration
)
else:
query = query.filter(or_(models.Secret.expiration.is_(None),
models.Secret.expiration > utcnow))
if sort:
query = self._build_sort_filter_query(query, sort)
if acl_only and acl_only.lower() == 'true' and user_id:
query = query.join(models.SecretACL)
@ -696,6 +711,63 @@ class SecretRepo(BaseRepo):
return query
def _build_date_filter_query(self, query, attribute, date_filters):
"""Parses date_filters to apply each filter to the given query
:param query: query object to apply filters to
:param attribute: name of the model attribute to be filtered
:param date_filters: comma separated string of date filters to apply
"""
parse = timeutils.parse_isotime
for filter in date_filters.split(','):
if filter.startswith('lte:'):
isotime = filter[4:]
query = query.filter(or_(
getattr(models.Secret, attribute) < parse(isotime),
getattr(models.Secret, attribute) == parse(isotime))
)
elif filter.startswith('lt:'):
isotime = filter[3:]
query = query.filter(
getattr(models.Secret, attribute) < parse(isotime)
)
elif filter.startswith('gte:'):
isotime = filter[4:]
query = query.filter(or_(
getattr(models.Secret, attribute) > parse(isotime),
getattr(models.Secret, attribute) == parse(isotime))
)
elif filter.startswith('gt:'):
isotime = filter[3:]
query = query.filter(
getattr(models.Secret, attribute) > parse(isotime)
)
else:
query = query.filter(
getattr(models.Secret, attribute) == parse(filter)
)
return query
def _build_sort_filter_query(self, query, sort_filters):
"""Parses sort_filters to order the query"""
key_to_column_map = {
'created': 'created_at',
'updated': 'updated_at'
}
ordering = list()
for sort in sort_filters.split(','):
if ':' in sort:
key, direction = sort.split(':')
else:
key, direction = sort, 'asc'
ordering.append(
getattr(
getattr(models.Secret, key_to_column_map.get(key, key)),
direction
)()
)
return query.order_by(*ordering)
def get_secret_by_id(self, entity_id, suppress_exception=False,
session=None):
"""Gets secret by its entity id without project id check."""

View File

@ -18,6 +18,7 @@ import os
import mock
from oslo_utils import timeutils
from barbican.api.controllers import secrets
from barbican.common import validators
from barbican.model import models
from barbican.model import repositories
@ -271,6 +272,16 @@ class WhenGettingSecretsList(utils.BarbicanAPIBaseTestCase):
self.assertNotIn('previous', get_resp.json)
self.assertNotIn('next', get_resp.json)
def test_bad_date_filter_results_in_400(self):
params = {'expiration': 'bogus'}
get_resp = self.app.get('/secrets/', params, expect_errors=True)
self.assertEqual(400, get_resp.status_int)
def test_bad_sorting_results_in_400(self):
params = {'sort': 'bogus'}
get_resp = self.app.get('/secrets/', params, expect_errors=True)
self.assertEqual(400, get_resp.status_int)
class WhenGettingPuttingOrDeletingSecret(utils.BarbicanAPIBaseTestCase):
@ -716,6 +727,94 @@ class WhenPerformingUnallowedOperations(utils.BarbicanAPIBaseTestCase):
self.assertEqual(405, resp.status_int)
class WhenValidatingDateFilters(utils.BarbicanAPIBaseTestCase):
def setUp(self):
super(WhenValidatingDateFilters, self).setUp()
self.controller = secrets.SecretsController()
def test_validates_plain_timestamp(self):
date_filter = '2016-01-01T00:00:00'
self.assertTrue(self.controller._is_valid_date_filter(date_filter))
def test_validates_gt_and_lt_timestamps(self):
date_filter = 'gt:2016-01-01T00:00:00,lt:2016-12-31T00:00:00'
self.assertTrue(self.controller._is_valid_date_filter(date_filter))
def test_validates_gte_and_lte_timestamps(self):
date_filter = 'gte:2016-01-01T00:00:00,lte:2016-12-31T00:00:00'
self.assertTrue(self.controller._is_valid_date_filter(date_filter))
def test_validation_fails_with_two_plain_timestamps(self):
date_filter = '2016-01-01T00:00:00,2016-01-02T00:00:00'
self.assertFalse(self.controller._is_valid_date_filter(date_filter))
def test_validation_fails_with_two_gt_timestamps(self):
date_filter = 'gt:2016-01-01T00:00:00,gt:2016-01-02T00:00:00'
self.assertFalse(self.controller._is_valid_date_filter(date_filter))
def test_validation_fails_with_two_lt_timestamps(self):
date_filter = 'lt:2016-01-01T00:00:00,lt:2016-01-02T00:00:00'
self.assertFalse(self.controller._is_valid_date_filter(date_filter))
def test_validation_fails_with_two_gte_timestamps(self):
date_filter = 'gte:2016-01-01T00:00:00,gte:2016-01-02T00:00:00'
self.assertFalse(self.controller._is_valid_date_filter(date_filter))
def test_validation_fails_with_two_lte_timestamps(self):
date_filter = 'lte:2016-01-01T00:00:00,lte:2016-01-02T00:00:00'
self.assertFalse(self.controller._is_valid_date_filter(date_filter))
def test_validation_fails_with_plain_and_gte_timestamps(self):
date_filter = '2016-01-01T00:00:00,gte:2016-01-02T00:00:00'
self.assertFalse(self.controller._is_valid_date_filter(date_filter))
date_filter = 'gte:2016-01-01T00:00:00,2016-01-02T00:00:00'
self.assertFalse(self.controller._is_valid_date_filter(date_filter))
def test_validation_fails_with_plain_and_lte_timestamps(self):
date_filter = '2016-01-01T00:00:00,lte:2016-01-02T00:00:00'
self.assertFalse(self.controller._is_valid_date_filter(date_filter))
date_filter = 'lte:2016-01-01T00:00:00,2016-01-02T00:00:00'
self.assertFalse(self.controller._is_valid_date_filter(date_filter))
def test_validation_fails_with_gt_and_gte_timestamps(self):
date_filter = 'gt:2016-01-01T00:00:00,gte:2016-01-02T00:00:00'
self.assertFalse(self.controller._is_valid_date_filter(date_filter))
def test_validation_fails_with_lt_and_lte_timestamps(self):
date_filter = 'lt:2016-01-01T00:00:00,lte:2016-01-02T00:00:00'
self.assertFalse(self.controller._is_valid_date_filter(date_filter))
def test_validation_fails_with_bogus_timestamp(self):
date_filter = 'bogus'
self.assertFalse(self.controller._is_valid_date_filter(date_filter))
class WhenValidatingSortFilters(utils.BarbicanAPIBaseTestCase):
def setUp(self):
super(WhenValidatingSortFilters, self).setUp()
self.controller = secrets.SecretsController()
def test_validates_name_sorting(self):
sorting = 'name'
self.assertTrue(self.controller._is_valid_sorting(sorting))
def test_validation_fails_for_bogus_attribute(self):
sorting = 'bogus'
self.assertFalse(self.controller._is_valid_sorting(sorting))
def test_validation_fails_for_duplicate_keys(self):
sorting = 'name,name:asc'
self.assertFalse(self.controller._is_valid_sorting(sorting))
def test_validation_fails_for_too_many_colons(self):
sorting = 'name:asc:foo'
self.assertFalse(self.controller._is_valid_sorting(sorting))
# ----------------------- Helper Functions ---------------------------
def create_secret(app, name=None, algorithm=None, bit_length=None, mode=None,
expiration=None, payload=None, content_type=None,

73
barbican/tests/fixture.py Normal file
View File

@ -0,0 +1,73 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import fixtures
from oslo_utils import timeutils
import sqlalchemy as sa
from barbican.model import models
class SessionQueryFixture(fixtures.Fixture):
"""Fixture for testing queries on a session
This fixture creates a SQLAlchemy sessionmaker for an in-memory
sqlite database with sample data.
"""
def _setUp(self):
self._engine = sa.create_engine('sqlite:///:memory:')
self.Session = sa.orm.sessionmaker(bind=self._engine)
self.external_id = 'EXTERNAL_ID'
models.BASE.metadata.create_all(self._engine)
self._load_sample_data()
def _load_sample_data(self):
sess = self.Session()
proj = models.Project()
proj.external_id = self.external_id
sess.add(proj)
sess.commit() # commit to add proj.id
self._add_secret(sess, proj, 'A',
'2016-01-01T00:00:00',
'2016-01-01T00:00:00')
self._add_secret(sess, proj, 'B',
'2016-02-01T00:00:00',
'2016-02-01T00:00:00')
self._add_secret(sess, proj, 'C',
'2016-03-01T00:00:00',
'2016-03-01T00:00:00')
self._add_secret(sess, proj, 'D',
'2016-04-01T00:00:00',
'2016-04-01T00:00:00')
self._add_secret(sess, proj, 'E',
'2016-05-01T00:00:00',
'2016-05-01T00:00:00')
self._add_secret(sess, proj, 'F',
'2016-06-01T00:00:00',
'2016-06-01T00:00:00')
sess.commit() # commit all secrets
def _add_secret(self, session, project, name, created_at, updated_at):
s = models.Secret()
s.name = name
s.created_at = timeutils.parse_isotime(created_at)
s.updated_at = timeutils.parse_isotime(updated_at)
s.project_id = project.id
session.add(s)

View File

@ -10,15 +10,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
import fixtures
import testtools
from barbican.common import exception
from barbican.model import models
from barbican.model import repositories
from barbican.plugin.interface import secret_store as ss
from barbican.tests import database_utils
from barbican.tests import fixture
from barbican.tests import utils
import datetime
@utils.parameterized_test_case
class WhenTestingSecretRepository(database_utils.RepositoryTestCase):
@ -55,7 +59,7 @@ class WhenTestingSecretRepository(database_utils.RepositoryTestCase):
super(WhenTestingSecretRepository, self).setUp()
self.repo = repositories.SecretRepo()
def test_get_by_create_date(self):
def test_get_secret_list(self):
session = self.repo.get_session()
project = models.Project()
@ -68,7 +72,7 @@ class WhenTestingSecretRepository(database_utils.RepositoryTestCase):
session.commit()
secrets, offset, limit, total = self.repo.get_by_create_date(
secrets, offset, limit, total = self.repo.get_secret_list(
"my keystone id",
session=session,
)
@ -103,8 +107,8 @@ class WhenTestingSecretRepository(database_utils.RepositoryTestCase):
suppress_exception=True))
@utils.parameterized_dataset(dataset_for_filter_tests)
def test_get_by_create_date_with_filter(self, secret_1_dict, secret_2_dict,
query_dict):
def test_get_secret_list_with_filter(self, secret_1_dict, secret_2_dict,
query_dict):
session = self.repo.get_session()
project = models.Project()
@ -124,7 +128,7 @@ class WhenTestingSecretRepository(database_utils.RepositoryTestCase):
session.commit()
secrets, offset, limit, total = self.repo.get_by_create_date(
secrets, offset, limit, total = self.repo.get_secret_list(
"my keystone id",
session=session,
**query_dict
@ -138,7 +142,7 @@ class WhenTestingSecretRepository(database_utils.RepositoryTestCase):
def test_get_by_create_date_nothing(self):
session = self.repo.get_session()
secrets, offset, limit, total = self.repo.get_by_create_date(
secrets, offset, limit, total = self.repo.get_secret_list(
"my keystone id",
bits=1024,
session=session,
@ -158,7 +162,7 @@ class WhenTestingSecretRepository(database_utils.RepositoryTestCase):
self.assertRaises(
exception.NotFound,
self.repo.get_by_create_date,
self.repo.get_secret_list,
"my keystone id",
session=session,
suppress_exception=False)
@ -242,3 +246,91 @@ class WhenTestingSecretRepository(database_utils.RepositoryTestCase):
count = self.repo.get_count(project.id, session=session)
self.assertEqual(1, count)
class WhenTestingQueryFilters(testtools.TestCase,
fixtures.TestWithFixtures):
def setUp(self):
super(WhenTestingQueryFilters, self).setUp()
self._session_fixture = self.useFixture(fixture.SessionQueryFixture())
self.session = self._session_fixture.Session()
self.query = self.session.query(models.Secret)
self.repo = repositories.SecretRepo()
def test_data_includes_six_secrets(self):
self.assertEqual(6, len(self.query.all()))
def test_sort_by_name_defaults_ascending(self):
query = self.repo._build_sort_filter_query(self.query, 'name')
secrets = query.all()
self.assertEqual('A', secrets[0].name)
def test_sort_by_name_desc(self):
query = self.repo._build_sort_filter_query(self.query, 'name:desc')
secrets = query.all()
self.assertEqual('F', secrets[0].name)
def test_sort_by_created_asc(self):
query = self.repo._build_sort_filter_query(self.query, 'created:asc')
secrets = query.all()
self.assertEqual('A', secrets[0].name)
def test_sort_by_updated_desc(self):
query = self.repo._build_sort_filter_query(self.query, 'updated:desc')
secrets = query.all()
self.assertEqual('F', secrets[0].name)
def test_filter_by_created_on_new_years(self):
query = self.repo._build_date_filter_query(
self.query, 'created_at',
'2016-01-01T00:00:00'
)
secrets = query.all()
self.assertEqual(1, len(secrets))
self.assertEqual('A', secrets[0].name)
def test_filter_by_created_after_march(self):
query = self.repo._build_date_filter_query(
self.query, 'created_at',
'gt:2016-03-01T00:00:00'
)
secrets = query.all()
self.assertEqual(3, len(secrets))
def test_filter_by_created_on_or_after_march(self):
query = self.repo._build_date_filter_query(
self.query, 'created_at',
'gte:2016-03-01T00:00:00'
)
secrets = query.all()
self.assertEqual(4, len(secrets))
def test_filter_by_created_before_march(self):
query = self.repo._build_date_filter_query(
self.query, 'created_at',
'lt:2016-03-01T00:00:00'
)
secrets = query.all()
self.assertEqual(2, len(secrets))
def test_filter_by_created_on_or_before_march(self):
query = self.repo._build_date_filter_query(
self.query, 'created_at',
'lte:2016-03-01T00:00:00'
)
secrets = query.all()
self.assertEqual(3, len(secrets))
def test_filter_by_created_between_march_and_may_inclusive(self):
query = self.repo._build_date_filter_query(
self.query, 'created_at',
'gte:2016-03-01T00:00:00,lte:2016-05-01T00:00:00'
)
secrets = query.all()
secret_names = [s.name for s in secrets]
self.assertEqual(3, len(secrets))
self.assertIn('C', secret_names)
self.assertIn('D', secret_names)
self.assertIn('E', secret_names)